From e159005545fc3e92574d47ca96225be5f75d96fb Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:54:34 -0700 Subject: [PATCH 001/155] fix(spend): await the per-request rollup enqueues and report their failures (#48131) * fix(spend): await the per-request rollup enqueues and report their failures The daily and entity spend rollups ran in an unreferenced asyncio task that nothing awaited or drained, so a shutdown after the spend log was enqueued could drop every rollup for that request, and a malformed metadata payload dropped the request from all daily tables at debug level. The rollups now run inline in update_database, each helper failure is logged through spend_log_error with the request id, and unreadable metadata only costs the row its metadata-derived fields Resolves LIT-6027 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(spend): type the rollup failure helpers for the lint gates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(spend): track the rollup enqueue task so shutdown drains it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(spend): shutdown drain waits for an in-flight rollup enqueue Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(spend): register the rollup task without blocking the cost callback Awaiting the rollups inline made a blocked queue put stall increment_spend_counters, so budget checks could run on stale counters. The task is now only registered via spawn_logging_task; drain_logging_tasks still waits for it on shutdown Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: joshua Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/db/db_spend_update_writer.py | 300 ++++++++---------- .../proxy/db/test_db_spend_update_writer.py | 169 ++++++++-- 2 files changed, 274 insertions(+), 195 deletions(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 4e98823ba77..be5b307b5ae 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -12,7 +12,6 @@ import json import os import random import time -import traceback from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable, Mapping, Sequence from contextlib import asynccontextmanager from contextvars import ContextVar @@ -90,6 +89,7 @@ from litellm.proxy.spend_tracking.savings import ( ) from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.repositories.prisma_protocols import BatchTable +from litellm.responses.streaming_iterator import spawn_logging_task from litellm.types.utils import CallTypes if TYPE_CHECKING: @@ -313,6 +313,38 @@ def _spend_tables_left_to_send( return cast(DBSpendUpdateTransactions, remaining) if any(remaining.values()) else None +def _report_spend_update_step_failure(step_name: str, *, payload: Mapping[str, object], exc: Exception) -> None: + spend_log_error( + "Spend tracking - %s failed, this request is missing from that spend rollup. request_id=%s api_key=%s", + step_name, + payload.get("request_id"), + payload.get("api_key"), + exc=exc, + ) + + +_EMPTY_SPEND_LOG_METADATA: Final = cast(SpendLogsMetadata, MappingProxyType({})) # cast-ok: empty read-only metadata + + +def _parse_spend_log_metadata(payload: Mapping[str, object]) -> SpendLogsMetadata: + """Malformed metadata costs the request its metadata-derived fields, never its daily rollup row.""" + raw: Final = payload.get("metadata") + if not isinstance(raw, str): + return _EMPTY_SPEND_LOG_METADATA + try: + parsed: Final[SpendLogsMetadata] = json.loads(raw) + except json.JSONDecodeError as e: + spend_log_error( + "Spend tracking - unreadable spend log metadata, rollups for this request are written without " + "metadata-derived fields. request_id=%s api_key=%s", + payload.get("request_id"), + payload.get("api_key"), + exc=e, + ) + return _EMPTY_SPEND_LOG_METADATA + return parsed + + def _is_billed_retry_attempt(metadata: SpendLogsMetadata) -> bool: """An earlier billed retry attempt's own spend row, not a request the caller sent.""" return BILLED_RETRY_ATTEMPT_METADATA_KEY in metadata and metadata[BILLED_RETRY_ATTEMPT_METADATA_KEY] is True @@ -647,8 +679,8 @@ class DBSpendUpdateWriter: "disable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur." ) - # Single task replaces 11 create_task() calls - asyncio.create_task( + spawn_logging_task( + asyncio.get_running_loop(), self._batch_database_updates( response_cost=response_cost, user_id=user_id, @@ -661,7 +693,8 @@ class DBSpendUpdateWriter: litellm_proxy_budget_name=litellm_proxy_budget_name, payload=payload, request_model_access_groups=get_request_model_access_groups(kwargs), - ) + ), + task_name="spend rollups", ) self._enqueue_tool_registry_upsert( @@ -1086,91 +1119,52 @@ class DBSpendUpdateWriter: project_id: str | None = None, ): """ - Runs all 13 spend-update helpers sequentially inside a single asyncio task. - - Each helper is wrapped in try/except so one failure doesn't prevent the others. - - The deepcopy runs here, off the awaited request path, so the daily spend - helpers get a payload isolated from the spend-log queue entry and the caller. + Runs every spend-update helper for one request. Each helper goes through + _run_spend_update_step so one failure never skips the sibling writes. The deepcopy + isolates the daily spend helpers' payload from the spend-log queue entry and the caller. """ payload_copy: Final = copy.deepcopy(payload) request_tags: Final = payload_copy.get("request_tags") - try: - await self._update_user_db( + await self._run_spend_update_step( + "_update_user_db", + self._update_user_db( response_cost=response_cost, user_id=user_id, prisma_client=prisma_client, litellm_proxy_budget_name=litellm_proxy_budget_name, end_user_id=end_user_id, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_user_db failed: %s", - traceback.format_exc(), - ) - - try: - await self._update_key_db( - response_cost=response_cost, - hashed_token=hashed_token, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_key_db failed: %s", - traceback.format_exc(), - ) - - try: - await self._update_team_db( - response_cost=response_cost, - team_id=team_id, - user_id=user_id, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_team_db failed: %s", - traceback.format_exc(), - ) - - try: - await self._update_org_db( - response_cost=response_cost, - org_id=org_id, - user_id=user_id, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_org_db failed: %s", - traceback.format_exc(), - ) - - try: - await self._update_project_db( - response_cost=response_cost, - project_id=project_id, - prisma_client=prisma_client, - ) - except Exception: # noqa: BLE001 # a project enqueue failure must not skip the sibling spend writes - verbose_proxy_logger.debug( - "_batch_database_updates: _update_project_db failed: %s", - traceback.format_exc(), - ) - - try: - await self._update_tag_db( - response_cost=response_cost, - request_tags=request_tags, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_tag_db failed: %s", - traceback.format_exc(), - ) - + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "_update_key_db", + self._update_key_db(response_cost=response_cost, hashed_token=hashed_token, prisma_client=prisma_client), + payload=payload_copy, + ) + await self._run_spend_update_step( + "_update_team_db", + self._update_team_db( + response_cost=response_cost, team_id=team_id, user_id=user_id, prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "_update_org_db", + self._update_org_db( + response_cost=response_cost, org_id=org_id, user_id=user_id, prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "_update_project_db", + self._update_project_db(response_cost=response_cost, project_id=project_id, prisma_client=prisma_client), + payload=payload_copy, + ) + await self._run_spend_update_step( + "_update_tag_db", + self._update_tag_db(response_cost=response_cost, request_tags=request_tags, prisma_client=prisma_client), + payload=payload_copy, + ) await self._update_model_access_group_db( response_cost=response_cost, request_model_access_groups=request_model_access_groups, @@ -1178,96 +1172,64 @@ class DBSpendUpdateWriter: prisma_client=prisma_client, router=get_llm_router(), ) - - _agent_id_for_spend: Final = payload_copy.get("agent_id") - try: - await self._update_agent_db( - response_cost=response_cost, - agent_id=_agent_id_for_spend, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_agent_db failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_user_transaction( - payload=payload_copy, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_user_transaction failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_end_user_transaction( - payload=payload_copy, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_end_user_transaction failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_agent_transaction( - payload=payload_copy, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_agent_transaction failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_team_transaction( - payload=payload_copy, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_team_transaction failed: %s", - traceback.format_exc(), - ) + await self._run_spend_update_step( + "_update_agent_db", + self._update_agent_db( + response_cost=response_cost, agent_id=payload_copy.get("agent_id"), prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_user_transaction", + self.add_spend_log_transaction_to_daily_user_transaction(payload=payload_copy, prisma_client=prisma_client), + payload=payload_copy, + ) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_end_user_transaction", + self.add_spend_log_transaction_to_daily_end_user_transaction( + payload=payload_copy, prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_agent_transaction", + self.add_spend_log_transaction_to_daily_agent_transaction( + payload=payload_copy, prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_team_transaction", + self.add_spend_log_transaction_to_daily_team_transaction(payload=payload_copy, prisma_client=prisma_client), + payload=payload_copy, + ) try: self._record_request_error(payload=payload_copy, prisma_client=prisma_client) - except Exception: # noqa: BLE001 # the rollup must never skip the sibling spend writes - verbose_proxy_logger.debug( - "_batch_database_updates: _record_request_error failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_org_transaction( - payload=payload_copy, - org_id=org_id, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_org_transaction failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_tag_transaction( - payload=payload_copy, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_tag_transaction failed: %s", - traceback.format_exc(), - ) - + except Exception as e: # noqa: BLE001 # the rollup must never skip the sibling spend writes + _report_spend_update_step_failure("_record_request_error", payload=payload_copy, exc=e) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_org_transaction", + self.add_spend_log_transaction_to_daily_org_transaction( + payload=payload_copy, org_id=org_id, prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_tag_transaction", + self.add_spend_log_transaction_to_daily_tag_transaction(payload=payload_copy, prisma_client=prisma_client), + payload=payload_copy, + ) await self._enqueue_model_usage_transaction(payload=payload_copy, prisma_client=prisma_client) + @staticmethod + async def _run_spend_update_step( + step_name: str, step: Coroutine[object, object, object], *, payload: SpendLogsPayload + ) -> None: + try: + await step + except Exception as e: # noqa: BLE001 # one helper failing must not skip the sibling spend writes + _report_spend_update_step_failure(step_name, payload=payload, exc=e) + async def _update_key_db( self, response_cost: float | None, @@ -2810,7 +2772,7 @@ class DBSpendUpdateWriter: date: Final = start_time.isoformat() if isinstance(start_time, datetime) else str(start_time or "") if not date: return - metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"]) + metadata: Final = _parse_spend_log_metadata(payload) internal_origin: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults request_error_accumulator.record( payload=payload, @@ -2861,7 +2823,7 @@ class DBSpendUpdateWriter: # spend, token and per-entity columns written here stay either way. request_status: Final = prisma_client.get_request_status(payload) verbose_proxy_logger.debug("Logged request status: %s", request_status) - _metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"]) + _metadata: Final = _parse_spend_log_metadata(payload) usage_obj: Final = _metadata.get("usage_object", {}) or {} if isinstance(payload["startTime"], datetime): start_time: Final = payload["startTime"].isoformat() diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index 7437042e4bd..9b2a96bbc58 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -36,6 +36,7 @@ from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( build_window_spend_transaction, ) from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL +from litellm.responses.streaming_iterator import drain_logging_tasks from litellm.types.caching import RedisPipelineLpopOperation, RedisPipelineRpushOperation from tests.unit.proxy.db.fake_prisma_engine import engine_call @@ -2158,46 +2159,122 @@ async def test_commit_spend_updates_to_db_increments_key_total_spend_alongside_s @pytest.mark.asyncio -async def test_update_database_creates_single_task(): +async def test_shutdown_drain_finishes_the_rollups_update_database_started(): """ - Test that update_database() fires exactly 1 asyncio.create_task() call - (the batched task) instead of the previous 11. + Regression for LIT-6027. The key, team and daily rollups ran in an unreferenced asyncio task, + so the shutdown sequence (flush, drain_logging_tasks, flush) never waited for them and spend + logs landed without matching daily rows. Once update_database returns, drain_logging_tasks + must leave every rollup for that request in its queue. """ - db_writer = DBSpendUpdateWriter() - - # Mock all helpers so nothing real runs + db_writer: Final = DBSpendUpdateWriter() db_writer._insert_spend_log_to_db = AsyncMock() - db_writer._batch_database_updates = AsyncMock() - + prisma: Final = MagicMock() + prisma.get_request_status = MagicMock(return_value="success") + fake_payload: Final = { + "request_id": "req-lit-6027", + "api_key": "hashed-key", + "user": "test-user", + "team_id": "test-team", + "agent_id": None, + "startTime": "2024-01-01T00:00:00", + "endTime": "2024-01-01T00:00:01", + "model": "gpt-4o-mini", + "custom_llm_provider": "openai", + "model_group": "gpt-4o-mini", + "prompt_tokens": 10, + "completion_tokens": 5, + "spend": 0.0, + "metadata": '{"usage_object": {}}', + } with ( patch("litellm.proxy.proxy_server.disable_spend_logs", False), - patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.prisma_client", prisma), patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"), - patch("litellm.proxy.db.db_spend_update_writer.asyncio.create_task") as mock_create_task, + patch("litellm.proxy.spend_tracking.spend_tracking_utils.get_logging_payload", return_value=fake_payload), ): await db_writer.update_database( token="test-token", user_id="test-user", - end_user_id="test-end-user", + end_user_id=None, + team_id="test-team", + org_id=None, + kwargs={"model": "gpt-4o-mini", "custom_llm_provider": "openai"}, + completion_response=MagicMock(), start_time=datetime.now(), end_time=datetime.now(), - team_id="test-team", - org_id="test-org", - completion_response=MagicMock(), response_cost=0.1, - kwargs={"model": "gpt-4", "custom_llm_provider": "openai"}, ) + await drain_logging_tasks() - # Exactly 1 create_task call (the batch), not 11 - assert mock_create_task.call_count == 1 + spend: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions() + assert spend["key_list_transactions"] == {"test-token": 0.1} + assert spend["team_list_transactions"] == {"test-team": 0.1} + daily_user: Final = ( + await db_writer.daily_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() + ) + daily_team: Final = ( + await db_writer.daily_team_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() + ) + assert daily_user is not None and daily_team is not None + (user_row,) = daily_user.values() + (team_row,) = daily_team.values() + assert (user_row["api_requests"], user_row["spend"]) == (1, 0.1) + assert (team_row["api_requests"], team_row["spend"]) == (1, 0.1) @pytest.mark.asyncio -async def test_batch_database_updates_isolation_on_failure(): +async def test_shutdown_drain_waits_for_an_in_flight_rollup_enqueue(): + """ + Regression for LIT-6027. A rollup enqueue that is still in flight when the shutdown drain + starts (here: blocked on a full spend queue) must be finished by the time drain_logging_tasks + returns, so the second shutdown flush commits it. + """ + enqueue_started: Final = asyncio.Event() + release_enqueue: Final = asyncio.Event() + + class _BlockedSpendQueue(SpendUpdateQueue): + async def add_update(self, update: SpendUpdateQueueItem): + enqueue_started.set() + await release_enqueue.wait() + await super().add_update(update) + + db_writer: Final = DBSpendUpdateWriter() + db_writer.spend_update_queue = _BlockedSpendQueue() + db_writer._insert_spend_log_to_db = AsyncMock() + with ( + patch("litellm.proxy.proxy_server.disable_spend_logs", True), + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), + patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"), + ): + await db_writer.update_database( + token="t1", + user_id=None, + end_user_id=None, + team_id="team-1", + org_id=None, + kwargs={"model": "gpt-4o-mini", "custom_llm_provider": "openai"}, + completion_response=MagicMock(), + start_time=datetime.now(), + end_time=datetime.now(), + response_cost=0.1, + ) + await enqueue_started.wait() + drain: Final = asyncio.create_task(drain_logging_tasks()) + release_enqueue.set() + await drain + spend: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions() + + assert spend["key_list_transactions"] == {"t1": 0.1} + assert spend["team_list_transactions"] == {"team-1": 0.1} + + +@pytest.mark.asyncio +async def test_batch_database_updates_isolation_on_failure(caplog: pytest.LogCaptureFixture): """ Test that if one helper inside _batch_database_updates raises, - all other helpers still execute. + all other helpers still execute and the failure is reported at ERROR with the request id. """ db_writer = DBSpendUpdateWriter() @@ -2217,6 +2294,7 @@ async def test_batch_database_updates_isolation_on_failure(): db_writer.add_spend_log_transaction_to_daily_org_transaction = AsyncMock() db_writer.add_spend_log_transaction_to_daily_tag_transaction = AsyncMock() + caplog.set_level(logging.ERROR, logger=verbose_proxy_logger.name) await db_writer._batch_database_updates( response_cost=0.1, user_id="u1", @@ -2226,7 +2304,7 @@ async def test_batch_database_updates_isolation_on_failure(): end_user_id="eu1", prisma_client=MagicMock(), litellm_proxy_budget_name="budget", - payload={"key": "value"}, + payload={"request_id": "req-iso", "key": "value"}, ) # _update_key_db raised, but all others should still have been called @@ -2242,6 +2320,48 @@ async def test_batch_database_updates_isolation_on_failure(): db_writer.add_spend_log_transaction_to_daily_team_transaction.assert_awaited_once() db_writer.add_spend_log_transaction_to_daily_org_transaction.assert_awaited_once() db_writer.add_spend_log_transaction_to_daily_tag_transaction.assert_awaited_once() + assert any( + "_update_key_db" in record.getMessage() and "req-iso" in record.getMessage() + for record in caplog.records + if record.levelno >= logging.ERROR + ) + + +@pytest.mark.asyncio +async def test_malformed_metadata_keeps_the_daily_rollup_row_and_is_reported(caplog: pytest.LogCaptureFixture): + """ + Regression for LIT-6027. A spend log whose metadata is not JSON used to vanish from every + daily rollup while the spend log itself persisted, with nothing above debug in the logs. + """ + writer: Final = DBSpendUpdateWriter() + prisma: Final = MagicMock() + prisma.get_request_status = MagicMock(return_value="success") + payload: Final = { + "request_id": "req-bad-metadata", + "user": "test-user", + "startTime": "2024-01-01T00:00:00", + "endTime": "2024-01-01T00:00:01", + "api_key": "test-key", + "model": "gpt-4o-mini", + "custom_llm_provider": "openai", + "model_group": "gpt-4o-mini", + "prompt_tokens": 100, + "completion_tokens": 50, + "spend": 0.05, + "metadata": "{not-json", + } + caplog.set_level(logging.ERROR, logger=verbose_proxy_logger.name) + + await writer.add_spend_log_transaction_to_daily_user_transaction( # pyright: ignore[reportUnknownMemberType] # prisma_client param is untyped on the writer + payload=payload, prisma_client=prisma + ) + + daily: Final = await writer.daily_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() + (row,) = daily.values() + assert (row["api_requests"], row["spend"], row["prompt_tokens"]) == (1, 0.05, 100) + assert any( + "req-bad-metadata" in record.getMessage() for record in caplog.records if record.levelno >= logging.ERROR + ) @pytest.mark.asyncio @@ -2775,12 +2895,12 @@ async def test_update_database_does_not_deepcopy_on_request_path(): """ Regression for LIT-4088: copy.deepcopy must not run while the caller awaits update_database(). The deepcopy used to isolate the daily-spend helpers is - relocated into the _batch_database_updates background task, and the spend-log + relocated into the _batch_database_updates logging task, and the spend-log insert receives the payload directly (all consumers are read-only). Asserts: - zero copy.deepcopy calls happen on the awaited request path - - the batch background task still hands the daily helpers an isolated copy + - the tracked rollup task still hands the daily helpers an isolated copy (mutating the original after the task ran does not bleed into it) - the spend-log insert receives the payload on the request path with the correct content @@ -2856,16 +2976,13 @@ async def test_update_database_does_not_deepcopy_on_request_path(): response_cost=0.1, ) - # Request path is clean: nothing was deepcopied while the caller awaited. assert len(deepcopy_calls) == 0 # The spend-log insert ran inline on the request path with the real payload. assert captured_spend_log["ref"] is fake_payload assert captured_spend_log["model_at_call"] == "gpt-4" assert fake_payload["spend"] == 0.1 - - # Now let the batch background task run; the deepcopy happens here. - await asyncio.sleep(0) + await drain_logging_tasks() assert len(deepcopy_calls) >= 1 assert len(captured_batch_payloads) == 1 From ecdc0f9ab9beacc13240078fa95c94a68c3248c4 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:23:27 -0700 Subject: [PATCH 002/155] fix(cost-map): declare Bedrock gpt-oss rows text-only so modality routing skips them (#48149) Co-authored-by: joshua Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 16 +++++++++++++ model_prices_and_context_window.json | 16 +++++++++++++ .../test_model_prices_and_context_window.py | 23 +++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3edb30b4218..14fe355a805 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45847,6 +45847,7 @@ "output_cost_per_token": 0.0 }, "openai.gpt-oss-120b-1:0": { + "supports_vision": false, "supported_endpoints": [ "/v1/chat/completions" ], @@ -45872,6 +45873,7 @@ "supports_tool_choice": true }, "openai.gpt-oss-20b-1:0": { + "supports_vision": false, "supported_endpoints": [ "/v1/chat/completions" ], @@ -49587,6 +49589,7 @@ "supports_vision": false }, "us-gov.openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -49606,6 +49609,7 @@ "supports_tool_choice": true }, "us-gov.openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -60156,6 +60160,7 @@ } }, "bedrock_mantle/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "input_cost_per_token_flex": 7.5e-08, @@ -60181,6 +60186,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 7e-08, "input_cost_per_token_batches": 3.5e-08, "input_cost_per_token_flex": 3.5e-08, @@ -60206,6 +60212,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-safeguard-120b": { + "supports_vision": false, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7e-08, "input_cost_per_token_flex": 7e-08, @@ -60229,6 +60236,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-safeguard-20b": { + "supports_vision": false, "input_cost_per_token": 7e-08, "input_cost_per_token_batches": 3e-08, "input_cost_per_token_flex": 3e-08, @@ -68080,6 +68088,7 @@ "supports_vision": false }, "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68099,6 +68108,7 @@ "supports_tool_choice": true }, "bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68377,6 +68387,7 @@ "supports_vision": false }, "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68396,6 +68407,7 @@ "supports_tool_choice": true }, "bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68784,6 +68796,7 @@ "supports_vision": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", @@ -68803,6 +68816,7 @@ "supports_tool_choice": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", @@ -68943,6 +68957,7 @@ "supports_vision": true }, "bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", @@ -68962,6 +68977,7 @@ "supports_tool_choice": true }, "bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3edb30b4218..14fe355a805 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45847,6 +45847,7 @@ "output_cost_per_token": 0.0 }, "openai.gpt-oss-120b-1:0": { + "supports_vision": false, "supported_endpoints": [ "/v1/chat/completions" ], @@ -45872,6 +45873,7 @@ "supports_tool_choice": true }, "openai.gpt-oss-20b-1:0": { + "supports_vision": false, "supported_endpoints": [ "/v1/chat/completions" ], @@ -49587,6 +49589,7 @@ "supports_vision": false }, "us-gov.openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -49606,6 +49609,7 @@ "supports_tool_choice": true }, "us-gov.openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -60156,6 +60160,7 @@ } }, "bedrock_mantle/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "input_cost_per_token_flex": 7.5e-08, @@ -60181,6 +60186,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 7e-08, "input_cost_per_token_batches": 3.5e-08, "input_cost_per_token_flex": 3.5e-08, @@ -60206,6 +60212,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-safeguard-120b": { + "supports_vision": false, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7e-08, "input_cost_per_token_flex": 7e-08, @@ -60229,6 +60236,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-safeguard-20b": { + "supports_vision": false, "input_cost_per_token": 7e-08, "input_cost_per_token_batches": 3e-08, "input_cost_per_token_flex": 3e-08, @@ -68080,6 +68088,7 @@ "supports_vision": false }, "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68099,6 +68108,7 @@ "supports_tool_choice": true }, "bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68377,6 +68387,7 @@ "supports_vision": false }, "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68396,6 +68407,7 @@ "supports_tool_choice": true }, "bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68784,6 +68796,7 @@ "supports_vision": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", @@ -68803,6 +68816,7 @@ "supports_tool_choice": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", @@ -68943,6 +68957,7 @@ "supports_vision": true }, "bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", @@ -68962,6 +68977,7 @@ "supports_tool_choice": true }, "bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 701b79d0ba9..9b9fec4c9a1 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -791,3 +791,26 @@ def test_volcengine_doubao_seed_2_1_pro_260915_row_matches_published_metadata(pa sibling = rows["volcengine/doubao-seed-2-1-pro-260628"] assert isinstance(sibling, dict) assert all(entry[key] == sibling[key] for key in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost")) + + +def _bedrock_gpt_oss_rows(prices: Mapping[str, JsonValue]) -> tuple[str, ...]: + return tuple( + key + for key, row in prices.items() + if "gpt-oss" in key and isinstance(row, dict) and str(row.get("litellm_provider", "")).startswith("bedrock") + ) + + +@pytest.mark.parametrize("path", [PRICES_PATH, BACKUP_PRICES_PATH], ids=["main", "backup"]) +def test_bedrock_gpt_oss_rows_declare_vision_support_like_their_safeguard_siblings(path: Path) -> None: + prices: Final = json.loads(path.read_text()) + rows: Final = _bedrock_gpt_oss_rows(prices) + assert rows, "no Bedrock gpt-oss rows found" + reference: Final = prices["openai.gpt-oss-safeguard-120b"]["supports_vision"] + undeclared: Final = tuple(key for key in rows if prices[key].get("supports_vision") is not reference) + assert undeclared == (), f"modality routing only skips rows that declare supports_vision: {undeclared}" + + +def test_bedrock_gpt_oss_vision_is_explicitly_disabled_for_the_router_gate() -> None: + assert litellm.utils.is_vision_explicitly_disabled("bedrock/openai.gpt-oss-120b-1:0") is True + assert litellm.utils.is_vision_explicitly_disabled("bedrock_mantle/openai.gpt-oss-20b") is True From f29e10c0ef641da97d9f1746ad9d5aa382ab9797 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:52:16 -0700 Subject: [PATCH 003/155] fix(router): keep image requests off deployments declared supports_vision false (#48152) * fix(router): keep image requests off deployments declared supports_vision false in pre-call checks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): type the vision pre-call helpers so the basedpyright gate holds Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): keep the pre-call index set typed without a mutable annotation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): cover the vision pre-call helpers directly for the router coverage check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): type the vision pre-call test helpers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): cover the no-params and unresolvable-provider vision branches Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: joshua Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 55 +++++++++++++- tests/unit/test_router/test_router.py | 102 ++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index da04e59f18a..705accbcf8d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -14186,6 +14186,43 @@ class Router: deployment, group, Router._params_neither_dropped_nor_allowed(call_settings, params) ) + @staticmethod + def _deployment_declares_no_vision(deployment: DeploymentTypedDict, group: str) -> bool: + plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts + model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict + declared: Final = model_info.get("supports_vision") + if declared is not None: + return declared is False + raw_params: Final = plain_deployment.get("litellm_params") or {} + params: Final = cast(Mapping[str, object], raw_params) # cast-ok: router deployments are plain dicts + if not params: + return False + base_model: Final = model_info.get("base_model") or params.get("base_model") + try: + model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group) + return litellm.utils.is_vision_explicitly_disabled( + base_model if isinstance(base_model, str) else model, custom_llm_provider + ) + except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request + verbose_router_logger.debug( + "litellm.router.py::_deployment_declares_no_vision: keeping deployment for model=%s. Got - %s", group, e + ) + return False + + @staticmethod + def _request_carries_image( + messages: list[dict[str, str]] | None, # mutable-ok: handed to resolve_structured_messages as-is + request_kwargs: Mapping[str, object], + ) -> bool: + from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content + from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages + + structured: Final = resolve_structured_messages( + messages=cast(list[dict[str, object]] | None, messages), # cast-ok: chat messages are plain dicts + request_kwargs=cast(dict[str, object], request_kwargs), # cast-ok: the resolver only reads the mapping + ) + return bool(structured) and request_contains_image_content(structured) + @staticmethod def _pre_call_check_requested_chat_params(request_kwargs: Mapping[str, object]) -> frozenset[str]: requested: Final = litellm.utils.get_non_default_params(passed_params=dict(request_kwargs)) @@ -14212,6 +14249,7 @@ class Router: - model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models - filter models above rpm limits - if region given, filter out models not in that region / unknown region + - request carries an image and the deployment declares supports_vision false, unless every candidate does - [TODO] function call and model doesn't support function calling """ @@ -14221,7 +14259,7 @@ class Router: # We only pop from the list, not modify deployment dicts - 100x+ faster on hot path (every request) _returned_deployments = list(healthy_deployments) - invalid_model_indices: Final = set() # Use set for O(1) membership checks + invalid_model_indices: Final = set[int]() # Use set for O(1) membership checks _context_window_error = False _potential_error_str = "" @@ -14371,9 +14409,20 @@ class Router: model=model, llm_provider="", ) - if len(excluded_indices) > 0: + remaining: Final = tuple( + (idx, deployment) for idx, deployment in param_candidates if idx not in excluded_indices + ) + text_only_indices: Final[frozenset[int]] = ( + frozenset(idx for idx, deployment in remaining if self._deployment_declares_no_vision(deployment, model)) + if remaining and self._request_carries_image(messages, request_settings) + else frozenset() + ) + dropped_indices: Final[frozenset[int]] = excluded_indices | ( + text_only_indices if len(text_only_indices) < len(remaining) else frozenset() + ) + if len(dropped_indices) > 0: # Single-pass filter using set for O(1) lookups (avoids O(n^2) from repeated pops) - _returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in excluded_indices] + _returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in dropped_indices] return _returned_deployments diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 9cc1bf6ce79..f06f9e08f7e 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -29871,3 +29871,105 @@ async def test_responses_fallback_wrapper_disconnect_billing_keeps_the_primary_s wrapped.adopt_fallback_headers(fallback) assert wrapped.partial_response_on_disconnect().response.usage.output_tokens == primary_tokens + fallback_tokens + + +_IMAGE_PART = {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}} +_IMAGE_CHAT = [{"role": "user", "content": [{"type": "text", "text": "What is in this image?"}, _IMAGE_PART]}] +_IMAGE_RESPONSES_INPUT = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "What is in this image?"}, + {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo="}, + ], + } +] +_VISION_LEG: Final[dict[str, object]] = {"litellm_params": {"model": "openai/gpt-5.6-terra"}, "model_info": {"id": "vision"}} +_DECLARED_TEXT_LEG: Final[dict[str, object]] = { + "litellm_params": {"model": "bedrock/openai.gpt-oss-120b-1:0"}, + "model_info": {"id": "declared-text", "supports_vision": False}, +} + + +def _pre_call_router(*deployments: dict[str, object], routing_strategy: str = "latency-based-routing") -> litellm.Router: + return litellm.Router( + model_list=[{"model_name": "mixed", **deployment} for deployment in deployments], + routing_strategy=routing_strategy, + enable_pre_call_checks=True, + ) + + +def _pre_call_ids(router: litellm.Router, deployments: tuple[dict[str, object], ...], **request: object) -> list[str]: + return [d["model_info"]["id"] for d in router._pre_call_checks(model="mixed", healthy_deployments=list(deployments), **request)] + + +def test_pre_call_checks_drop_declared_text_only_legs_for_image_requests(): + router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG) + assert _pre_call_ids(router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=_IMAGE_CHAT) == ["vision"] + + +def test_pre_call_checks_keep_declared_text_only_legs_for_text_requests(): + router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG) + assert _pre_call_ids(router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=[{"role": "user", "content": "Say hi"}]) == [ + "vision", + "declared-text", + ] + + +def test_pre_call_checks_drop_declared_text_only_legs_for_responses_input_image(): + router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG) + assert _pre_call_ids( + router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=None, request_kwargs={"input": _IMAGE_RESPONSES_INPUT} + ) == ["vision"] + + +def test_pre_call_checks_send_image_to_the_group_when_every_leg_is_declared_text_only(): + other_text_leg = {**_DECLARED_TEXT_LEG, "model_info": {"id": "declared-text-2", "supports_vision": False}} + router = _pre_call_router(_DECLARED_TEXT_LEG, other_text_leg) + assert _pre_call_ids(router, (_DECLARED_TEXT_LEG, other_text_leg), messages=_IMAGE_CHAT) == [ + "declared-text", + "declared-text-2", + ] + + +def test_pre_call_checks_keep_undeclared_custom_legs_for_image_requests(): + custom_leg = { + "litellm_params": {"model": "hosted_vllm/Qwen2.5-VL-7B-Instruct", "api_base": "http://localhost:8000"}, + "model_info": {"id": "custom"}, + } + router = _pre_call_router(_VISION_LEG, custom_leg) + assert _pre_call_ids(router, (_VISION_LEG, custom_leg), messages=_IMAGE_CHAT) == ["vision", "custom"] + + +def test_pre_call_checks_resolve_vision_through_the_cost_map_and_base_model(): + catalog_text_leg = {"litellm_params": {"model": "bedrock/deepseek.v3.2"}, "model_info": {"id": "catalog-text"}} + azure_text_leg = { + "litellm_params": {"model": "azure/my-audio-deployment", "api_base": "https://x.openai.azure.com"}, + "model_info": {"id": "azure-text", "base_model": "azure/gpt-audio"}, + } + router = _pre_call_router(_VISION_LEG, catalog_text_leg, azure_text_leg) + assert _pre_call_ids(router, (_VISION_LEG, catalog_text_leg, azure_text_leg), messages=_IMAGE_CHAT) == ["vision"] + + +@pytest.mark.parametrize( + ("deployment", "expected"), + [ + (_DECLARED_TEXT_LEG, True), + ({"litellm_params": {"model": "bedrock/openai.gpt-oss-120b-1:0"}, "model_info": {"supports_vision": True}}, False), + ({"litellm_params": {"model": "bedrock/deepseek.v3.2"}, "model_info": {}}, True), + ({"litellm_params": {"model": "azure/my-deploy"}, "model_info": {"base_model": "azure/gpt-audio"}}, True), + ({"litellm_params": {"model": "hosted_vllm/Qwen2.5-VL-7B-Instruct"}, "model_info": {}}, False), + ({"litellm_params": {"model": "no-such-model-anywhere"}, "model_info": {}}, False), + ({"litellm_params": {}, "model_info": {}}, False), + ], + ids=["override-false", "override-true", "cost-map-false", "base-model-false", "undeclared", "unresolvable", "no-params"], +) +def test_deployment_declares_no_vision_reads_the_override_then_the_cost_map(deployment: dict[str, object], expected: bool): + assert litellm.Router._deployment_declares_no_vision(deployment, "mixed") is expected + + +def test_request_carries_image_sees_chat_parts_and_responses_input_items(): + assert litellm.Router._request_carries_image(_IMAGE_CHAT, {}) is True + assert litellm.Router._request_carries_image(None, {"input": _IMAGE_RESPONSES_INPUT}) is True + assert litellm.Router._request_carries_image([{"role": "user", "content": "Say hi"}], {}) is False + assert litellm.Router._request_carries_image(None, {"input": "Say hi"}) is False From e04e89c7010809cc051b14f0e43d71dd018cdaee Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Fri, 23 Oct 2026 22:53:04 -0700 Subject: [PATCH 004/155] fix(responses): honor per-call custom pricing on the native Responses API path (#48119) * fix(responses): honor per-call custom pricing on the native Responses API path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): narrow the custom pricing regression test to the type gates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(main): expose register_custom_pricing_for_request for the responses path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): mark custom pricing test locals Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): price per-call custom rates request-locally instead of registering them globally Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): bind per-call rates on the request logging object instead of inferring them in Logging Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): keep Router deployments on the cost-map pricing path Router inherits cache rates onto the deployment's cost-map entry, so a routed Responses call binds no request-local price and keeps billing cached tokens at the inherited rate. Moves the deployment-id lookup to router_utils so the Responses and completion paths share it. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): gate request-local pricing on the typed deployment model_info Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 4 + litellm/litellm_core_utils/litellm_logging.py | 5 + litellm/responses/main.py | 57 ++++++- .../test_responses_api_request_body.py | 146 ++++++++++++++++-- 4 files changed, 199 insertions(+), 13 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 550bbfcde98..99944662bf8 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2264,6 +2264,8 @@ def response_cost_calculator( ### VERTEX LOCATION ### vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") region_name: str | None = None, + custom_cost_per_token: CostPerToken | None = None, + custom_cost_per_second: float | None = None, ) -> float: """ Returns @@ -2292,6 +2294,8 @@ def response_cost_calculator( optional_params=optional_params, custom_pricing=custom_pricing, base_model=base_model, + custom_cost_per_token=custom_cost_per_token, + custom_cost_per_second=custom_cost_per_second, prompt=prompt, standard_built_in_tools_params=standard_built_in_tools_params, litellm_model_name=litellm_model_name, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index d75d35e4531..41f13817d75 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -161,6 +161,7 @@ from litellm.types.utils import ( CachingDetails, CallTypes, CostBreakdown, + CostPerToken, CustomPricingLiteLLMParams, DynamicPromptManagementParamLiteral, EmbeddingResponse, @@ -621,6 +622,8 @@ class Logging(LiteLLMLoggingBaseClass): prometheusLogger, \ slack_app custom_pricing: bool = False + custom_cost_per_token: CostPerToken | None = None + custom_cost_per_second: float | None = None replaced_global_callback_keys: frozenset[str] = frozenset() stream_options = None litellm_request_debug: bool = False @@ -2096,6 +2099,8 @@ class Logging(LiteLLMLoggingBaseClass): "call_type": self.call_type, "optional_params": self.optional_params, "custom_pricing": custom_pricing, + "custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None, + "custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None, "prompt": prompt, "standard_built_in_tools_params": self.standard_built_in_tools_params, "router_model_id": router_model_id, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 0b526adf4fa..e7380c26529 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -6,10 +6,10 @@ from contextlib import contextmanager from dataclasses import dataclass from functools import partial from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast +from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast import httpx -from pydantic import BaseModel, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError from typing_extensions import ( TypeIs, # noqa: TID251 # narrows a client input item before its mark is read assert_never, @@ -87,7 +87,13 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency from litellm.types.responses.main import * from litellm.types.responses.streaming_websocket import ResponsesWebSocketRequestDefaults from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypes, all_litellm_params +from litellm.types.utils import ( + ADDRESSED_RESPONSE_ID_FIELD, + CallTypes, + CostPerToken, + CustomPricingLiteLLMParams, + all_litellm_params, +) from litellm.utils import ( ProviderConfigManager, client, @@ -546,6 +552,47 @@ async def aresponses_api_with_mcp( return response +def _per_call_custom_pricing(litellm_params: GenericLiteLLMParams) -> Mapping[str, float]: + return MappingProxyType( + litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True) + ) + + +class _DeploymentModelInfo(BaseModel): + model_config = ConfigDict(frozen=True) + id: object = None + + +_DEPLOYMENT_MODEL_INFO: Final[TypeAdapter[_DeploymentModelInfo | object]] = TypeAdapter( + Annotated[_DeploymentModelInfo | object, Field(union_mode="left_to_right")] +) + + +def _is_router_deployment(deployment_model_info: object) -> bool: + """A Router stamps the deployment's model_info (with its id) onto every call it routes.""" + parsed: Final = _DEPLOYMENT_MODEL_INFO.validate_python(deployment_model_info) + return isinstance(parsed, _DeploymentModelInfo) and parsed.id is not None + + +_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken) + + +def _per_call_cost_per_token(litellm_params: GenericLiteLLMParams) -> CostPerToken | None: + if litellm_params.input_cost_per_token is None or litellm_params.output_cost_per_token is None: + return None + return _COST_PER_TOKEN.validate_python( + litellm_params.model_dump( + include={ + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", + }, + exclude_none=True, + ) + ) + + def _bridges_to_chat_completions( responses_api_provider_config: BaseResponsesAPIConfig | None, use_chat_completions_api: bool ) -> bool: @@ -1673,6 +1720,7 @@ def responses( optional_params=dict(responses_api_request_params), litellm_params={ **responses_api_request_params, + **_per_call_custom_pricing(litellm_params), "aresponses": _is_async, "litellm_call_id": litellm_call_id, "model_info": kwargs.get("model_info"), @@ -1683,6 +1731,9 @@ def responses( custom_llm_provider=custom_llm_provider, ) + if not _is_router_deployment(deployment_model_info): + litellm_logging_obj.custom_cost_per_token = _per_call_cost_per_token(litellm_params) + litellm_logging_obj.custom_cost_per_second = litellm_params.cost_per_second # Decode any litellm-encoded encrypted-content item IDs back to their original IDs input = with_compaction_items_as_summaries( with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input)) diff --git a/tests/unit/responses/test_responses_api_request_body.py b/tests/unit/responses/test_responses_api_request_body.py index 36b5d8f135b..bf571c74e22 100644 --- a/tests/unit/responses/test_responses_api_request_body.py +++ b/tests/unit/responses/test_responses_api_request_body.py @@ -8,7 +8,8 @@ import asyncio import copy import json import logging -from collections.abc import Awaitable, Callable +import math +from collections.abc import Awaitable, Callable, Mapping from pathlib import Path from importlib import import_module from typing import Final @@ -17,10 +18,12 @@ from unittest.mock import AsyncMock, patch import httpx import pytest import respx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.types.llms.openai import ResponsesAPIResponse from tests.unit.llms.bedrock.slow_upstream import ( STREAM_TIMEOUT_SECONDS, UPSTREAM_FIRST_BYTE_SECONDS, @@ -333,7 +336,9 @@ async def test_aresponses_sends_the_declared_reasoning_effort_level_rounded_up( def _effort_levels_disabled_by(entry: dict) -> tuple[str, ...]: - return tuple(level for level in ("none", "minimal", "low") if entry.get(f"supports_{level}_reasoning_effort") is False) + return tuple( + level for level in ("none", "minimal", "low") if entry.get(f"supports_{level}_reasoning_effort") is False + ) def _gpt_5_row_disabling_a_level(provider: str) -> tuple[str, str]: @@ -623,7 +628,8 @@ async def test_aresponses_websocket_strips_responses_routing_prefix_from_openai_ from litellm.responses.main import _aresponses_websocket with patch.object( - import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket", + import_module("litellm.responses.main").base_llm_http_handler, + "async_responses_websocket", new_callable=AsyncMock, ) as mock_ws: await _aresponses_websocket( @@ -645,7 +651,8 @@ async def test_aresponses_websocket_keeps_routing_hints_out_of_the_relay_kwargs( from litellm.responses.main import _aresponses_websocket with patch.object( - import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket", + import_module("litellm.responses.main").base_llm_http_handler, + "async_responses_websocket", new_callable=AsyncMock, ) as mock_ws: await _aresponses_websocket( @@ -671,7 +678,9 @@ _ORIGINAL_WS_INPUT = [ @pytest.mark.asyncio @pytest.mark.parametrize("nested", [False, True]) -async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame(nested: bool): # test-quality-ok: the first frame handed to the relay is the only place the routed input is observable before the provider socket +async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame( + nested: bool, +): # test-quality-ok: the first frame handed to the relay is the only place the routed input is observable before the provider socket from unittest.mock import MagicMock from litellm.responses.main import _aresponses_websocket @@ -682,7 +691,8 @@ async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame ) with patch.object( - import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket", + import_module("litellm.responses.main").base_llm_http_handler, + "async_responses_websocket", new_callable=AsyncMock, ) as mock_ws: await _aresponses_websocket( @@ -711,7 +721,8 @@ async def test_aresponses_websocket_forwards_the_first_frame_verbatim_when_routi first_message = '{"type": "response.create", "model": "gpt-5.6", "input": [{"role": "user", "content": "hi"}]}' with patch.object( - import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket", + import_module("litellm.responses.main").base_llm_http_handler, + "async_responses_websocket", new_callable=AsyncMock, ) as mock_ws: await _aresponses_websocket( @@ -968,9 +979,7 @@ async def test_injection_points_reach_a_foundry_deployment_of_an_openai_model(mo Azure OpenAI one the Foundry deployment got Anthropic marks the Responses transform then stripped.""" monkeypatch.setenv("AZURE_AI_API_BASE", "https://other-deployment.openai.azure.com") injected_client = AsyncHTTPHandler() - mock_post = AsyncMock( - return_value=MockResponse(_minimal_responses_api_payload("resp_foundry", "gpt-6-astra"), 200) - ) + mock_post = AsyncMock(return_value=MockResponse(_minimal_responses_api_payload("resp_foundry", "gpt-6-astra"), 200)) injected_client.post = mock_post await litellm.aresponses( @@ -1234,6 +1243,77 @@ def test_faked_stream_accepts_a_text_timeout(monkeypatch: pytest.MonkeyPatch) -> ] +async def _invoke_responses(sync_mode: bool, model: str, pricing: Mapping[str, float]) -> ResponsesAPIResponse: + response: Final = ( + litellm.responses(model=model, input="hi", **pricing) + if sync_mode + else await litellm.aresponses(model=model, input="hi", **pricing) + ) + assert isinstance(response, ResponsesAPIResponse) + return response + + +def _response_cost(response: ResponsesAPIResponse) -> float: + hidden_params: Final[dict[str, float]] = getattr(response, "_hidden_params") + return hidden_params["response_cost"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", (True, False)) +@pytest.mark.parametrize("model", ("openai/gpt-5.4-nano", "openai/custom-priced-model-not-in-cost-map")) +async def test_responses_bills_per_call_custom_pricing_from_usage( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, sync_mode: bool, model: str +): + monkeypatch.setenv("OPENAI_API_KEY", "fake-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost") + monkeypatch.setattr(litellm, "model_cost", dict(model_cost)) + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=_minimal_responses_api_payload("resp_custom_price", model.split("/")[1])) + ) + pricing: Final = {"input_cost_per_token": 1e-5, "output_cost_per_token": 2e-5} + + response: Final = await _invoke_responses(sync_mode, model, pricing) + + assert response.usage is not None + expected: Final = ( + response.usage.input_tokens * pricing["input_cost_per_token"] + + response.usage.output_tokens * pricing["output_cost_per_token"] + ) + assert math.isclose(_response_cost(response), expected) + + +@pytest.mark.asyncio +async def test_concurrent_responses_calls_keep_their_own_per_call_prices( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +): + monkeypatch.setenv("OPENAI_API_KEY", "fake-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + model: Final = "openai/gpt-5.4-nano" + model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost") + cost_map_row_before: Final = copy.deepcopy(model_cost.get(model)) + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=_minimal_responses_api_payload("resp_concurrent", "gpt-5.4-nano")) + ) + rates: Final = ((1e-5, 2e-5), (3e-5, 4e-5)) + + responses: Final = await asyncio.gather( + *( + _invoke_responses(False, model, {"input_cost_per_token": in_rate, "output_cost_per_token": out_rate}) + for in_rate, out_rate in rates + ) + ) + + for response, (in_rate, out_rate) in zip(responses, rates, strict=True): + assert response.usage is not None + assert math.isclose( + _response_cost(response), response.usage.input_tokens * in_rate + response.usage.output_tokens * out_rate + ) + assert model_cost.get(model) == cost_map_row_before + + @pytest.mark.parametrize("asynchronous", [False, True]) @pytest.mark.parametrize("stream", [False, True]) @pytest.mark.parametrize( @@ -1339,3 +1419,49 @@ async def test_responses_opencode_session_uses_effective_destination( assert sent.headers.get("x-opencode-session") == (None if explicit_base else "conversation-123") assert sent.headers["authorization"] == "Bearer test-key" assert json.loads(sent.content) == {"model": "gpt-6-luna", "input": "hello"} + + +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) + + +@pytest.mark.asyncio +async def test_router_responses_deployment_pricing_keeps_inherited_cache_rates( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost") + monkeypatch.setattr(litellm, "model_cost", dict(model_cost)) + litellm.in_memory_llm_clients_cache.flush_cache() + rates: Final = {"input_cost_per_token": 1e-5, "output_cost_per_token": 2e-5} + router: Final = litellm.Router( + model_list=[ + { + "model_name": "nano", + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "fake-api-key", **rates}, + "model_info": {"id": "nano-deployment"}, + } + ] + ) + payload: Final[dict[str, object]] = _minimal_responses_api_payload("resp_router_cache", "gpt-5.4-nano") + usage: Final = _JSON_OBJECT.validate_python(payload["usage"]) + cached_tokens: Final = 8 + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response( + 200, + json={**payload, "usage": {**usage, "input_tokens_details": {"cached_tokens": cached_tokens}}}, + ) + ) + + response: Final = await router.aresponses(model="nano", input="hi") + + assert isinstance(response, ResponsesAPIResponse) + assert response.usage is not None + uncached_input: Final = response.usage.input_tokens - cached_tokens + full_price: Final = ( + response.usage.input_tokens * rates["input_cost_per_token"] + + response.usage.output_tokens * rates["output_cost_per_token"] + ) + floor: Final = ( + uncached_input * rates["input_cost_per_token"] + response.usage.output_tokens * rates["output_cost_per_token"] + ) + assert floor < _response_cost(response) < full_price From 455923014a6ef95ac5d98a3f8845969fd44d1450 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sat, 24 Oct 2026 09:07:02 -0700 Subject: [PATCH 005/155] feat(cost): bill openai 12-hour prompt cache writes at their own rate (#48088) * feat: add OpenAI 12-hour cache write pricing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix: omit empty cache-write response fields * style: format cache-write fields * fix(cost): bill openai 30m and 12h cache writes from the usage split, add gpt-5.6 rates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost): apply the 12h long-context write rate only above the token threshold Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(cost): key cache write rates by TTL instead of growing the rate tuple _get_token_base_cost returned a positional 6-tuple and every cache lifetime added a slot to it, to the off-peak helper and to every caller. It now returns a TokenBaseCosts dataclass whose cache write rates live in one mapping keyed by TTL. The TTLs come from the ephemeral_*_input_tokens fields on CacheCreationTokenDetails, so supporting a new lifetime only means adding that usage field and its rate key A lifetime with no rate of its own now always falls back to the cache creation rate after off-peak pricing is applied. Before this the non-tiered 12h rate and the tiered 1h rate fell back to the rate from before off-peak, so the same request billed differently depending on the pricing path * refactor(cost): hardcode the cache write TTLs as a Literal instead of parsing usage field names Parsing ephemeral_*_input_tokens names saved nothing, since a new TTL still needs a usage field and provider parsing code. The TTLs are now a CacheWriteTTL Literal with an explicit rate key per TTL * refactor(cost): resolve cache write lifetime rates to floats at lookup Decide the 1h and 12h cost map key in the threshold loop, then resolve each rate once after off-peak with the base write rate as the default, so no None flows through the cost code. Add an integration case for a 12h write on a row without a 12h rate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(cost): name every cache write TTL, marking 5m and 30m as base rate Co-Authored-By: Claude Opus 5.5 * refactor(cost): list every cache write TTL in one rate key table, None meaning the base write rate Co-Authored-By: Claude Opus 5.5 * refactor(cost): drop base rate entries from the cache write rate table and rely on fallback Co-Authored-By: Claude Opus 5.5 * refactor(cost): key cache write rates by plain string TTL Co-Authored-By: Claude Opus 5.5 * refactor(cost): fill missing cache write TTL rates with the base write rate after off-peak Co-Authored-By: Claude Opus 5.5 * refactor(cost): default missing cache write TTL rates to the base write rate inline Co-Authored-By: Claude Opus 5.5 * refactor(cost): rename cache write rate key table to make clear it only covers long TTLs Co-Authored-By: Claude Opus 5.5 * fix(cost): list every cache write TTL and bill writes of an unknown TTL at the base write rate Co-Authored-By: Claude Opus 5.5 * refactor(cost): resolve tiered cache write TTL rates after off-peak instead of passing None Co-Authored-By: Claude Opus 5.5 * refactor(cost): carry cache write TTL rates on TokenRates and drop TokenBaseCosts Co-Authored-By: Claude Opus 5.5 * refactor(cost): define TokenRates before its first use to drop the string annotation Co-Authored-By: Claude Opus 5.5 * fix(cost): build the TokenRates TTL default with default_factory and type tiers as Mapping MappingProxyType({}) as a dataclass default is rejected on Python 3.11, where mappingproxy is unhashable, so import litellm failed there. The tier type moves from dict[str, object] to Mapping[str, object] (LIT001), with tier_rate accepting a Mapping to match. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost): type select_tier_for_input tiers as Mapping so tier_rate callers stay typed The tier returned as a bare dict made every tier_rate call site a partially unknown argument once tier_rate took Mapping[str, object]. The range bounds are now validated with a TypeAdapter instead of indexing an unknown list. --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- .../llm_cost_calc/tiered_pricing.py | 42 ++- .../litellm_core_utils/llm_cost_calc/utils.py | 278 ++++++++++-------- litellm/llms/dashscope/cost_calculator.py | 3 +- ...odel_prices_and_context_window_backup.json | 48 +++ litellm/proxy/_lazy_openapi_snapshot.json | 40 +++ .../transformation.py | 26 +- litellm/responses/utils.py | 22 +- litellm/types/llms/openai.py | 17 +- litellm/types/utils.py | 14 + litellm/utils.py | 18 ++ model_prices_and_context_window.json | 48 +++ .../cost_calculation/cases/openai.py | 102 +++++++ .../integration/cost_calculation/cost_map.py | 2 + .../cost_calculation/cost_tracking_case.py | 6 + .../llm_cost_calc/test_llm_cost_calc_utils.py | 269 ++++++++++++----- .../test_litellm_completion_responses.py | 4 +- tests/unit/responses/test_responses_utils.py | 41 ++- tests/unit/test_utils.py | 12 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 52 +++- 19 files changed, 811 insertions(+), 233 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tiered_pricing.py b/litellm/litellm_core_utils/llm_cost_calc/tiered_pricing.py index 9bcc2b1743c..3040e36ec5f 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tiered_pricing.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tiered_pricing.py @@ -5,30 +5,42 @@ Shared by provider cost calculators (e.g. Dashscope) and the proxy budget reservation logic so neither has to depend on the other. """ -from typing import Final +from collections.abc import Mapping, Sequence +from typing import Final, SupportsFloat + +from pydantic import TypeAdapter, ValidationError + +_TIER_RANGE: Final = TypeAdapter(tuple[float, float]) -def _coerce_cost_per_token(value: float | str | None) -> float: +def _coerce_cost_per_token(value: object) -> float: """ Coerce a per-token cost into a float. Model cost values loaded from YAML config may arrive as strings (e.g. scientific notation like "4e-07"), which would break arithmetic. """ - if value is None: - return 0.0 if isinstance(value, str): try: return float(value) except ValueError: return 0.0 - return float(value) + if isinstance(value, SupportsFloat): + return float(value) + return 0.0 + + +def _tier_range(tier: Mapping[str, object]) -> tuple[float, float] | None: + try: + return _TIER_RANGE.validate_python(tier.get("range")) + except ValidationError: + return None def select_tier_for_input( - tiered_pricing: list[dict], + tiered_pricing: Sequence[Mapping[str, object]], input_tokens: int, -) -> dict | None: +) -> Mapping[str, object] | None: """ Select the pricing tier for a request based on its total input token count. @@ -44,19 +56,21 @@ def select_tier_for_input( if not tiered_pricing or input_tokens <= 0: return None - sorted_tiers: Final = sorted(tiered_pricing, key=lambda t: t.get("range", [0, 0])[0]) - valid_tiers: Final = [tier for tier in sorted_tiers if len(tier.get("range", [])) == 2] - if not valid_tiers: + ranged_tiers: Final = sorted( + ((bounds, tier) for tier in tiered_pricing if (bounds := _tier_range(tier)) is not None), + key=lambda ranged: ranged[0][0], + ) + if not ranged_tiers: return None - matching: Final = [tier for tier in valid_tiers if tier["range"][0] < input_tokens <= tier["range"][1]] + matching: Final = tuple(tier for (start, end), tier in ranged_tiers if start < input_tokens <= end) if matching: return matching[0] - return valid_tiers[-1] + return ranged_tiers[-1][1] def tier_rate( - tier: dict, + tier: Mapping[str, object], cost_key: str, fallback_cost_key: str | None = None, ) -> float: @@ -68,4 +82,6 @@ def tier_rate( primary: Final = tier.get(cost_key) if primary is not None: return _coerce_cost_per_token(primary) + if fallback_cost_key is None: + return 0.0 return _coerce_cost_per_token(tier.get(fallback_cost_key, 0)) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index a874459798d..5d4b1a82759 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -3,7 +3,7 @@ import re from collections.abc import Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field, replace from datetime import datetime, timezone, tzinfo from types import MappingProxyType from typing import Final, Literal, TypedDict, cast @@ -52,6 +52,22 @@ _VALID_DATA_RESIDENCIES: Final = frozenset(r.value for r in DataResidency) _DEPLOYMENT_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields) +_BASE_CACHE_WRITE_RATE_KEY: Final = "cache_creation_input_token_cost" + +_CACHE_WRITE_RATE_KEYS: Final[Mapping[str, str]] = MappingProxyType( + { + "5m": _BASE_CACHE_WRITE_RATE_KEY, + "30m": _BASE_CACHE_WRITE_RATE_KEY, + "1hr": "cache_creation_input_token_cost_above_1hr", + "12hr": "cache_creation_input_token_cost_above_12hr", + } +) + +_TTL_PRICED_CACHE_WRITE_RATE_KEYS: Final[Mapping[str, str]] = MappingProxyType( + {ttl: key for ttl, key in _CACHE_WRITE_RATE_KEYS.items() if key != _BASE_CACHE_WRITE_RATE_KEY} +) + + _IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = ( "input_cost_per_token", "output_cost_per_token", @@ -356,7 +372,21 @@ def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provide ) -def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict[str, object] | None: +@dataclass(frozen=True, slots=True) +class TokenRates: + input_rate: float + output_rate: float + cache_read_rate: float + cache_creation_rate: float + reasoning_rate: float | None + cache_creation_rates_by_ttl: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({})) + + @property + def billed_reasoning_rate(self) -> float: + return self.output_rate if self.reasoning_rate is None else self.reasoning_rate + + +def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> Mapping[str, object] | None: tiered_pricing: Final = model_info.get("tiered_pricing") if not isinstance(tiered_pricing, list) or not tiered_pricing: return None @@ -376,7 +406,7 @@ def _get_tiered_reasoning_rate(model_info: ModelInfo, usage: Usage) -> float | N return tier_rate(tier, "output_cost_per_reasoning_token", "output_cost_per_token") -def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[float, float, float, float, float] | None: +def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[TokenRates, Mapping[str, object]] | None: """ Resolve the base rates from a model's ``tiered_pricing`` table, if it has one. @@ -391,22 +421,20 @@ def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[float, if tier is None: return None - cache_creation_cost: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token") completion_cost: Final = ( tier_rate(tier, "output_cost_per_token") if "output_cost_per_token" in tier else get_cost_per_unit(model_info, "output_cost_per_token") or 0.0 ) - return ( - tier_rate(tier, "input_cost_per_token"), - completion_cost, - cache_creation_cost, - tier_rate(tier, "cache_creation_input_token_cost_above_1hr") - if "cache_creation_input_token_cost_above_1hr" in tier - and tier["cache_creation_input_token_cost_above_1hr"] is not None - else cache_creation_cost, - tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"), + cache_creation_rate: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token") + rates: Final = TokenRates( + input_rate=tier_rate(tier, "input_cost_per_token"), + output_rate=completion_cost, + cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"), + cache_creation_rate=cache_creation_rate, + reasoning_rate=None, ) + return rates, tier def _is_within_off_peak_window(off_peak_hours_utc: str | Sequence[str], current_time: datetime | None = None) -> bool: @@ -532,19 +560,6 @@ def _is_off_peak(off_peak: Mapping[str, object], current_time: datetime | None = return False -@dataclass(frozen=True, slots=True) -class TokenRates: - input_rate: float - output_rate: float - cache_read_rate: float - cache_creation_rate: float - reasoning_rate: float | None - - @property - def billed_reasoning_rate(self) -> float: - return self.output_rate if self.reasoning_rate is None else self.reasoning_rate - - def resolve_explicit_cache_read_cost_rate( model_info: ModelInfo, usage: Usage, @@ -649,7 +664,8 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None, if off_peak is None: return rates off_peak_reasoning_rate: Final = _parse_off_peak_rate(off_peak.get("output_cost_per_reasoning_token")) - return TokenRates( + return replace( + rates, input_rate=_off_peak_rate(off_peak, "input_cost_per_token", rates.input_rate), output_rate=_off_peak_rate(off_peak, "output_cost_per_token", rates.output_rate), cache_read_rate=_off_peak_rate(off_peak, "cache_read_input_token_cost", rates.cache_read_rate), @@ -661,32 +677,28 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None, def _apply_off_peak_to_base_costs( model_info: ModelInfo, current_time: datetime | None, - base_costs: tuple[float, float, float, float | None, float], -) -> tuple[float, float, float, float, float]: - """Apply off-peak rates to an already-resolved set of base costs, whichever pricing path - produced them. off_peak_pricing has no field for the one-hour cache-creation rate, so a - present one passes through untouched and an absent one resolves to the applied - cache-creation rate. Reasoning is left to _resolve_billed_reasoning_rate. + base_rates: TokenRates, + tier: Mapping[str, object], +) -> TokenRates: + """Apply off-peak rates to the base costs resolved from a pricing tier. off_peak_pricing has no + field for the long-TTL cache-creation rates, so a tier that declares one keeps it, and a tier + that does not bills that TTL at the off-peak-adjusted write rate. Reasoning is left to + _resolve_billed_reasoning_rate. """ - prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs - rates: Final = apply_off_peak_pricing( - model_info, - current_time, - TokenRates( - input_rate=prompt, - output_rate=completion, - cache_read_rate=cache_read, - cache_creation_rate=cache_creation, - reasoning_rate=None, + rates: Final = apply_off_peak_pricing(model_info, current_time, base_rates) + return replace( + rates, + cache_creation_rates_by_ttl=MappingProxyType( + { + ttl: rates.cache_creation_rate if tier.get(key) is None else tier_rate(tier, key) + for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items() + } ), ) - return ( - rates.input_rate, - rates.output_rate, - rates.cache_creation_rate, - rates.cache_creation_rate if cache_creation_above_1hr is None else cache_creation_above_1hr, - rates.cache_read_rate, - ) + + +def _first_present_key(model_info: ModelInfo, preferred_key: str, fallback_key: str) -> str: + return preferred_key if model_info.get(preferred_key) is not None else fallback_key def _get_token_base_cost( @@ -696,7 +708,7 @@ def _get_token_base_cost( current_time: datetime | None = None, *, threshold_is_inclusive: bool = False, -) -> tuple[float, float, float, float, float]: +) -> TokenRates: """ Return prompt cost, completion cost, and cache costs for a given model and usage. @@ -709,15 +721,12 @@ def _get_token_base_cost( An absent cache-creation or cache-read rate always resolves to the resolved input rate, the way the tiered table and custom deployment pricing already do, since a provider that publishes no cache price bills cached tokens as ordinary input. An - absent 1h write rate resolves to the cache-creation rate, off-peak included. An + absent long-TTL write rate resolves to the cache-creation rate, off-peak included. An explicit 0.0 stays a real price for all of them. - - Returns: - Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost) """ tiered_base_costs: Final = _get_tiered_base_costs(model_info=model_info, usage=usage) if tiered_base_costs is not None: - return _apply_off_peak_to_base_costs(model_info, current_time, tiered_base_costs) + return _apply_off_peak_to_base_costs(model_info, current_time, *tiered_base_costs) # Get service tier aware cost keys input_cost_key: Final = get_service_tier_cost_key("input_cost_per_token", service_tier) @@ -734,13 +743,13 @@ def _get_token_base_cost( # For image generation models that don't have output_cost_per_token, # use output_cost_per_image_token as the base cost (all output tokens are image tokens) - if completion_base_cost == 0.0 or completion_base_cost is None: + if completion_base_cost == 0.0: output_image_cost: Final = get_cost_per_unit(model_info, "output_cost_per_image_token", None) if output_image_cost is not None: completion_base_cost = output_image_cost cache_creation_cost = get_cost_per_unit(model_info, cache_creation_cost_key, default_value=None) - cache_creation_cost_above_1hr = get_cost_per_unit( - model_info, "cache_creation_input_token_cost_above_1hr", default_value=None + cache_creation_keys_by_ttl: Mapping[str, str] = MappingProxyType( + {ttl: get_service_tier_cost_key(key, service_tier) for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()} ) cache_read_cost = get_cost_per_unit(model_info, cache_read_cost_key, default_value=None) @@ -811,14 +820,6 @@ def _get_token_base_cost( if service_tier else f"cache_creation_input_token_cost_above_{threshold_str}_tokens" ) - cache_creation_1hr_tiered_key = ( - get_service_tier_cost_key( - f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens" - ) cache_read_tiered_key = ( get_service_tier_cost_key( f"cache_read_input_token_cost_above_{threshold_str}_tokens", @@ -830,10 +831,19 @@ def _get_token_base_cost( cache_creation_cost = get_cost_per_unit(model_info, cache_creation_tiered_key, cache_creation_cost) - cache_creation_cost_above_1hr = get_cost_per_unit( - model_info, cache_creation_1hr_tiered_key, cache_creation_cost_above_1hr + cache_creation_keys_by_ttl = MappingProxyType( + { + ttl: _first_present_key( + model_info, + get_service_tier_cost_key( + f"{_TTL_PRICED_CACHE_WRITE_RATE_KEYS[ttl]}_above_{threshold_str}_tokens", + service_tier, + ), + key, + ) + for ttl, key in cache_creation_keys_by_ttl.items() + } ) - cache_read_cost = get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost) except (IndexError, ValueError): continue @@ -849,16 +859,26 @@ def _get_token_base_cost( resolved_cache_creation_cost: Final = ( input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost ) - - return _apply_off_peak_to_base_costs( + base_rates: Final = apply_off_peak_pricing( model_info, current_time, - ( - prompt_base_cost, - completion_base_cost, - resolved_cache_creation_cost, - cache_creation_cost_above_1hr, - resolved_cache_read_cost, + TokenRates( + input_rate=prompt_base_cost, + output_rate=completion_base_cost, + cache_read_rate=resolved_cache_read_cost, + cache_creation_rate=resolved_cache_creation_cost, + reasoning_rate=None, + ), + ) + return replace( + base_rates, + cache_creation_rates_by_ttl=MappingProxyType( + { + ttl: base_rates.cache_creation_rate + if (rate := get_cost_per_unit(model_info, key, default_value=None)) is None + else rate + for ttl, key in cache_creation_keys_by_ttl.items() + } ), ) @@ -975,25 +995,28 @@ def resolve_image_model_info(model: str, custom_llm_provider: str, model_info: M def calculate_cache_writing_cost( cache_creation_tokens: int, cache_creation_token_details: CacheCreationTokenDetails | None, - cache_creation_cost_above_1hr: float, cache_creation_cost: float, + cache_creation_cost_by_ttl: Mapping[str, float], ) -> float: """ - Adjust cost of cache creation tokens based on the cache creation token details. + Bill cache writes per lifetime: Anthropic reports 5m and 1h counts, OpenAI reports 30m and 12h counts. + Writes no lifetime count covers, such as a lifetime a provider adds later, bill at the base write rate. """ - total_cost: float = 0.0 - if cache_creation_token_details is not None: - # get the number of 5m and 1h cache creation tokens - cache_creation_tokens_5m: Final = cache_creation_token_details.ephemeral_5m_input_tokens - cache_creation_tokens_1h: Final = cache_creation_token_details.ephemeral_1h_input_tokens - # add the number of 5m and 1h cache creation tokens to the cache creation tokens - total_cost += cache_creation_tokens_5m * cache_creation_cost if cache_creation_tokens_5m is not None else 0.0 - total_cost += ( - cache_creation_tokens_1h * cache_creation_cost_above_1hr if cache_creation_tokens_1h is not None else 0.0 - ) - else: - total_cost += cache_creation_tokens * cache_creation_cost - return total_cost + if cache_creation_token_details is None: + return cache_creation_tokens * cache_creation_cost + tokens_by_ttl: Final[Mapping[str, int | None]] = MappingProxyType( + { + "5m": cache_creation_token_details.ephemeral_5m_input_tokens, + "30m": cache_creation_token_details.ephemeral_30m_input_tokens, + "1hr": cache_creation_token_details.ephemeral_1h_input_tokens, + "12hr": cache_creation_token_details.ephemeral_12h_input_tokens, + } + ) + unattributed_tokens: Final = max(cache_creation_tokens - sum(t or 0 for t in tokens_by_ttl.values()), 0) + return unattributed_tokens * cache_creation_cost + sum( + (tokens or 0) * cache_creation_cost_by_ttl.get(ttl, cache_creation_cost) + for ttl, tokens in tokens_by_ttl.items() + ) class PromptTokensDetailsResult(TypedDict): @@ -1178,7 +1201,7 @@ def _calculate_input_cost( prompt_base_cost: float, cache_read_cost: float, cache_creation_cost: float, - cache_creation_cost_above_1hr: float, + cache_creation_cost_by_ttl: Mapping[str, float], service_tier: str | None = None, ) -> float: """ @@ -1248,8 +1271,8 @@ def _calculate_input_cost( prompt_cost += calculate_cache_writing_cost( cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"], cache_creation_token_details=prompt_tokens_details["cache_creation_token_details"], - cache_creation_cost_above_1hr=cache_creation_cost_above_1hr, cache_creation_cost=cache_creation_cost, + cache_creation_cost_by_ttl=cache_creation_cost_by_ttl, ) ### CHARACTER COST @@ -1510,34 +1533,31 @@ def generic_cost_per_token( ) billing_time: Final = current_time if current_time is not None else current_billing_time() - ( - prompt_base_cost, - completion_base_cost, - cache_creation_cost, - cache_creation_cost_above_1hr, - cache_read_cost, - ) = _get_token_base_cost( + base_costs: Final = _get_token_base_cost( model_info=resolved_model_info, usage=usage, service_tier=service_tier, current_time=billing_time, threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), ) + completion_base_cost: Final = base_costs.output_rate resolved_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate( model_info=resolved_model_info, usage=usage, current_time=billing_time, service_tier=service_tier, ) - effective_cache_read_cost: Final = cache_read_cost if resolved_cache_read_cost is None else resolved_cache_read_cost + effective_cache_read_cost: Final = ( + base_costs.cache_read_rate if resolved_cache_read_cost is None else resolved_cache_read_cost + ) prompt_cost = _calculate_input_cost( prompt_tokens_details=prompt_tokens_details, model_info=resolved_model_info, - prompt_base_cost=prompt_base_cost, + prompt_base_cost=base_costs.input_rate, cache_read_cost=effective_cache_read_cost, - cache_creation_cost=cache_creation_cost, - cache_creation_cost_above_1hr=cache_creation_cost_above_1hr, + cache_creation_cost=base_costs.cache_creation_rate, + cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl, service_tier=service_tier, ) @@ -1640,7 +1660,7 @@ class BilledTokenRates: cache_read_input_token_cost: float cache_read_input_audio_token_cost: float cache_creation_input_token_cost: float - cache_creation_input_token_cost_above_1hr: float + cache_creation_input_token_cost_by_ttl: Mapping[str, float] output_cost_per_reasoning_token: float def scaled(self, multiplier: float) -> "BilledTokenRates": @@ -1652,7 +1672,9 @@ class BilledTokenRates: cache_read_input_token_cost=self.cache_read_input_token_cost * multiplier, cache_read_input_audio_token_cost=self.cache_read_input_audio_token_cost * multiplier, cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier, - cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier, + cache_creation_input_token_cost_by_ttl=MappingProxyType( + {ttl: rate * multiplier for ttl, rate in self.cache_creation_input_token_cost_by_ttl.items()} + ), output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier, ) @@ -1702,7 +1724,7 @@ def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRat cache_read_input_token_cost=cache_read_rate, cache_read_input_audio_token_cost=cache_read_rate, cache_creation_input_token_cost=cache_creation_rate, - cache_creation_input_token_cost_above_1hr=cache_creation_rate, + cache_creation_input_token_cost_by_ttl=MappingProxyType({}), output_cost_per_reasoning_token=output_rate, ) @@ -1717,13 +1739,7 @@ def _cost_map_billed_rates( current_time: datetime | None, ) -> BilledTokenRates: billing_time: Final = current_time if current_time is not None else current_billing_time() - ( - prompt_base_cost, - completion_base_cost, - cache_creation_cost_rate, - cache_creation_cost_above_1hr_rate, - cache_read_cost_rate, - ) = _get_token_base_cost( + base_costs: Final = _get_token_base_cost( model_info=model_info, usage=usage, service_tier=service_tier, @@ -1737,14 +1753,14 @@ def _cost_map_billed_rates( service_tier=service_tier, ) effective_cache_read_cost_rate: Final = ( - cache_read_cost_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate + base_costs.cache_read_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate ) reasoning_rate: Final = _resolve_billed_reasoning_rate( model_info=model_info, usage=usage, service_tier=service_tier, - completion_base_cost=completion_base_cost, + completion_base_cost=base_costs.output_rate, current_time=billing_time, ) audio_cache_read_rate: Final = get_cost_per_unit( @@ -1759,14 +1775,14 @@ def _cost_map_billed_rates( * get_provider_specific_speed_multiplier(model_info=model_info, usage=usage) ) return BilledTokenRates( - input_cost_per_token=prompt_base_cost, - output_cost_per_token=completion_base_cost, + input_cost_per_token=base_costs.input_rate, + output_cost_per_token=base_costs.output_rate, cache_read_input_token_cost=effective_cache_read_cost_rate, cache_read_input_audio_token_cost=( audio_cache_read_rate if audio_cache_read_rate is not None else effective_cache_read_cost_rate ), - cache_creation_input_token_cost=cache_creation_cost_rate, - cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate, + cache_creation_input_token_cost=base_costs.cache_creation_rate, + cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl, output_cost_per_reasoning_token=reasoning_rate, ).scaled(multiplier) @@ -1854,8 +1870,8 @@ def get_token_type_cost_breakdown( else calculate_cache_writing_cost( cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_token_details, - cache_creation_cost_above_1hr=rates.cache_creation_input_token_cost_above_1hr, cache_creation_cost=rates.cache_creation_input_token_cost, + cache_creation_cost_by_ttl=rates.cache_creation_input_token_cost_by_ttl, ) ) return TokenTypeCostBreakdown( @@ -1886,7 +1902,7 @@ def calculate_prompt_caching_savings( biller saw them rather than at the later spend write. """ model_info_with_cache_read_default: Final = apply_provider_cache_read_default(model_info, custom_llm_provider) - prompt_base_cost, _, cache_creation_cost, cache_creation_cost_above_1hr, cache_read_cost = _get_token_base_cost( + base_costs: Final = _get_token_base_cost( model_info=model_info_with_cache_read_default, usage=usage, service_tier=service_tier, @@ -1899,9 +1915,11 @@ def calculate_prompt_caching_savings( current_time=billed_at, service_tier=service_tier, ) - effective_cache_read_cost: Final = cache_read_cost if provider_cache_read_cost is None else provider_cache_read_cost - write_rate: Final = cache_creation_cost or prompt_base_cost - write_rate_1h: Final = cache_creation_cost_above_1hr or write_rate + prompt_base_cost: Final = base_costs.input_rate + effective_cache_read_cost: Final = ( + base_costs.cache_read_rate if provider_cache_read_cost is None else provider_cache_read_cost + ) + write_rate: Final = base_costs.cache_creation_rate or prompt_base_cost prompt_tokens_details: Final = parse_prompt_tokens_details(usage) cache_read_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"], 0) cache_creation_tokens: Final = max(prompt_tokens_details["cache_creation_tokens"], 0) @@ -1909,7 +1927,9 @@ def calculate_prompt_caching_savings( cache_creation_details: Final = ( CacheCreationTokenDetails( ephemeral_5m_input_tokens=max(details.ephemeral_5m_input_tokens or 0, 0), + ephemeral_30m_input_tokens=max(details.ephemeral_30m_input_tokens or 0, 0), ephemeral_1h_input_tokens=max(details.ephemeral_1h_input_tokens or 0, 0), + ephemeral_12h_input_tokens=max(details.ephemeral_12h_input_tokens or 0, 0), ) if details is not None else None @@ -1918,8 +1938,10 @@ def calculate_prompt_caching_savings( write_premium: Final = calculate_cache_writing_cost( cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_details, - cache_creation_cost_above_1hr=write_rate_1h - prompt_base_cost, cache_creation_cost=write_rate - prompt_base_cost, + cache_creation_cost_by_ttl=MappingProxyType( + {ttl: rate - prompt_base_cost for ttl, rate in base_costs.cache_creation_rates_by_ttl.items()} + ), ) uplift: Final = get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift( model_info, vertex_location diff --git a/litellm/llms/dashscope/cost_calculator.py b/litellm/llms/dashscope/cost_calculator.py index 98504f54533..3f388637a83 100644 --- a/litellm/llms/dashscope/cost_calculator.py +++ b/litellm/llms/dashscope/cost_calculator.py @@ -7,6 +7,7 @@ cached, cache-creation, output, reasoning) is billed at that one tier's rate. See https://help.aliyun.com/zh/model-studio/billing-for-model-studio """ +from collections.abc import Mapping from dataclasses import dataclass, replace from datetime import datetime from typing import Final @@ -72,7 +73,7 @@ def _flat_rates(model_info: ModelInfo) -> TokenRates: ) -def _tier_rates(model_info: ModelInfo, tier: dict) -> TokenRates: +def _tier_rates(model_info: ModelInfo, tier: Mapping[str, object]) -> TokenRates: # A tier that declares output rates keeps the request on them, all-or-nothing. A tier table # spelling out only input rates would serve every completion for free, so there the model's # own output rates stand in diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 14fe355a805..8c0c6dffdc7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -36795,6 +36795,12 @@ }, "gpt-6-astra": { "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_12hr": 2.5e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 0.0001, + "cache_creation_input_token_cost_above_12hr_flex": 1.25e-05, + "cache_creation_input_token_cost_above_12hr_priority": 5e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05, @@ -36880,6 +36886,12 @@ }, "gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_12hr": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, @@ -36957,6 +36969,12 @@ }, "gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_12hr": 2.5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-06, + "cache_creation_input_token_cost_above_12hr_flex": 1.25e-07, + "cache_creation_input_token_cost_above_12hr_priority": 5e-07, "cache_creation_input_token_cost_decisions": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07, @@ -37043,6 +37061,12 @@ }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_12hr": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05, + "cache_creation_input_token_cost_above_12hr_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 2e-05, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, @@ -37117,6 +37141,12 @@ }, "gpt-5.6-sol": { "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_12hr": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05, + "cache_creation_input_token_cost_above_12hr_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 2e-05, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, @@ -37193,6 +37223,12 @@ }, "gpt-5.6-terra": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_12hr": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, @@ -37268,6 +37304,12 @@ }, "gpt-5.6-luna": { "cache_creation_input_token_cost": 2.5e-07, + "cache_creation_input_token_cost_above_12hr": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-06, + "cache_creation_input_token_cost_above_12hr_flex": 2.5e-07, + "cache_creation_input_token_cost_above_12hr_priority": 1e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, @@ -74933,6 +74975,12 @@ }, "gpt-6.1-sol": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_12hr": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 0dd978ee855..4e183bf9d5c 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -22858,6 +22858,25 @@ "title": "Body_image_edit_api_openai_deployments__model__images_edits_post", "type": "object" }, + "CacheWriteTokensDetails": { + "additionalProperties": true, + "properties": { + "ttl_12h_tokens": { + "title": "Ttl 12H Tokens", + "type": "integer" + }, + "ttl_30m_tokens": { + "title": "Ttl 30M Tokens", + "type": "integer" + } + }, + "required": [ + "ttl_30m_tokens", + "ttl_12h_tokens" + ], + "title": "CacheWriteTokensDetails", + "type": "object" + }, "CachedTokensDetails": { "properties": { "audio_tokens": { @@ -24352,6 +24371,27 @@ ], "title": "Audio Tokens" }, + "cache_write_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Cache Write Tokens" + }, + "cache_write_tokens_details": { + "anyOf": [ + { + "$ref": "#/components/schemas/CacheWriteTokensDetails" + }, + { + "type": "null" + } + ] + }, "cached_tokens": { "default": 0, "title": "Cached Tokens", diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 02d9566d396..a9c7818b577 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -3663,18 +3663,20 @@ class LiteLLMCompletionResponsesConfig: if count is not None } ) - response_usage.input_tokens_details = InputTokensDetails( - cached_tokens=prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0, - text_tokens=prompt_details.text_tokens, - audio_tokens=prompt_details.audio_tokens, - image_tokens=prompt_details.image_tokens, - video_tokens=prompt_details.video_tokens, - cached_tokens_details=( - cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None - ), - **cache_write_extra, - **cache_type_extra, - **grounding_request_counts, + response_usage.input_tokens_details = InputTokensDetails.model_validate( + { + "cached_tokens": prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0, + "text_tokens": prompt_details.text_tokens, + "audio_tokens": prompt_details.audio_tokens, + "image_tokens": prompt_details.image_tokens, + "video_tokens": prompt_details.video_tokens, + "cached_tokens_details": ( + cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None + ), + **cache_write_extra, + **cache_type_extra, + **grounding_request_counts, + } ) # Translate completion_tokens_details to output_tokens_details diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 503d85aaed9..f256480d952 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -13,6 +13,7 @@ from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel from litellm.types.llms.openai import ( AllMessageValues, + InputTokensDetails, OutputTokensDetails, ResponseAPIUsage, ResponseInputParam, @@ -23,6 +24,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.responses.main import DecodedResponseId from litellm.types.utils import ( + CacheCreationTokenDetails, CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, SpecialEnums, @@ -41,6 +43,17 @@ def _output_token_detail(details: object, field: str) -> int | None: return value if isinstance(value, int) else None +def _cache_write_lifetimes(cache_write_tokens_details: object) -> CacheCreationTokenDetails | None: + """OpenAI Responses splits `cache_write_tokens` into `ttl_30m_tokens` and `ttl_12h_tokens`.""" + ttl_30m_tokens: Final = _output_token_detail(cache_write_tokens_details, "ttl_30m_tokens") + ttl_12h_tokens: Final = _output_token_detail(cache_write_tokens_details, "ttl_12h_tokens") + if ttl_30m_tokens is None and ttl_12h_tokens is None: + return None + return CacheCreationTokenDetails( + ephemeral_30m_input_tokens=ttl_30m_tokens, ephemeral_12h_input_tokens=ttl_12h_tokens + ) + + def _is_object_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a list is a Sequence of anything return isinstance(value, list) @@ -1247,7 +1260,11 @@ class ResponseAPILoggingUtils: prompt_tokens_details: PromptTokensDetailsWrapper | None = None if response_api_usage.input_tokens_details: if isinstance(response_api_usage.input_tokens_details, dict): - prompt_tokens_details = PromptTokensDetailsWrapper(**response_api_usage.input_tokens_details) + input_details: Final = InputTokensDetails.model_validate(response_api_usage.input_tokens_details) + prompt_tokens_details = PromptTokensDetailsWrapper( + **input_details.model_dump(exclude={"cache_write_tokens_details"}), + cache_creation_token_details=_cache_write_lifetimes(input_details.cache_write_tokens_details), + ) else: prompt_tokens_details = PromptTokensDetailsWrapper( cached_tokens=getattr(response_api_usage.input_tokens_details, "cached_tokens", None), @@ -1259,6 +1276,9 @@ class ResponseAPILoggingUtils: ), video_tokens=getattr(response_api_usage.input_tokens_details, "video_tokens", None), cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None), + cache_creation_token_details=_cache_write_lifetimes( + getattr(response_api_usage.input_tokens_details, "cache_write_tokens_details", None) + ), cache_type=getattr(response_api_usage.input_tokens_details, "cache_type", None), web_search_requests=getattr(response_api_usage.input_tokens_details, "web_search_requests", None), google_maps_grounding_requests=getattr( diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 62dc152ca84..9a4ecf37127 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -683,12 +683,12 @@ class ChatCompletionCachedContent(TypedDict): class PromptCacheBreakpoint(TypedDict): mode: ReadOnly[Literal["explicit"]] - ttl: NotRequired[ReadOnly[Literal["30m"]]] + ttl: NotRequired[ReadOnly[Literal["30m", "12h"]]] class PromptCacheOptions(TypedDict, total=False): mode: ReadOnly[Literal["implicit", "explicit"]] - ttl: ReadOnly[Literal["30m"]] + ttl: ReadOnly[Literal["30m", "12h"]] class ChatCompletionThinkingBlock(TypedDict, total=False): @@ -1382,10 +1382,23 @@ class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject): model_config = {"extra": "allow"} +class CacheWriteTokensDetails(BaseLiteLLMOpenAIResponseObject): + ttl_30m_tokens: int + ttl_12h_tokens: int + + model_config = ConfigDict(extra="allow", frozen=True) + + +def _is_none(value: object) -> bool: + return value is None + + class InputTokensDetails(BaseLiteLLMOpenAIResponseObject): audio_tokens: int | None = None cached_tokens: int = 0 cached_tokens_details: CachedTokensDetails | None = None + cache_write_tokens: int | None = Field(default=None, exclude_if=_is_none) + cache_write_tokens_details: CacheWriteTokensDetails | None = Field(default=None, exclude_if=_is_none) image_tokens: int | None = None text_tokens: int | None = None video_tokens: int | None = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5939a350970..ed66cbd8cf0 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -322,6 +322,12 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_creation_input_token_cost_above_272k_tokens_flex: float | None cache_creation_input_token_cost_above_272k_tokens_ultrafast: ReadOnly[float | None] cache_creation_input_token_cost_above_1hr: float | None + cache_creation_input_token_cost_above_12hr: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_flex: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_priority: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_above_272k_tokens: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: ReadOnly[float | None] cache_creation_input_token_cost_flex: float | None # OpenAI flex service tier pricing cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing @@ -1804,7 +1810,9 @@ class CompletionTokensDetailsWrapper(CompletionTokensDetails): # wrapper for ol class CacheCreationTokenDetails(LiteLLMBaseModel): ephemeral_5m_input_tokens: int | None = None + ephemeral_30m_input_tokens: int | None = None ephemeral_1h_input_tokens: int | None = None + ephemeral_12h_input_tokens: int | None = None class PromptTokensDetailsWrapper( @@ -3954,6 +3962,12 @@ class MirroredPricingParams(LiteLLMBaseModel): explicit_cache_read_input_token_cost: float | None = None cache_creation_input_token_cost: float | None = None cache_creation_input_token_cost_above_1hr: float | None = None + cache_creation_input_token_cost_above_12hr: float | None = None + cache_creation_input_token_cost_above_12hr_flex: float | None = None + cache_creation_input_token_cost_above_12hr_priority: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: float | None = None tiered_pricing: list[dict[str, Any]] | None = None diff --git a/litellm/utils.py b/litellm/utils.py index 809a94cd93e..58eaa43e687 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6817,6 +6817,24 @@ def get_model_info_helper( cache_creation_input_token_cost_above_1hr=_model_info.get( "cache_creation_input_token_cost_above_1hr", None ), + cache_creation_input_token_cost_above_12hr=_model_info.get( + "cache_creation_input_token_cost_above_12hr", None + ), + cache_creation_input_token_cost_above_12hr_flex=_model_info.get( + "cache_creation_input_token_cost_above_12hr_flex", None + ), + cache_creation_input_token_cost_above_12hr_priority=_model_info.get( + "cache_creation_input_token_cost_above_12hr_priority", None + ), + cache_creation_input_token_cost_above_12hr_above_272k_tokens=_model_info.get( + "cache_creation_input_token_cost_above_12hr_above_272k_tokens", None + ), + cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex=_model_info.get( + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex", None + ), + cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority=_model_info.get( + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority", None + ), off_peak_pricing=_model_info.get("off_peak_pricing", None), input_cost_per_character=_model_info.get("input_cost_per_character", None), input_cost_per_token_above_128k_tokens=_model_info.get("input_cost_per_token_above_128k_tokens", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 14fe355a805..8c0c6dffdc7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -36795,6 +36795,12 @@ }, "gpt-6-astra": { "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_12hr": 2.5e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 0.0001, + "cache_creation_input_token_cost_above_12hr_flex": 1.25e-05, + "cache_creation_input_token_cost_above_12hr_priority": 5e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05, @@ -36880,6 +36886,12 @@ }, "gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_12hr": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, @@ -36957,6 +36969,12 @@ }, "gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_12hr": 2.5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-06, + "cache_creation_input_token_cost_above_12hr_flex": 1.25e-07, + "cache_creation_input_token_cost_above_12hr_priority": 5e-07, "cache_creation_input_token_cost_decisions": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07, @@ -37043,6 +37061,12 @@ }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_12hr": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05, + "cache_creation_input_token_cost_above_12hr_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 2e-05, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, @@ -37117,6 +37141,12 @@ }, "gpt-5.6-sol": { "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_12hr": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05, + "cache_creation_input_token_cost_above_12hr_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 2e-05, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, @@ -37193,6 +37223,12 @@ }, "gpt-5.6-terra": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_12hr": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, @@ -37268,6 +37304,12 @@ }, "gpt-5.6-luna": { "cache_creation_input_token_cost": 2.5e-07, + "cache_creation_input_token_cost_above_12hr": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-06, + "cache_creation_input_token_cost_above_12hr_flex": 2.5e-07, + "cache_creation_input_token_cost_above_12hr_priority": 1e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, @@ -74933,6 +74975,12 @@ }, "gpt-6.1-sol": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_12hr": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index acba66223e5..53fd5c1cec2 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -6985,6 +6985,106 @@ GPT_5_6_RESPONSES_CACHE_READ: Final = CostTrackingTestCase( ), ) +GPT_5_6_RESPONSES_CACHE_WRITE_12H: Final = CostTrackingTestCase( + name="gpt-5.6-responses_cache_write_12h", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gpt-5.6", + endpoint="/v1/responses", + request={ + "model": "$MODEL", + "input": "summarize this text", + "prompt_cache_options": {"mode": "implicit", "ttl": "12h"}, + }, + response=JsonResponse( + content_type="application/json", + body={ + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "scripted response", "annotations": []}], + } + ], + "usage": { + "input_tokens": 12928, + "output_tokens": 380, + "total_tokens": 13308, + "input_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 5000, + "cache_write_tokens_details": {"ttl_30m_tokens": 3000, "ttl_12h_tokens": 2000}, + }, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + ), + expected=ExactExpected( + spend=0.0344865, + input_cost=0.0291865, + output_cost=0.00532, + prompt_tokens=12928, + completion_tokens=380, + cache_creation_cost=0.0153125, + ), +) + +GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE: Final = CostTrackingTestCase( + name="gpt-6-luna-responses_cache_write_12h_no_12h_rate", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gpt-6-luna", + endpoint="/v1/responses", + request={ + "model": "$MODEL", + "input": "summarize this text", + "prompt_cache_options": {"mode": "implicit", "ttl": "12h"}, + }, + response=JsonResponse( + content_type="application/json", + body={ + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-6-luna", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "scripted response", "annotations": []}], + } + ], + "usage": { + "input_tokens": 12928, + "output_tokens": 380, + "total_tokens": 13308, + "input_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 5000, + "cache_write_tokens_details": {"ttl_30m_tokens": 3000, "ttl_12h_tokens": 2000}, + }, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + ), + expected=ExactExpected( + spend=0.0016078, + input_cost=0.0014178, + output_cost=0.00019, + prompt_tokens=12928, + completion_tokens=380, + cache_creation_cost=0.000625, + ), +) + GPT_5_6_RESPONSES_REASONING: Final = CostTrackingTestCase( name="gpt-5.6-responses_reasoning", covers="quota_management.spend_tracking.cost_matrix.logs_cost", @@ -11913,6 +12013,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( OMNI_MODERATIONS_NEXT_SINGLE, OMNI_MODERATIONS_NEXT_LIST, GPT_5_6_RESPONSES_CACHE_READ, + GPT_5_6_RESPONSES_CACHE_WRITE_12H, + GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE, GPT_5_6_RESPONSES_REASONING, GPT_5_6_RESPONSES_STREAM, GPT_5_6_RESPONSES_STREAM_CACHE_READ, diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index 012d847e676..cd979528fe6 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -174,6 +174,8 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { }, }, "gpt-5.6": { + "cache_creation_input_token_cost": 2.1875e-06, + "cache_creation_input_token_cost_above_12hr": 4.375e-06, "cache_read_input_token_cost": 1.75e-07, "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 1.75e-06, diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index e98f6a44222..e978a9a316a 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -85,6 +85,12 @@ class CostMapEntry(BaseModel): cache_read_input_token_cost: float | None = None cache_creation_input_token_cost: float | None = None cache_creation_input_token_cost_above_1hr: float | None = None + cache_creation_input_token_cost_above_12hr: float | None = None + cache_creation_input_token_cost_above_12hr_flex: float | None = None + cache_creation_input_token_cost_above_12hr_priority: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: float | None = None cache_creation_input_token_cost_above_1hr_above_200k_tokens: float | None = None cache_read_input_token_cost_above_200k_tokens: float | None = None cache_creation_input_token_cost_above_200k_tokens: float | None = None diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index c99147d2e63..1729db94ed8 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -65,8 +65,8 @@ def test_missing_cache_read_rate_resolves_to_input_rate(prompt_tokens, read_rate prompt_cost, _ = generic_cost_per_token( "policy-fixture", usage, "openai", service_tier=service_tier, model_info=info ) - assert billed[4] == pytest.approx(read_rate if read_rate is not None else billed[0]) - assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed[0] + 100 * billed[4]) + assert billed.cache_read_rate == pytest.approx(read_rate if read_rate is not None else billed.input_rate) + assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed.input_rate + 100 * billed.cache_read_rate) def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_read_rate() -> None: @@ -248,7 +248,7 @@ def test_missing_cache_read_uses_off_peak_input_rate(): } when = datetime(2026, 9, 7, 12, tzinfo=timezone.utc) billed = _get_token_base_cost(info, Usage(prompt_tokens=100), current_time=when) - assert billed[0] == billed[4] == 5e-6 + assert billed.input_rate == billed.cache_read_rate == 5e-6 def test_reasoning_tokens_no_price_set(_local_model_cost_map): @@ -324,8 +324,6 @@ def test_reasoning_tokens_gemini(_local_model_cost_map): ) - - def test_image_tokens_with_custom_pricing(): """Test that image_tokens in completion are properly costed with output_cost_per_image_token.""" from unittest.mock import patch @@ -578,7 +576,7 @@ def test_get_token_base_cost_picks_highest_crossed_tier(): } usage = Usage(prompt_tokens=150_000, completion_tokens=10, total_tokens=150_010) - prompt_base_cost = _get_token_base_cost(model_info, usage)[0] + prompt_base_cost = _get_token_base_cost(model_info, usage).input_rate assert prompt_base_cost == 9e-6 @@ -779,12 +777,12 @@ def test_get_token_base_cost_weekend_only_off_peak_rate(): saturday_peak_hours = _get_token_base_cost( model_info, usage, current_time=datetime(2026, 8, 29, 2, 0, tzinfo=timezone.utc) ) - assert saturday_peak_hours[:2] == (5e-7, 1e-6) + assert (saturday_peak_hours.input_rate, saturday_peak_hours.output_rate) == (5e-7, 1e-6) monday_same_hours = _get_token_base_cost( model_info, usage, current_time=datetime(2026, 8, 24, 2, 0, tzinfo=timezone.utc) ) - assert monday_same_hours[:2] == (1e-6, 2e-6) + assert (monday_same_hours.input_rate, monday_same_hours.output_rate) == (1e-6, 2e-6) def test_get_token_base_cost_applies_off_peak_pricing(): @@ -810,14 +808,14 @@ def test_get_token_base_cost_applies_off_peak_pricing(): usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) off_peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert off_peak[0] == 5e-7 - assert off_peak[1] == 1e-6 - assert off_peak[4] == 5e-8 + assert off_peak.input_rate == 5e-7 + assert off_peak.output_rate == 1e-6 + assert off_peak.cache_read_rate == 5e-8 peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert peak[0] == 1e-6 - assert peak[1] == 2e-6 - assert peak[4] == 1e-7 + assert peak.input_rate == 1e-6 + assert peak.output_rate == 2e-6 + assert peak.cache_read_rate == 1e-7 def test_get_token_base_cost_non_mapping_off_peak_block_bills_standard_rates(): @@ -843,8 +841,8 @@ def test_get_token_base_cost_non_mapping_off_peak_block_bills_standard_rates(): }, ) result = _get_token_base_cost(model_info, usage, current_time=when) - assert result[0] == 1e-6 - assert result[1] == 2e-6 + assert result.input_rate == 1e-6 + assert result.output_rate == 2e-6 def test_get_token_base_cost_off_peak_falls_back_to_standard_when_unset(): @@ -864,8 +862,8 @@ def test_get_token_base_cost_off_peak_falls_back_to_standard_when_unset(): usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) result = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert result[0] == 5e-7 - assert result[1] == 2e-6 + assert result.input_rate == 5e-7 + assert result.output_rate == 2e-6 def test_get_token_base_cost_off_peak_wins_over_threshold(): @@ -891,12 +889,12 @@ def test_get_token_base_cost_off_peak_wins_over_threshold(): usage = Usage(prompt_tokens=250000, completion_tokens=250000, total_tokens=500000) off_peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert off_peak[0] == 5e-7 - assert off_peak[1] == 1e-6 + assert off_peak.input_rate == 5e-7 + assert off_peak.output_rate == 1e-6 peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert peak[0] == 3e-6 - assert peak[1] == 4e-6 + assert peak.input_rate == 3e-6 + assert peak.output_rate == 4e-6 def test_get_model_info_propagates_off_peak_fields(): @@ -948,10 +946,10 @@ def test_get_token_base_cost_off_peak_wins_over_tiered_pricing(): usage = Usage(prompt_tokens=1_000, completion_tokens=100, total_tokens=1_100) inside = _get_token_base_cost(info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert inside[:2] == (5e-7, 1e-6) + assert (inside.input_rate, inside.output_rate) == (5e-7, 1e-6) outside = _get_token_base_cost(info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert outside[:2] == (3e-6, 6e-6) + assert (outside.input_rate, outside.output_rate) == (3e-6, 6e-6) def _register_off_peak_reasoning_model( @@ -1153,17 +1151,17 @@ def test_get_token_base_cost_off_peak_cache_creation_rate(): inside_window = datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc) inside = _get_token_base_cost(model_info, usage, current_time=inside_window) - assert inside[2] == 5e-7 - assert inside[3] == 2e-6 + assert inside.cache_creation_rate == 5e-7 + assert inside.cache_creation_rates_by_ttl["1hr"] == 2e-6 outside = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert outside[2] == 1.25e-6 + assert outside.cache_creation_rate == 1.25e-6 without_key = cast( ModelInfo, {**model_info, "off_peak_pricing": {"hours_utc": "16:30-00:30", "input_cost_per_token": 5e-7}}, ) - assert _get_token_base_cost(without_key, usage, current_time=inside_window)[2] == 1.25e-6 + assert _get_token_base_cost(without_key, usage, current_time=inside_window).cache_creation_rate == 1.25e-6 def test_get_token_type_cost_breakdown_reflects_off_peak_reasoning_and_cache_creation_rates(): @@ -1583,19 +1581,14 @@ def test_generic_cost_per_token_tier_1hr_cache_rate_preserves_zero_and_fallback( ), ) costs: Final = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openrouter") - breakdown: Final = get_token_type_cost_breakdown( - model=model, usage=usage, custom_llm_provider="openrouter" - ) + breakdown: Final = get_token_type_cost_breakdown(model=model, usage=usage, custom_llm_provider="openrouter") ordinary_rate: Final = cache_rates.get("cache_creation_input_token_cost") - expected_write_cost: Final = ( - (20 - one_hour_tokens) * (ordinary_rate if ordinary_rate is not None else 1.0) - + one_hour_tokens * expected_write_rate - ) + expected_write_cost: Final = (20 - one_hour_tokens) * ( + ordinary_rate if ordinary_rate is not None else 1.0 + ) + one_hour_tokens * expected_write_rate assert costs == pytest.approx((40 * 1.0 + 40 * 0.1 + expected_write_cost, 10 * 2.0)) assert breakdown.cache_creation_cost == pytest.approx(expected_write_cost) - assert breakdown.rates is not None - assert breakdown.rates.cache_creation_input_token_cost_above_1hr == expected_write_rate finally: litellm.model_cost.pop(model, None) @@ -2020,8 +2013,8 @@ def test_calculate_cache_writing_cost(): result = calculate_cache_writing_cost( cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_token_details, - cache_creation_cost_above_1hr=cache_creation_cost_above_1hr, cache_creation_cost=cache_creation_cost, + cache_creation_cost_by_ttl={"1hr": cache_creation_cost_above_1hr, "12hr": cache_creation_cost_above_1hr}, ) # Expected calculation: @@ -2041,8 +2034,8 @@ def test_calculate_cache_writing_cost(): result_no_details = calculate_cache_writing_cost( cache_creation_tokens=cache_creation_tokens_no_details, cache_creation_token_details=cache_creation_token_details_none, - cache_creation_cost_above_1hr=cache_creation_cost_above_1hr, cache_creation_cost=cache_creation_cost_fallback, + cache_creation_cost_by_ttl={"1hr": cache_creation_cost_above_1hr, "12hr": cache_creation_cost_above_1hr}, ) # Expected calculation when no details: 1000 * 5e-06 = 0.005 @@ -2059,15 +2052,13 @@ def test_calculate_cache_writing_cost(): result_partial = calculate_cache_writing_cost( cache_creation_tokens=500, cache_creation_token_details=cache_creation_token_details_partial, - cache_creation_cost_above_1hr=6e-06, cache_creation_cost=3e-06, + cache_creation_cost_by_ttl={"1hr": 6e-06, "12hr": 6e-06}, ) - # Expected calculation: 0 (for None 5m tokens) + (100 * 6e-06) = 0.0006 - expected_cost_partial = (0.0) + (100 * 6e-06) + expected_cost_partial = (400 * 3e-06) + (100 * 6e-06) assert round(result_partial, 6) == round(expected_cost_partial, 6) - assert round(result_partial, 6) == 0.0006 # Test case 4: Zero costs result_zero = calculate_cache_writing_cost( @@ -2075,13 +2066,119 @@ def test_calculate_cache_writing_cost(): cache_creation_token_details=CacheCreationTokenDetails( ephemeral_5m_input_tokens=50, ephemeral_1h_input_tokens=950 ), - cache_creation_cost_above_1hr=0.0, cache_creation_cost=0.0, + cache_creation_cost_by_ttl={"1hr": 0.0, "12hr": 0.0}, ) assert result_zero == 0.0 +@pytest.mark.parametrize("field", CacheCreationTokenDetails.model_fields) +def test_calculate_cache_writing_cost_bills_every_reported_lifetime(field: str) -> None: + cost = calculate_cache_writing_cost( + cache_creation_tokens=10, + cache_creation_token_details=CacheCreationTokenDetails(**{field: 10}), + cache_creation_cost=2.0, + cache_creation_cost_by_ttl={"1hr": 2.0, "12hr": 2.0}, + ) + assert cost == 20.0 + + +def test_calculate_cache_writing_cost_bills_writes_of_an_unknown_lifetime_at_the_base_rate() -> None: + cost = calculate_cache_writing_cost( + cache_creation_tokens=10, + cache_creation_token_details=CacheCreationTokenDetails( + ephemeral_30m_input_tokens=2, ephemeral_12h_input_tokens=3 + ), + cache_creation_cost=1.0, + cache_creation_cost_by_ttl={"1hr": 4.0, "12hr": 4.0}, + ) + assert cost == 2 * 1.0 + 3 * 4.0 + 5 * 1.0 + + +@pytest.mark.parametrize( + ("prompt_tokens", "service_tier", "write_rate", "twelve_hour_rate"), + [ + (10_000, "default", 2e-6, 4e-6), + (10_000, "flex", 1e-6, 2e-6), + (10_000, "priority", 3e-6, 6e-6), + (300_000, "default", 4e-6, 8e-6), + (300_000, "flex", 2e-6, 4e-6), + (300_000, "priority", 6e-6, 12e-6), + ], +) +def test_generic_cost_per_token_prices_12h_cache_writes_at_prompt_threshold( + prompt_tokens: int, service_tier: str, write_rate: float, twelve_hour_rate: float +) -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_creation_input_token_cost": 2e-6, + "cache_creation_input_token_cost_flex": 1e-6, + "cache_creation_input_token_cost_priority": 3e-6, + "cache_creation_input_token_cost_above_272k_tokens": 4e-6, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2e-6, + "cache_creation_input_token_cost_above_272k_tokens_priority": 6e-6, + "cache_creation_input_token_cost_above_12hr": 4e-6, + "cache_creation_input_token_cost_above_12hr_flex": 2e-6, + "cache_creation_input_token_cost_above_12hr_priority": 6e-6, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 8e-6, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 4e-6, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 12e-6, + } + usage: Final = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=0, + total_tokens=prompt_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper( + cache_creation_tokens=5_000, + cache_creation_token_details=CacheCreationTokenDetails( + ephemeral_30m_input_tokens=3_000, ephemeral_12h_input_tokens=2_000 + ), + ), + ) + + result: Final = generic_cost_per_token( + model="test-12h-cache-write", + usage=usage, + custom_llm_provider="openai", + service_tier=None if service_tier == "default" else service_tier, + model_info=model_info, + ) + + assert result[0] == pytest.approx( + (usage.prompt_tokens - usage.prompt_tokens_details.cache_creation_tokens) * model_info["input_cost_per_token"] + + 3_000 * write_rate + + 2_000 * twelve_hour_rate + ) + + +def test_generic_cost_per_token_falls_back_to_default_for_missing_12h_rate() -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_creation_input_token_cost": 3e-6, + } + usage: Final = Usage( + prompt_tokens=100, + completion_tokens=0, + total_tokens=100, + prompt_tokens_details=PromptTokensDetailsWrapper( + cache_creation_tokens=100, + cache_creation_token_details=CacheCreationTokenDetails(ephemeral_12h_input_tokens=100), + ), + ) + + result: Final = generic_cost_per_token( + model="test-12h-cache-write-fallback", + usage=usage, + custom_llm_provider="openai", + model_info=model_info, + ) + + assert result[0] == pytest.approx(100 * model_info["cache_creation_input_token_cost"]) + + def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): """ Regression test: when cache_creation_tokens is 0 but cache_creation_token_details @@ -2119,7 +2216,7 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): prompt_base_cost=0.0, cache_read_cost=0.0, cache_creation_cost=cache_creation_cost, - cache_creation_cost_above_1hr=cache_creation_cost_above_1hr, + cache_creation_cost_by_ttl={"1hr": cache_creation_cost_above_1hr, "12hr": cache_creation_cost_above_1hr}, ) # Expected: (100 * 3.75e-06) + (200 * 6e-06) = 0.000375 + 0.0012 = 0.001575 @@ -2330,10 +2427,6 @@ def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(_local_mo assert round(cost, 10) == round(expected_cost, 10) - - - - def test_query_count_is_free_without_a_per_query_price(_local_model_cost_map): usage = Usage( prompt_tokens=0, @@ -2512,8 +2605,6 @@ def test_vertex_global_or_absent_location_no_uplift(vertex_location, _local_mode assert base == located - - def test_vertex_uplift_invalid_multiplier_defaults_to_one(): """A malformed multiplier in the cost map degrades to base pricing, never raises.""" from litellm.litellm_core_utils.llm_cost_calc.utils import ( @@ -2569,10 +2660,10 @@ def test_flat_input_price_keeps_other_token_thresholds(field: str) -> None: model_info=info, usage=Usage(prompt_tokens=300_001, completion_tokens=3, total_tokens=300_004), ) - assert rates[0] == 0.001 - assert rates[1] == (0.005 if field == "output_cost_per_token" else 0.002) - assert rates[2] == (0.005 if field == "cache_creation_input_token_cost" else 0.004) - assert rates[4] == (0.005 if field == "cache_read_input_token_cost" else 0.003) + assert rates.input_rate == 0.001 + assert rates.output_rate == (0.005 if field == "output_cost_per_token" else 0.002) + assert rates.cache_creation_rate == (0.005 if field == "cache_creation_input_token_cost" else 0.004) + assert rates.cache_read_rate == (0.005 if field == "cache_read_input_token_cost" else 0.003) @pytest.mark.parametrize("prompt_tokens", (200_000, 200_001, 512_000, 512_001)) @@ -2599,10 +2690,12 @@ def test_mixed_token_thresholds_preserve_each_crossed_rate(prompt_tokens: int, h model_info=info, usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=3, total_tokens=prompt_tokens + 3), ) - assert rates[0] == (0.005 if prompt_tokens > 200_000 else 0.001) - assert rates[1] == (higher_rate if prompt_tokens > 512_000 else 0.006 if prompt_tokens > 200_000 else 0.002) - assert rates[2] == (0.007 if prompt_tokens > 200_000 else 0.004) - assert rates[4] == (higher_rate if prompt_tokens > 512_000 else 0.003) + assert rates.input_rate == (0.005 if prompt_tokens > 200_000 else 0.001) + assert rates.output_rate == ( + higher_rate if prompt_tokens > 512_000 else 0.006 if prompt_tokens > 200_000 else 0.002 + ) + assert rates.cache_creation_rate == (0.007 if prompt_tokens > 200_000 else 0.004) + assert rates.cache_read_rate == (higher_rate if prompt_tokens > 512_000 else 0.003) def test_threshold_keys_exclude_service_tier_variants(): @@ -2626,8 +2719,7 @@ def test_threshold_keys_exclude_service_tier_variants(): }, ) usage = Usage(prompt_tokens=350_000, completion_tokens=1_000, total_tokens=351_000) - prompt_base, *_ = _get_token_base_cost(model_info=model_info, usage=usage) - assert prompt_base == 3e-6 + assert _get_token_base_cost(model_info=model_info, usage=usage).input_rate == 3e-6 @pytest.mark.parametrize( @@ -2897,7 +2989,7 @@ def test_billed_token_rates_follow_the_token_tier_the_breakdown_bills_at(monkeyp cache_read_input_token_cost=6e-7, cache_read_input_audio_token_cost=6e-7, cache_creation_input_token_cost=7.5e-6, - cache_creation_input_token_cost_above_1hr=7.5e-6, + cache_creation_input_token_cost_by_ttl={"1hr": 7.5e-6, "12hr": 7.5e-6}, output_cost_per_reasoning_token=3e-5, ) assert breakdown.cache_read_cost == pytest.approx(200_000 * rates.cache_read_input_token_cost) @@ -3713,6 +3805,23 @@ def test_generic_cost_per_token_bills_cache_creation_at_the_input_rate_without_a 3e-7, id="no-1h-price-uses-the-off-peak-write-price", ), + pytest.param( + { + "tiered_pricing": [ + { + "range": [0, 128000], + "input_cost_per_token": 2e-7, + "output_cost_per_token": 1.25e-6, + "cache_creation_input_token_cost": 2.5e-7, + } + ], + "off_peak_pricing": {"hours_utc": "00:00-23:59", "cache_creation_input_token_cost": 3e-7}, + }, + datetime(2026, 9, 14, 12, tzinfo=timezone.utc), + 3e-7, + 3e-7, + id="tiered-no-1h-price-uses-the-off-peak-write-price", + ), ), ) def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_path( @@ -3724,10 +3833,17 @@ def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_ model_info = {"input_cost_per_token": 2e-7, "output_cost_per_token": 1.25e-6, **cache_rates} usage = Usage(prompt_tokens=10, completion_tokens=1, total_tokens=11) - _, _, creation, creation_1h, _ = _get_token_base_cost(model_info, usage, current_time=current_time) + rates = _get_token_base_cost(model_info, usage, current_time=current_time) - assert creation == pytest.approx(expected_creation) - assert creation_1h == pytest.approx(expected_creation_1h) + assert rates.cache_creation_rate == pytest.approx(expected_creation) + assert calculate_cache_writing_cost( + cache_creation_tokens=3, + cache_creation_token_details=CacheCreationTokenDetails( + ephemeral_1h_input_tokens=1, ephemeral_12h_input_tokens=2 + ), + cache_creation_cost=rates.cache_creation_rate, + cache_creation_cost_by_ttl=rates.cache_creation_rates_by_ttl, + ) == pytest.approx(3 * expected_creation_1h) def _image_response(num_images: int = 1, usage: ImageUsage | None = None) -> ImageResponse: @@ -3787,8 +3903,6 @@ def test_route_image_generation_cost_openai_honors_deployment_input_cost_per_ima assert cost == pytest.approx(0.07) - - @pytest.mark.parametrize( ("custom_llm_provider", "model"), [ @@ -4206,17 +4320,28 @@ def test_token_breakdown_uses_supplied_pricing_without_catalog_lookup(rate: floa "cache_creation_input_token_cost": rate, } usage: Final = Usage( - prompt_tokens=20, completion_tokens=3, total_tokens=23, - prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=5), cache_creation_input_tokens=5, + prompt_tokens=20, + completion_tokens=3, + total_tokens=23, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=5), + cache_creation_input_tokens=5, ) prompt_cost, output_cost = generic_cost_per_token( - model="supplied-pricing-test", usage=usage, custom_llm_provider="bedrock", model_info=model_info, + model="supplied-pricing-test", + usage=usage, + custom_llm_provider="bedrock", + model_info=model_info, ) breakdown: Final = get_token_type_cost_breakdown( - model="supplied-pricing-test", usage=usage, custom_llm_provider="bedrock", model_info=model_info, + model="supplied-pricing-test", + usage=usage, + custom_llm_provider="bedrock", + model_info=model_info, ) assert breakdown.cache_read_cost == pytest.approx(5 * rate) assert breakdown.cache_creation_cost == pytest.approx(5 * rate) assert breakdown.rates is not None - assert prompt_cost == pytest.approx(10 * breakdown.rates.input_cost_per_token + breakdown.cache_read_cost + breakdown.cache_creation_cost) + assert prompt_cost == pytest.approx( + 10 * breakdown.rates.input_cost_per_token + breakdown.cache_read_cost + breakdown.cache_creation_cost + ) assert output_cost == pytest.approx(3 * breakdown.rates.output_cost_per_token) diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 30e13442c92..023d0ea26fb 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -3398,7 +3398,9 @@ class TestUsageTransformation: assert response_usage.input_tokens_details is not None assert response_usage.input_tokens_details.cached_tokens == 5 assert response_usage.input_tokens_details.text_tokens == 8 - assert "cache_write_tokens" not in response_usage.input_tokens_details.model_dump() + assert {"cache_write_tokens", "cache_write_tokens_details"}.isdisjoint( + response_usage.input_tokens_details.model_dump() + ) def test_transform_usage_with_cached_tokens_gemini(self): """Test that cached_tokens from Gemini are properly transformed to input_tokens_details""" diff --git a/tests/unit/responses/test_responses_utils.py b/tests/unit/responses/test_responses_utils.py index 6ee2c495705..97a39f64001 100644 --- a/tests/unit/responses/test_responses_utils.py +++ b/tests/unit/responses/test_responses_utils.py @@ -1,5 +1,6 @@ from importlib import import_module import base64 +from typing import Final from unittest.mock import MagicMock, patch import pytest @@ -7,10 +8,11 @@ import pytest import litellm from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse -from litellm.types.utils import Usage +from litellm.types.utils import ModelInfo, Usage from litellm.utils import _invalidate_model_cost_lowercase_map @@ -413,6 +415,43 @@ class TestResponseAPILoggingUtils: assert result.prompt_tokens_details.cache_creation_tokens == 10059 assert result.prompt_tokens_details.cached_tokens == 0 + @pytest.mark.parametrize("as_model", [False, True]) + def test_transform_response_api_usage_prices_12h_cache_writes(self, as_model: bool) -> None: + usage_data: Final = { + "input_tokens": 5_000, + "output_tokens": 0, + "total_tokens": 5_000, + "input_tokens_details": { + "cache_write_tokens": 5_000, + "cache_write_tokens_details": {"ttl_30m_tokens": 3_000, "ttl_12h_tokens": 2_000}, + }, + } + usage_input: Final = ResponseAPIUsage(**usage_data) if as_model else usage_data + usage: Final = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage_input) + model_info: Final[ModelInfo] = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_creation_input_token_cost": 3e-6, + "cache_creation_input_token_cost_above_12hr": 5e-6, + } + + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.cache_write_tokens == 5_000 + assert usage.prompt_tokens_details.cache_creation_token_details is not None + assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_12h_input_tokens == 2_000 + assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_30m_input_tokens == 3_000 + cost: Final = generic_cost_per_token( + model="test-12h-cache-write", + usage=usage, + custom_llm_provider="openai", + model_info=model_info, + )[0] + + assert cost == pytest.approx( + 3_000 * model_info["cache_creation_input_token_cost"] + + 2_000 * model_info["cache_creation_input_token_cost_above_12hr"] + ) + def test_transform_response_api_usage_mixed_details(self): """Test transformation handles mixed token details (cached + image + audio).""" # Setup - hypothetical usage with mixed token types diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ad069aa987b..fcd6b8aa402 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -939,6 +939,18 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_creation_input_audio_token_cost": {"type": "number"}, "cache_creation_input_token_cost": {"type": "number"}, "cache_creation_input_token_cost_above_1hr": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_flex": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_priority": {"type": "number"}, "cache_creation_input_token_cost_above_32k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_100k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_100k_tokens_batches": {"type": "number"}, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 1d36f5fd9a2..da9e77e9b58 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -28439,6 +28439,15 @@ export interface components { */ uncached_input_tokens: number; }; + /** CacheWriteTokensDetails */ + CacheWriteTokensDetails: { + /** Ttl 12H Tokens */ + ttl_12h_tokens: number; + /** Ttl 30M Tokens */ + ttl_30m_tokens: number; + } & { + [key: string]: unknown; + }; /** CachedTokensDetails */ CachedTokensDetails: { /** Audio Tokens */ @@ -33589,6 +33598,9 @@ export interface components { InputTokensDetails: { /** Audio Tokens */ audio_tokens?: number | null; + /** Cache Write Tokens */ + cache_write_tokens?: number | null; + cache_write_tokens_details?: components["schemas"]["CacheWriteTokensDetails"] | null; /** * Cached Tokens * @default 0 @@ -35207,6 +35219,18 @@ export interface components { cache_creation_input_token_cost_above_100k_tokens?: number | null; /** Cache Creation Input Token Cost Above 100K Tokens Batches */ cache_creation_input_token_cost_above_100k_tokens_batches?: number | null; + /** Cache Creation Input Token Cost Above 12Hr */ + cache_creation_input_token_cost_above_12hr?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Flex */ + cache_creation_input_token_cost_above_12hr_flex?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Priority */ + cache_creation_input_token_cost_above_12hr_priority?: number | null; /** Cache Creation Input Token Cost Above 1Hr */ cache_creation_input_token_cost_above_1hr?: number | null; /** Cache Creation Input Token Cost Above 1Hr Above 100K Tokens */ @@ -41506,9 +41530,9 @@ export interface components { mode?: "implicit" | "explicit"; /** * Ttl - * @constant + * @enum {string} */ - ttl?: "30m"; + ttl?: "30m" | "12h"; }; /** PromptCachingRequest */ PromptCachingRequest: { @@ -50199,6 +50223,18 @@ export interface components { blocked?: boolean | null; /** Cache Creation Input Token Cost */ cache_creation_input_token_cost?: number | null; + /** Cache Creation Input Token Cost Above 12Hr */ + cache_creation_input_token_cost_above_12hr?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Flex */ + cache_creation_input_token_cost_above_12hr_flex?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Priority */ + cache_creation_input_token_cost_above_12hr_priority?: number | null; /** Cache Creation Input Token Cost Above 1Hr */ cache_creation_input_token_cost_above_1hr?: number | null; /** Cache Read Input Token Cost */ @@ -50492,6 +50528,18 @@ export interface components { cache_creation_input_token_cost_above_100k_tokens?: number | null; /** Cache Creation Input Token Cost Above 100K Tokens Batches */ cache_creation_input_token_cost_above_100k_tokens_batches?: number | null; + /** Cache Creation Input Token Cost Above 12Hr */ + cache_creation_input_token_cost_above_12hr?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Flex */ + cache_creation_input_token_cost_above_12hr_flex?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Priority */ + cache_creation_input_token_cost_above_12hr_priority?: number | null; /** Cache Creation Input Token Cost Above 1Hr */ cache_creation_input_token_cost_above_1hr?: number | null; /** Cache Creation Input Token Cost Above 1Hr Above 100K Tokens */ From dc0237ccd722c7e4d010b9a29ff4ba22af217f5b Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sat, 24 Oct 2026 09:12:31 -0700 Subject: [PATCH 006/155] fix(cost): bill gemini audio input and cached audio at flex, priority and batch rates (#48086) * fix(gemini): price tiered audio token rates * fix: correct Gemini audio pricing catalog * fix(gemini): preserve cached modality pricing --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 6 +- .../llm_cost_calc/zero_cost_diagnostic.py | 29 +++- .../vertex_and_google_ai_studio_gemini.py | 31 ++-- ...odel_prices_and_context_window_backup.json | 16 ++ litellm/types/utils.py | 10 ++ litellm/utils.py | 9 ++ model_prices_and_context_window.json | 16 ++ .../llm_cost_calc/test_utils.py | 62 ++++++++ .../test_zero_cost_diagnostic.py | 31 ++++ ...test_vertex_and_google_ai_studio_gemini.py | 147 +++++++++++++++++- tests/unit/test_cost_calculator.py | 77 +++++---- tests/unit/test_utils.py | 4 + tests/unit/test_utils_get_model_info.py | 24 +++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 20 +++ 14 files changed, 429 insertions(+), 53 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 99944662bf8..98f6f1409f2 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2879,6 +2879,7 @@ def _batch_rate( model_info: ModelInfo, key: Literal[ "input_cost_per_audio_token_batches", + "cache_read_input_audio_token_cost_batches", "input_cost_per_image_token_batches", "input_cost_per_video_token_batches", ], @@ -3012,6 +3013,7 @@ def _batch_prompt_cost( ) -> float: details: Final = parse_prompt_tokens_details(usage) cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0 + cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0 written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0 audio_tokens, image_tokens, video_tokens = ( details["audio_tokens"], @@ -3026,7 +3028,9 @@ def _batch_prompt_cost( + audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate) + image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate) + video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate) - + cached_tokens * (cache_read_rate or 0.0) + + (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0) + + cached_audio_tokens + * _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0) + written_tokens * (cache_creation_rate or 0.0) ) diff --git a/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py b/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py index 9688b511ea8..3ed598b5e97 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py +++ b/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py @@ -5,6 +5,7 @@ from typing import Final from pydantic import TypeAdapter, ValidationError from typing_extensions import assert_never +from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details from litellm.types.utils import ( CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, @@ -16,6 +17,8 @@ ZERO_COST_COUNTER_NAME: Final = "litellm_zero_cost_requests_total" _TEXT_INPUT_RATE: Final = "input_cost_per_token" _AUDIO_INPUT_RATE: Final = "input_cost_per_audio_token" +_CACHE_READ_INPUT_RATE: Final = "cache_read_input_token_cost" +_CACHE_READ_AUDIO_INPUT_RATE: Final = "cache_read_input_audio_token_cost" _TEXT_OUTPUT_RATE: Final = "output_cost_per_token" _AUDIO_OUTPUT_RATE: Final = "output_cost_per_audio_token" _RATE_KEY_MARKERS: Final = ("cost", "pricing") @@ -33,19 +36,38 @@ def _tokens(value: object) -> int: def used_pricing_keys(usage: Usage) -> tuple[str, ...]: - prompt_audio: Final = _audio_tokens(usage.prompt_tokens_details) + prompt_details: Final = parse_prompt_tokens_details(usage) + prompt_audio: Final = prompt_details["audio_tokens"] completion_audio: Final = _audio_tokens(usage.completion_tokens_details) - prompt_text: Final = _tokens(usage.prompt_tokens) - prompt_audio + prompt_text: Final = max( + _tokens(usage.prompt_tokens) + - prompt_details["cache_hit_tokens"] + - prompt_audio + - prompt_details["image_tokens"] + - prompt_details["video_tokens"], + 0, + ) + cached_text: Final = max(prompt_details["cache_hit_tokens"] - prompt_details["cache_hit_audio_tokens"], 0) completion_text: Final = _tokens(usage.completion_tokens) - completion_audio components: Final = ( (_TEXT_INPUT_RATE, prompt_text), (_AUDIO_INPUT_RATE, prompt_audio), + (_CACHE_READ_INPUT_RATE, cached_text), + (_CACHE_READ_AUDIO_INPUT_RATE, prompt_details["cache_hit_audio_tokens"]), (_TEXT_OUTPUT_RATE, completion_text), (_AUDIO_OUTPUT_RATE, completion_audio), ) return tuple(key for key, count in components if count > 0) +def _effective_pricing_key(key: str, pricing_entry: Mapping[str, object]) -> str: + if key == _CACHE_READ_AUDIO_INPUT_RATE and pricing_entry.get(key) is None: + return _CACHE_READ_INPUT_RATE if pricing_entry.get(_CACHE_READ_INPUT_RATE) is not None else _TEXT_INPUT_RATE + if key == _CACHE_READ_INPUT_RATE and pricing_entry.get(key) is None: + return _TEXT_INPUT_RATE + return key + + def _nested_pricing(value: object) -> Mapping[str, object] | tuple[object, ...] | None: try: return _NESTED_PRICING.validate_python(value) @@ -89,7 +111,8 @@ def diagnose_zero_cost( pricing_entry: Mapping[str, object], calculation_failed: bool, ) -> StandardLoggingZeroCostDiagnostic | None: - used_keys: Final = used_pricing_keys(usage) + effective_keys: Final = tuple(_effective_pricing_key(key, pricing_entry) for key in used_pricing_keys(usage)) + used_keys: Final = tuple(key for index, key in enumerate(effective_keys) if key not in effective_keys[:index]) if not used_keys: return None missing_keys: Final = tuple(key for key in used_keys if pricing_entry.get(key) is None) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index ce632bc0eef..d788ccb0f1d 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -42,6 +42,7 @@ from litellm.llms.custom_httpx.http_handler import ( get_httpx_client, ) from litellm.types.llms.anthropic import AnthropicThinkingParam +from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.gemini import BidiGenerateContentServerMessage from litellm.types.llms.openai import ( AllMessageValues, @@ -2364,26 +2365,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): elif modality == "DOCUMENT": cached_text_tokens = (cached_text_tokens or 0) + token_count - ## Calculate non-cached tokens by subtracting cached from total (per modality) - ## This is necessary because promptTokensDetails includes both cached and non-cached tokens - ## See: https://github.com/BerriAI/litellm/issues/18750 - if cached_text_tokens is not None and prompt_text_tokens is not None: - # Explicit caching: subtract cached tokens per modality from cacheTokensDetails - prompt_text_tokens = prompt_text_tokens - cached_text_tokens - elif ( + if ( cached_tokens is not None and prompt_text_tokens is not None and cached_text_tokens is None and "cacheTokensDetails" not in usage_metadata ): - # Implicit caching: only cachedContentTokenCount is provided (no cacheTokensDetails) - # Subtract from text tokens since implicit caching is primarily for text content - # See: https://github.com/BerriAI/litellm/issues/16341 prompt_text_tokens = prompt_text_tokens - cached_tokens - if cached_audio_tokens is not None and prompt_audio_tokens is not None: - prompt_audio_tokens = prompt_audio_tokens - cached_audio_tokens - if cached_image_tokens is not None and prompt_image_tokens is not None: - prompt_image_tokens = prompt_image_tokens - cached_image_tokens if cached_video_tokens is not None and prompt_video_tokens is not None: prompt_video_tokens = prompt_video_tokens - cached_video_tokens @@ -2404,9 +2392,24 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): billable_text_tokens: Final = ( prompt_text_tokens if prompt_text_tokens is None else prompt_text_tokens + billable_tool_use_prompt_tokens ) + cached_tokens_details_fields: Final = MappingProxyType( + { + key: token_count + for key, token_count in ( + ("text_tokens", cached_text_tokens), + ("audio_tokens", cached_audio_tokens), + ("image_tokens", cached_image_tokens), + ) + if token_count is not None + } + ) + cached_tokens_details: Final = ( + CachedTokensDetails.model_validate(cached_tokens_details_fields) if cached_tokens_details_fields else None + ) prompt_tokens_details = PromptTokensDetailsWrapper( cached_tokens=cached_tokens, + cached_tokens_details=cached_tokens_details, audio_tokens=prompt_audio_tokens, text_tokens=billable_text_tokens, image_tokens=prompt_image_tokens, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8c0c6dffdc7..15dfd196c6f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -31137,11 +31137,15 @@ }, "gemini/gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, + "cache_read_input_audio_token_cost_batches": 1e-07, + "cache_read_input_audio_token_cost_flex": 1e-07, + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 3e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -31466,11 +31470,15 @@ }, "gemini/gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, + "cache_read_input_audio_token_cost_batches": 3e-08, + "cache_read_input_audio_token_cost_flex": 3e-08, + "cache_read_input_audio_token_cost_priority": 5.4e-08, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, "input_cost_per_audio_token": 3e-07, + "input_cost_per_audio_token_flex": 1.5e-07, "input_cost_per_token": 1e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -31760,12 +31768,16 @@ }, "gemini/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, + "cache_read_input_audio_token_cost_batches": 2.5e-08, + "cache_read_input_audio_token_cost_flex": 2.5e-08, + "cache_read_input_audio_token_cost_priority": 9e-08, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-05-07", "input_cost_per_audio_token": 5e-07, + "input_cost_per_audio_token_flex": 2.5e-07, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_flex": 1.25e-07, @@ -31886,10 +31898,14 @@ }, "gemini/gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, + "cache_read_input_audio_token_cost_batches": 1e-07, + "cache_read_input_audio_token_cost_flex": 1e-07, + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ed66cbd8cf0..270d9b43dea 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -333,6 +333,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing cache_read_input_token_cost: float | None cache_read_input_audio_token_cost: ReadOnly[float | None] + cache_read_input_audio_token_cost_flex: ReadOnly[float | None] + cache_read_input_audio_token_cost_priority: ReadOnly[float | None] cache_read_input_image_token_cost: ReadOnly[float | None] explicit_cache_read_input_token_cost: ReadOnly[float | None] cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing @@ -348,6 +350,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_read_input_token_cost_above_272k_tokens_ultrafast: ReadOnly[float | None] cache_read_input_token_cost_above_512k_tokens: float | None cache_read_input_token_cost_batches: ReadOnly[float | None] + cache_read_input_audio_token_cost_batches: ReadOnly[float | None] cache_read_input_token_cost_above_200k_tokens_batches: ReadOnly[float | None] cache_read_input_token_cost_above_100k_tokens_batches: ReadOnly[float | None] cache_read_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None] @@ -361,6 +364,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): off_peak_pricing: ReadOnly[OffPeakPricing | None] # time-windowed off-peak rates input_cost_per_character: float | None # only for vertex ai models input_cost_per_audio_token: float | None + input_cost_per_audio_token_flex: ReadOnly[float | None] + input_cost_per_audio_token_priority: ReadOnly[float | None] input_cost_per_token_above_128k_tokens: float | None # only for vertex ai models input_cost_per_token_above_200k_tokens: float | None # only for vertex ai gemini-2.5-pro models input_cost_per_token_above_100k_tokens: ReadOnly[float | None] @@ -4020,14 +4025,19 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): cache_read_input_token_cost_above_100k_tokens_batches: float | None = None cache_read_input_token_cost_above_200k_tokens_batches: float | None = None cache_read_input_token_cost_above_272k_tokens_batches: float | None = None + cache_read_input_audio_token_cost_batches: float | None = None cache_creation_input_token_cost_batches: float | None = None cache_creation_input_token_cost_above_100k_tokens_batches: float | None = None cache_creation_input_token_cost_above_200k_tokens_batches: float | None = None cache_creation_input_token_cost_above_272k_tokens_batches: float | None = None cache_read_input_audio_token_cost: float | None = None + cache_read_input_audio_token_cost_flex: float | None = None + cache_read_input_audio_token_cost_priority: float | None = None cache_read_input_image_token_cost: float | None = None input_cost_per_character_above_128k_tokens: float | None = None input_cost_per_audio_token: float | None = None + input_cost_per_audio_token_flex: float | None = None + input_cost_per_audio_token_priority: float | None = None input_cost_per_token_cache_hit: float | None = None input_cost_per_token_above_128k_tokens: float | None = None input_cost_per_token_above_100k_tokens: float | None = None diff --git a/litellm/utils.py b/litellm/utils.py index 58eaa43e687..bb94ca6d40a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6773,6 +6773,13 @@ def get_model_info_helper( ), cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None), cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None), + cache_read_input_audio_token_cost_flex=_model_info.get("cache_read_input_audio_token_cost_flex", None), + cache_read_input_audio_token_cost_priority=_model_info.get( + "cache_read_input_audio_token_cost_priority", None + ), + cache_read_input_audio_token_cost_batches=_model_info.get( + "cache_read_input_audio_token_cost_batches", None + ), explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( @@ -6858,6 +6865,8 @@ def get_model_info_helper( input_cost_per_second=_model_info.get("input_cost_per_second", None), input_cost_per_text_message=_model_info.get("input_cost_per_text_message", None), input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None), + input_cost_per_audio_token_flex=_model_info.get("input_cost_per_audio_token_flex", None), + input_cost_per_audio_token_priority=_model_info.get("input_cost_per_audio_token_priority", None), input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None), input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None), input_cost_per_audio_token_batches=_model_info.get("input_cost_per_audio_token_batches", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8c0c6dffdc7..15dfd196c6f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -31137,11 +31137,15 @@ }, "gemini/gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, + "cache_read_input_audio_token_cost_batches": 1e-07, + "cache_read_input_audio_token_cost_flex": 1e-07, + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 3e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -31466,11 +31470,15 @@ }, "gemini/gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, + "cache_read_input_audio_token_cost_batches": 3e-08, + "cache_read_input_audio_token_cost_flex": 3e-08, + "cache_read_input_audio_token_cost_priority": 5.4e-08, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, "input_cost_per_audio_token": 3e-07, + "input_cost_per_audio_token_flex": 1.5e-07, "input_cost_per_token": 1e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -31760,12 +31768,16 @@ }, "gemini/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, + "cache_read_input_audio_token_cost_batches": 2.5e-08, + "cache_read_input_audio_token_cost_flex": 2.5e-08, + "cache_read_input_audio_token_cost_priority": 9e-08, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-05-07", "input_cost_per_audio_token": 5e-07, + "input_cost_per_audio_token_flex": 2.5e-07, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_flex": 1.25e-07, @@ -31886,10 +31898,14 @@ }, "gemini/gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, + "cache_read_input_audio_token_cost_batches": 1e-07, + "cache_read_input_audio_token_cost_flex": 1e-07, + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py index 142d49bfff1..b6c18e0da86 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py @@ -11,6 +11,7 @@ import respx import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.types.llms.base import CachedTokensDetails from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage TIER_MODEL: Final = "tier-priced-test-model" @@ -31,6 +32,20 @@ PROMPT_TOKENS: Final = 1000 CACHED_TOKENS: Final = 200 COMPLETION_TOKENS: Final = 500 TIER_API_BASE: Final = "https://tier-pricing.invalid/v1" +AUDIO_TIER_MODEL: Final = "tier-priced-gemini-audio-test-model" +AUDIO_TIER_ROW: Final[Mapping[str, float]] = MappingProxyType( + { + "input_cost_per_token": 4e-06, + "output_cost_per_token": 8e-06, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_audio_token": 3e-06, + "cache_read_input_audio_token_cost": 2e-06, + "input_cost_per_audio_token_flex": 1e-06, + "input_cost_per_audio_token_priority": 2e-06, + "cache_read_input_audio_token_cost_flex": 4e-07, + "cache_read_input_audio_token_cost_priority": 8e-07, + } +) def _cost_at(prices: Mapping[str, float], column_suffix: str) -> float: @@ -78,6 +93,53 @@ def test_completion_cost_bills_the_price_columns_of_the_service_tier( assert cost == pytest.approx(_cost_at(TIER_ROW, column_suffix)) +@pytest.mark.parametrize( + ("service_tier", "audio_rate_key", "cache_audio_rate_key"), + ( + pytest.param("flex", "input_cost_per_audio_token_flex", "cache_read_input_audio_token_cost_flex"), + pytest.param("priority", "input_cost_per_audio_token_priority", "cache_read_input_audio_token_cost_priority"), + ), +) +def test_gemini_audio_and_cached_audio_use_service_tier_rates( + local_model_cost_map: None, service_tier: str, audio_rate_key: str, cache_audio_rate_key: str +) -> None: + litellm.register_model( + { + AUDIO_TIER_MODEL: { + "litellm_provider": "gemini", + "mode": "chat", + **AUDIO_TIER_ROW, + } + } + ) + usage: Final = Usage( + prompt_tokens=40, + completion_tokens=0, + total_tokens=40, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=20, + audio_tokens=20, + cached_tokens=10, + cached_tokens_details=CachedTokensDetails(audio_tokens=10), + ), + ) + response: Final = ModelResponse(model=AUDIO_TIER_MODEL, usage=usage) + + cost: Final = litellm.completion_cost( + completion_response=response, + model=AUDIO_TIER_MODEL, + custom_llm_provider="gemini", + service_tier=service_tier, + ) + + expected: Final = ( + 20 * AUDIO_TIER_ROW["input_cost_per_token"] + + 10 * AUDIO_TIER_ROW[audio_rate_key] + + 10 * AUDIO_TIER_ROW[cache_audio_rate_key] + ) + assert cost == pytest.approx(expected) + + LONG_CONTEXT_TIER_MODEL: Final = "long-context-tier-priced-test-model" LONG_CONTEXT_TIER_ROW: Final[Mapping[str, float]] = MappingProxyType( { diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py index 921dde8bf7b..bdc14c4934b 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py @@ -9,6 +9,7 @@ from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import ( used_pricing_keys, zero_cost_warning, ) +from litellm.types.llms.base import CachedTokensDetails from litellm.types.utils import CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, Usage PER_SECOND_ENTRY: Final = {"cost_per_second": 0.00042} @@ -135,6 +136,36 @@ def test_audio_tokens_need_the_audio_rates() -> None: assert diagnostic["missing_pricing_keys"] == ("input_cost_per_audio_token", "output_cost_per_audio_token") +def test_cached_audio_uses_its_cache_rate_in_zero_cost_diagnostic() -> None: + usage: Final = Usage( + prompt_tokens=20, + completion_tokens=0, + total_tokens=20, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=0, + audio_tokens=20, + cached_tokens=20, + cached_tokens_details=CachedTokensDetails(audio_tokens=20), + ), + ) + pricing_entry: Final = { + "input_cost_per_token": 1e-6, + "input_cost_per_audio_token": 2e-6, + "cache_read_input_audio_token_cost": 0, + } + + assert used_pricing_keys(usage) == ("cache_read_input_audio_token_cost",) + assert ( + diagnose_zero_cost( + usage=usage, + pricing_model="gemini-audio", + pricing_entry=pricing_entry, + calculation_failed=False, + ) + is None + ) + + def test_warning_names_the_request_the_entry_the_missing_keys_and_the_counter() -> None: diagnostic = diagnose_zero_cost( usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=PER_SECOND_ENTRY, calculation_failed=False diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 4c3ee9e7ca5..3bc864cc047 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -16,8 +16,10 @@ from pydantic import BaseModel import litellm from litellm import ModelResponse, completion, embedding, image_generation from litellm._logging import verbose_logger +from litellm.cost_calculator import batch_cost_calculator from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token, parse_prompt_tokens_details from litellm.llms.anthropic.pass_through.messages import handler as anthropic_messages_handler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig @@ -30,8 +32,9 @@ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( normalize_gemini_speech_config, reject_unknown_gemini_tts_voice_keys, ) +from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.vertex_ai import GeminiFinishReason, RequestBody, UsageMetadata -from litellm.types.utils import ChoiceLogprobs, Usage +from litellm.types.utils import ChoiceLogprobs, ModelInfo, Usage from litellm.utils import _invalidate_model_cost_lowercase_map, CustomStreamWrapper from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.integrations.custom_logger import CustomLogger @@ -1515,6 +1518,7 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities(): v = VertexGeminiConfig() usage_metadata = { "promptTokenCount": 210, + "cachedContentTokenCount": 54, "candidatesTokenCount": 50, "totalTokenCount": 260, "promptTokensDetails": [ @@ -1536,9 +1540,10 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities(): usage_metadata = UsageMetadata(**usage_metadata) result = v.calculate_usage(completion_response={"usageMetadata": usage_metadata}) - # prompt details are total - cached per modality - assert result.prompt_tokens_details.text_tokens == 16 # 20 - 4 - assert result.prompt_tokens_details.image_tokens == 140 # (90 + 100) - (40 + 10) + assert result.prompt_tokens_details.text_tokens == 20 + assert result.prompt_tokens_details.image_tokens == 190 + assert result.prompt_tokens_details.cached_tokens == 54 + assert result.prompt_tokens_details.cached_tokens_details == CachedTokensDetails(text_tokens=4, image_tokens=50) # candidates details accumulate duplicate modalities assert result.completion_tokens_details.text_tokens == 20 # 15 + 5 @@ -1546,6 +1551,118 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities(): assert result.completion_tokens == 50 +def _gemini_usage_with_cached_audio(cached_audio_tokens: int) -> Usage: + cache_tokens_details: Final = ( + ( + {"modality": "TEXT", "tokenCount": 5}, + {"modality": "AUDIO", "tokenCount": cached_audio_tokens}, + ) + if cached_audio_tokens > 0 + else ({"modality": "TEXT", "tokenCount": 5},) + ) + completion_response: Final = { + "usageMetadata": { + "promptTokenCount": 40, + "cachedContentTokenCount": 5 + cached_audio_tokens, + "promptTokensDetails": ( + {"modality": "TEXT", "tokenCount": 20}, + {"modality": "AUDIO", "tokenCount": 20}, + ), + "cacheTokensDetails": cache_tokens_details, + "candidatesTokenCount": 7, + "totalTokenCount": 47, + } + } + return VertexGeminiConfig.calculate_usage(completion_response=completion_response) + + +def test_gemini_raw_usage_metadata_prices_cached_audio_at_its_batch_rate() -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 2e-6, + "input_cost_per_token_batches": 1e-6, + "input_cost_per_audio_token": 6e-6, + "input_cost_per_audio_token_batches": 3e-6, + "cache_read_input_token_cost": 8e-7, + "cache_read_input_token_cost_batches": 4e-7, + "cache_read_input_audio_token_cost": 5e-7, + "cache_read_input_audio_token_cost_batches": 2e-7, + } + usage: Final = _gemini_usage_with_cached_audio(5) + expected_prompt_cost: Final = ( + 15 * model_info["input_cost_per_token_batches"] + + 15 * model_info["input_cost_per_audio_token_batches"] + + 5 * model_info["cache_read_input_token_cost_batches"] + + 5 * model_info["cache_read_input_audio_token_cost_batches"] + ) + + prompt_cost, _ = batch_cost_calculator( + usage=usage, + model="gemini-audio-batch-test-model", + custom_llm_provider="gemini", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx(expected_prompt_cost) + + +def test_gemini_raw_usage_metadata_prices_cached_audio_at_its_standard_rate() -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 2e-6, + "input_cost_per_audio_token": 6e-6, + "cache_read_input_token_cost": 8e-7, + "cache_read_input_audio_token_cost": 5e-7, + } + usage: Final = _gemini_usage_with_cached_audio(5) + expected_prompt_cost: Final = ( + 15 * model_info["input_cost_per_token"] + + 15 * model_info["input_cost_per_audio_token"] + + 5 * model_info["cache_read_input_token_cost"] + + 5 * model_info["cache_read_input_audio_token_cost"] + ) + + prompt_cost, _ = generic_cost_per_token( + model="gemini-audio-standard-test-model", + usage=usage, + custom_llm_provider="gemini", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx(expected_prompt_cost) + + +def test_gemini_usage_totals_are_subtracted_once_and_audio_cache_rate_matches_audio_rate() -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 2e-6, + "input_cost_per_audio_token": 6e-6, + "cache_read_input_token_cost": 8e-7, + "cache_read_input_audio_token_cost": 6e-6, + } + cached_usage: Final = _gemini_usage_with_cached_audio(5) + uncached_usage: Final = _gemini_usage_with_cached_audio(0) + cached_details: Final = parse_prompt_tokens_details(cached_usage) + cached_prompt_cost, _ = generic_cost_per_token( + model="gemini-audio-standard-test-model", + usage=cached_usage, + custom_llm_provider="gemini", + model_info=model_info, + ) + uncached_prompt_cost, _ = generic_cost_per_token( + model="gemini-audio-standard-test-model", + usage=uncached_usage, + custom_llm_provider="gemini", + model_info=model_info, + ) + + assert cached_usage.prompt_tokens_details.text_tokens == 20 + assert cached_usage.prompt_tokens_details.audio_tokens == 20 + assert cached_usage.prompt_tokens_details.cached_tokens_details == CachedTokensDetails( + text_tokens=5, audio_tokens=5 + ) + assert cached_details["text_tokens"] == 15 + assert cached_details["audio_tokens"] == 15 + assert cached_prompt_cost == pytest.approx(uncached_prompt_cost) + + def test_vertex_ai_map_thinking_param_with_budget_tokens_0(): """ If budget_tokens is 0, do not set includeThoughts to True @@ -6150,7 +6267,7 @@ def test_vertex_ai_usage_metadata_with_document_tokens_in_prompt(): def test_vertex_ai_usage_metadata_with_document_tokens_cached(): - """Test that cached DOCUMENT tokens are correctly subtracted from prompt text tokens.""" + """Test cached DOCUMENT token details and unchanged prompt cost.""" v = VertexGeminiConfig() usage_metadata_dict = { @@ -6174,10 +6291,24 @@ def test_vertex_ai_usage_metadata_with_document_tokens_cached(): completion_response = {"usageMetadata": usage_metadata_dict} result = v.calculate_usage(completion_response=completion_response) - # DOCUMENT cached tokens map to cached_text_tokens, so: - # text_tokens = (8 TEXT + 774 DOCUMENT) - 400 cached = 382 - assert result.prompt_tokens_details.text_tokens == 382, "text_tokens should be (8 + 774) - 400 cached = 382" + assert result.prompt_tokens_details.text_tokens == 782 assert result.prompt_tokens_details.cached_tokens == 400 + assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 400 + + model_info: Final[ModelInfo] = { + "input_cost_per_token": 2e-6, + "cache_read_input_token_cost": 5e-7, + } + prompt_cost, _ = generic_cost_per_token( + model="gemini-document-cache-test-model", + usage=result, + custom_llm_provider="gemini", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx( + 382 * model_info["input_cost_per_token"] + 400 * model_info["cache_read_input_token_cost"] + ) def test_async_streaming_uses_custom_client(): diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 5d4eb5fae12..550fe5f0d1a 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -16,6 +16,7 @@ import litellm from litellm import TranscriptionResponse, model_cost from litellm.constants import REALTIME_METERED_USAGE_KEY from litellm.cost_calculator import BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, batch_cost_calculator, completion_cost, cost_per_token, handle_realtime_stream_cost_calculation, response_cost_calculator +from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.fallback_generalizations import ( get_fallback_generalization_rules, @@ -3337,14 +3338,7 @@ def test_anthropic_us_data_residency_uplift_on_claude_4_6_and_later_models(_loca def test_gemini_cache_tokens_details_no_negative_values(): - """ - Test for Issue #18750: Negative text_tokens with Gemini caching - - When using Gemini with explicit caching, the response includes cacheTokensDetails - which breaks down cached tokens by modality. This test ensures that: - 1. text_tokens is never negative - 2. We correctly subtract cached tokens per modality (not total) - """ + """Ensure issue #18750 cache details remain non-negative after cost parsing.""" from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, ) @@ -3372,27 +3366,17 @@ def test_gemini_cache_tokens_details_no_negative_values(): usage = VertexGeminiConfig.calculate_usage(completion_response) - # Text tokens should be non-cached text only: 9402 - 9393 = 9 - assert usage.prompt_tokens_details.text_tokens == 9, ( - f"Expected text_tokens=9, got {usage.prompt_tokens_details.text_tokens}" - ) + prompt_details: Final = parse_prompt_tokens_details(usage) - # Image tokens should be non-cached image only: 258 - 258 = 0 - assert usage.prompt_tokens_details.image_tokens == 0, ( - f"Expected image_tokens=0, got {usage.prompt_tokens_details.image_tokens}" + assert usage.prompt_tokens_details.text_tokens == 9402 + assert usage.prompt_tokens_details.image_tokens == 258 + assert usage.prompt_tokens_details.cached_tokens == 9651 + assert usage.prompt_tokens_details.cached_tokens_details == CachedTokensDetails( + text_tokens=9393, image_tokens=258 ) - - # Total cached should match - assert usage.prompt_tokens_details.cached_tokens == 9651, ( - f"Expected cached_tokens=9651, got {usage.prompt_tokens_details.cached_tokens}" - ) - - # MOST IMPORTANT: text_tokens should NEVER be negative - assert usage.prompt_tokens_details.text_tokens >= 0, ( - f"BUG: text_tokens is negative ({usage.prompt_tokens_details.text_tokens})! This was the issue in #18750" - ) - - print("✅ Issue #18750 fix verified: text_tokens is correctly calculated and non-negative") + assert prompt_details["text_tokens"] == 9 + assert prompt_details["image_tokens"] == 0 + assert prompt_details["text_tokens"] >= 0 def test_gemini_without_cache_tokens_details(): @@ -5990,6 +5974,45 @@ def test_batch_cost_calculator_prices_modalities_and_cached_tokens_together_in_t assert prompt_cost == pytest.approx(298_926 * 3e-6 + 64 * 5e-6 + 10 * 3e-6 + 1_000 * 3e-7) +def test_batch_cost_calculator_prices_cached_audio_at_its_batch_rate() -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 2e-6, + "input_cost_per_token_batches": 1e-6, + "input_cost_per_audio_token": 6e-6, + "input_cost_per_audio_token_batches": 3e-6, + "cache_read_input_token_cost": 8e-7, + "cache_read_input_token_cost_batches": 4e-7, + "cache_read_input_audio_token_cost": 5e-7, + "cache_read_input_audio_token_cost_batches": 2e-7, + } + usage: Final = Usage( + prompt_tokens=40, + completion_tokens=0, + total_tokens=40, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=20, + audio_tokens=20, + cached_tokens=10, + cached_tokens_details=CachedTokensDetails(text_tokens=5, audio_tokens=5), + ), + ) + + prompt_cost, _ = batch_cost_calculator( + usage=usage, + model="gemini-audio-batch-test-model", + custom_llm_provider="gemini", + model_info=model_info, + ) + + expected: Final = ( + 15 * model_info["input_cost_per_token_batches"] + + 15 * model_info["input_cost_per_audio_token_batches"] + + 5 * model_info["cache_read_input_token_cost_batches"] + + 5 * model_info["cache_read_input_audio_token_cost_batches"] + ) + assert prompt_cost == pytest.approx(expected) + + QWEN3_NEXT_REGIONS: Final = ("ap-northeast-1", "ap-south-1", "ap-southeast-2", "eu-west-1", "eu-west-2", "sa-east-1") diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index fcd6b8aa402..338faae188f 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -994,6 +994,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_creation_input_token_cost_above_1hr_above_100k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"}, "cache_read_input_audio_token_cost": {"type": "number"}, + "cache_read_input_audio_token_cost_batches": {"type": "number"}, + "cache_read_input_audio_token_cost_flex": {"type": "number"}, + "cache_read_input_audio_token_cost_priority": {"type": "number"}, "cache_read_input_image_token_cost": {"type": "number"}, "audio_transcription_config": {"type": "string"}, "deprecation_date": {"type": "string"}, @@ -1001,6 +1004,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"}, "google_maps_grounding_cost_per_query": {"type": "number"}, "input_cost_per_audio_token": {"type": "number"}, + "input_cost_per_audio_token_flex": {"type": "number"}, "input_cost_per_image_token": {"type": "number"}, "input_cost_per_character": {"type": "number"}, "input_cost_per_character_above_128k_tokens": {"type": "number"}, diff --git a/tests/unit/test_utils_get_model_info.py b/tests/unit/test_utils_get_model_info.py index bfaac44a483..4bd16a2997f 100644 --- a/tests/unit/test_utils_get_model_info.py +++ b/tests/unit/test_utils_get_model_info.py @@ -61,6 +61,30 @@ def test_get_model_info_custom_llm_with_same_name_vllm(monkeypatch): assert model_info["input_cost_per_token"] == 0.0 +def test_get_model_info_passes_through_gemini_tiered_audio_rates() -> None: + model: Final = "gemini-tiered-audio-rates-test" + prices: Final = { + "input_cost_per_audio_token_flex": 1e-6, + "input_cost_per_audio_token_priority": 2e-6, + "cache_read_input_audio_token_cost_flex": 3e-7, + "cache_read_input_audio_token_cost_priority": 4e-7, + "cache_read_input_audio_token_cost_batches": 5e-7, + } + litellm.register_model( + { + model: { + "litellm_provider": "gemini", + "mode": "chat", + **prices, + } + } + ) + + model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="gemini") + + assert {key: model_info[key] for key in prices} == prices + + def test_get_model_info_ollama_chat(): from litellm.llms.ollama.completion.transformation import OllamaConfig diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index da9e77e9b58..b5a86dd2187 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35259,6 +35259,12 @@ export interface components { cache_creation_input_token_cost_ultrafast?: number | null; /** Cache Read Input Audio Token Cost */ cache_read_input_audio_token_cost?: number | null; + /** Cache Read Input Audio Token Cost Batches */ + cache_read_input_audio_token_cost_batches?: number | null; + /** Cache Read Input Audio Token Cost Flex */ + cache_read_input_audio_token_cost_flex?: number | null; + /** Cache Read Input Audio Token Cost Priority */ + cache_read_input_audio_token_cost_priority?: number | null; /** Cache Read Input Image Token Cost */ cache_read_input_image_token_cost?: number | null; /** Cache Read Input Token Cost */ @@ -35338,6 +35344,10 @@ export interface components { input_cost_per_audio_token?: number | null; /** Input Cost Per Audio Token Batches */ input_cost_per_audio_token_batches?: number | null; + /** Input Cost Per Audio Token Flex */ + input_cost_per_audio_token_flex?: number | null; + /** Input Cost Per Audio Token Priority */ + input_cost_per_audio_token_priority?: number | null; /** Input Cost Per Character */ input_cost_per_character?: number | null; /** Input Cost Per Character Above 128K Tokens */ @@ -50568,6 +50578,12 @@ export interface components { cache_creation_input_token_cost_ultrafast?: number | null; /** Cache Read Input Audio Token Cost */ cache_read_input_audio_token_cost?: number | null; + /** Cache Read Input Audio Token Cost Batches */ + cache_read_input_audio_token_cost_batches?: number | null; + /** Cache Read Input Audio Token Cost Flex */ + cache_read_input_audio_token_cost_flex?: number | null; + /** Cache Read Input Audio Token Cost Priority */ + cache_read_input_audio_token_cost_priority?: number | null; /** Cache Read Input Image Token Cost */ cache_read_input_image_token_cost?: number | null; /** Cache Read Input Token Cost */ @@ -50647,6 +50663,10 @@ export interface components { input_cost_per_audio_token?: number | null; /** Input Cost Per Audio Token Batches */ input_cost_per_audio_token_batches?: number | null; + /** Input Cost Per Audio Token Flex */ + input_cost_per_audio_token_flex?: number | null; + /** Input Cost Per Audio Token Priority */ + input_cost_per_audio_token_priority?: number | null; /** Input Cost Per Character */ input_cost_per_character?: number | null; /** Input Cost Per Character Above 128K Tokens */ From 082064569082761bbb8ce1dc1b1be2fb2a145b94 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Sat, 24 Oct 2026 10:17:14 -0700 Subject: [PATCH 007/155] fix(mcp): retain toolset scope when refreshing catalog authority (#48095) --- .../mcp_server/auth/user_api_key_auth_mcp.py | 9 +- .../proxy/_experimental/mcp_server/server.py | 2 +- .../auth/test_user_api_key_auth_mcp.py | 19 +++- .../_experimental/mcp_server/test_catalog.py | 101 +++++++++++++++++- 4 files changed, 125 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 53b56c7b31d..7f0d392ab1b 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -1368,7 +1368,7 @@ class MCPRequestHandler: if auth is None: return None - refreshed = auth.model_copy() + refreshed: Final = auth.model_copy(update={"requires_fresh_policy": True}) if (auth.mcp_admitted_user_subject or auth.is_session_token) and auth.user_id: # A session credential (``lite login`` or a redeemed ID-JAG) has no key row, its authority is # sealed in the token: refresh the live user policy and keep the token's team, routes and models. @@ -1386,7 +1386,12 @@ class MCPRequestHandler: refreshed.project_id = current.project_id refreshed.user_id = current.user_id refreshed.access_group_ids = current.access_group_ids - refreshed.requires_fresh_policy = True + else: + return refreshed + if refreshed.mcp_toolset_id is not None: + from litellm.proxy._experimental.mcp_server.server import apply_toolset_scope + + return await apply_toolset_scope(refreshed, refreshed.mcp_toolset_id) return refreshed @staticmethod diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index b3426089375..bb7c81f9d60 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -1797,7 +1797,7 @@ if MCP_AVAILABLE: return acting.model_copy(update={"mcp_toolset_id": toolset_id}) tool_permissions = await operations.global_mcp_server_manager.resolve_toolset_tool_permissions( - toolset_ids=[toolset_id] + toolset_ids=[toolset_id], requires_fresh_policy=acting.requires_fresh_policy ) server_ids: Final = list(tool_permissions.keys()) existing_op: Final = acting.object_permission diff --git a/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index f73a8547158..e8689b2ba89 100644 --- a/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -10906,7 +10906,19 @@ async def test_catalog_refresh_reads_current_user_org_without_losing_resource_sc @pytest.mark.asyncio @pytest.mark.parametrize("current_groups", [[], ["replacement-group"]]) async def test_catalog_refresh_uses_current_virtual_key_policy_and_keeps_session_scope(monkeypatch, current_groups): - permission = LiteLLM_ObjectPermissionTable(object_permission_id="current-policy", mcp_servers=["current-server"]) + from litellm.proxy import proxy_server + from litellm.types.mcp_server.mcp_toolset import MCPToolset + + permission = LiteLLM_ObjectPermissionTable( + object_permission_id="current-policy", mcp_toolsets=["session-toolset"] + ) + toolset: Final = MCPToolset( + toolset_id="session-toolset", toolset_name="session-toolset", + tools=[{"server_id": "current-server", "tool_name": "read"}], + ) + tables: Final = SimpleNamespace(litellm_mcptoolsettable=SimpleNamespace(find_many=AsyncMock(return_value=[toolset]))) + monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=tables, writer_db=tables)) + monkeypatch.setattr(proxy_server, "user_api_key_cache", DualCache()) current = UserAPIKeyAuth( object_permission=permission, object_permission_id="current-policy", @@ -10931,7 +10943,10 @@ async def test_catalog_refresh_uses_current_virtual_key_policy_and_keeps_session caller.mcp_toolset_id = "session-toolset" refreshed = await MCPRequestHandler.refresh_catalog_authority(caller) reload_key.assert_awaited_once_with("owned-key-hash", check_db_only=True) - assert refreshed.object_permission == permission + assert refreshed.object_permission == permission.model_copy(update={ + "mcp_servers": ["current-server"], "mcp_toolsets": [], + "mcp_tool_permissions": {"current-server": ["read"]}, + }) assert refreshed.object_permission_id == "current-policy" assert (refreshed.team_id, refreshed.org_id, refreshed.project_id, refreshed.user_id) == ( "new-team", diff --git a/tests/unit/proxy/_experimental/mcp_server/test_catalog.py b/tests/unit/proxy/_experimental/mcp_server/test_catalog.py index b383078c0ed..4f6530e5de1 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_catalog.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_catalog.py @@ -1,5 +1,5 @@ import asyncio -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from types import SimpleNamespace from typing import Final, Literal from unittest.mock import AsyncMock @@ -13,6 +13,7 @@ from mcp.types import ( ListResourcesResult, ListResourceTemplatesRequest, ListResourceTemplatesResult, + ListToolsRequest, ListToolsResult, PaginatedRequestParams, Tool, @@ -34,6 +35,104 @@ CatalogKind = Literal["tools", "prompts", "resources", "templates"] OptionalCatalogResult = ListPromptsResult | ListResourcesResult | ListResourceTemplatesResult +@pytest.mark.asyncio +@pytest.mark.parametrize( + "selected,revoked,expected", + [ + ("narrow", False, ("alpha-read_item",)), + ("full", False, ("alpha-read_item", "alpha-write_item", "beta-read_item", "beta-write_item")), + (None, False, ("alpha-read_item", "alpha-write_item", "beta-read_item", "beta-write_item")), + ("narrow", True, ()), + ], +) +async def test_catalog_refresh_keeps_selected_toolset_with_current_grants_and_contents( + monkeypatch: pytest.MonkeyPatch, + selected: str | None, + revoked: bool, + expected: tuple[str, ...], +) -> None: + from fastapi import HTTPException + + from litellm.caching.dual_cache import DualCache + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import operations + from litellm.proxy._experimental.mcp_server.server import apply_toolset_scope + from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy._types import LiteLLM_ObjectPermissionTable + from litellm.types.mcp_server.mcp_toolset import MCPToolset, MCPToolsetTool + + grants: Final = LiteLLM_ObjectPermissionTable( + object_permission_id="catalog-grants", mcp_toolsets=["narrow", "full"] + ) + live_grants: Final = grants.model_copy(update={"mcp_toolsets": ["full"]}) if revoked else grants + tools: Final[tuple[MCPToolsetTool, ...]] = ( + {"server_id": "alpha", "tool_name": "read_item"}, + {"server_id": "alpha", "tool_name": "write_item"}, + {"server_id": "beta", "tool_name": "read_item"}, + {"server_id": "beta", "tool_name": "write_item"}, + ) + toolsets: Final = ( + MCPToolset(toolset_id="narrow", toolset_name="narrow", tools=[tools[0]]), + MCPToolset(toolset_id="full", toolset_name="full", tools=list(tools)), + ) + + async def read_toolsets(where: Mapping[str, Mapping[str, Sequence[str]]]) -> tuple[MCPToolset, ...]: + return tuple(toolset for toolset in toolsets if toolset.toolset_id in where["toolset_id"]["in"]) + + tables: Final = SimpleNamespace( + litellm_objectpermissiontable=SimpleNamespace(find_unique=AsyncMock(return_value=live_grants)), + litellm_mcptoolsettable=SimpleNamespace(find_many=read_toolsets), + ) + database: Final = SimpleNamespace( + db=tables, + writer_db=tables, + get_data=AsyncMock(return_value=UserAPIKeyAuth(object_permission_id=grants.object_permission_id)), + ) + cache: Final = DualCache() + monkeypatch.setattr(proxy_server, "prisma_client", database) + monkeypatch.setattr(proxy_server, "user_api_key_cache", cache) + monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": []}) + await cache.async_set_cache(key="toolset_perms:narrow", value={"alpha": ["read_item", "write_item"]}) + manager: Final = operations.global_mcp_server_manager + for server_id in ("alpha", "beta"): + monkeypatch.setitem( + manager.registry, + server_id, + MCPServer(server_id=server_id, name=server_id, transport=MCPTransport.http, spec_path="/catalog.yaml"), + ) + + async def tool_handler() -> str: + return "ok" + + for tool in tools: + global_mcp_tool_registry.register_tool( + name=f"{tool['server_id']}-{tool['tool_name']}", + description=tool["tool_name"], + input_schema={"type": "object"}, + handler=tool_handler, + ) + original: Final = UserAPIKeyAuth( + api_key="catalog-key-hash", object_permission=grants, + object_permission_id=grants.object_permission_id, + ) + original.via_virtual_key = True + caller: Final = await apply_toolset_scope(original, selected) if selected is not None else original + before: Final = caller.model_dump() + context: Final = operations.prepare_context(user_api_key_auth=caller) + if revoked: + with pytest.raises(HTTPException, match="does not have access to toolset") as denied: + await catalog.list_gateway_catalog(context, ListToolsRequest(), log_list_tools_to_spendlogs=False) + assert denied.value.status_code == 403 + else: + result: Final = await catalog.list_gateway_catalog( + context, ListToolsRequest(), log_list_tools_to_spendlogs=False + ) + assert isinstance(result, ListToolsResult) + assert tuple(sorted(tool.name for tool in result.tools)) == expected + assert caller.model_dump() == before + assert original.object_permission == grants + + def page(name: str, cursor: str | None = None, revision: str = "stable") -> ListToolsResult: return ListToolsResult( tools=[Tool(name=name, input_schema={"type": "object"})], From 464932b1d19d7ed8d377ed79fc2a9c0110dff39b Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:31:09 -0700 Subject: [PATCH 008/155] refactor(typing): clear 1181 basedpyright Any errors across 431 files (#47789) * refactor(typing): narrow Any sources across providers, proxy and guardrails (wave 1) * refactor(typing): narrow Any sources across hubs, providers and proxy (wave 2) * refactor(typing): validate RunwayML task JSON and let dotprompt loaders accept exported prompts * fix: restore pass-through behavior on paths the typing pass tightened MCP client IP extraction trusts X-Forwarded-For only when use_x_forwarded_for is True, dotprompt skips a malformed JSON prompt entry instead of raising, RunwayML task payloads pass through unvalidated, and a malformed provider body raises the same AttributeError as before at the CompactifAI, Gemini, and mock transport parse sites * fix(runwayml): keep a malformed 2xx task body on the 500 path * fix(edenai,cli): keep a non-object body in the error and fail on a malformed migration report EdenAI's embedding and image generation parse dropped a non-object body from the error they raised, so the message no longer named what the provider sent. A helper now raises with the body in the message, and both paths have a regression test. `lite encryption migrate` exited 0 with no message when the proxy's report was not an object or its residual_legacy was not a number, where base crashed. It now fails with a message naming the value. The CompactifAI and Gemini image-edit non-object body tests accept TypeError or AttributeError, which both base and this branch raise, so they guard against a ValidationError instead of pinning one class. RunwayML's async parse gets the same non-object body test as the sync one. * fix: keep base's error classes, duck-typed checks and fail-closed guardrails the typing pass changed * test: annotate the new regression tests' locals with Final * test: annotate the remaining new test locals with Final --- enterprise/enterprise_hooks/aporia_ai.py | 13 +- .../enterprise_hooks/blocked_user_list.py | 17 +-- .../google_text_moderation.py | 13 +- .../example_logging_api.py | 7 +- .../enterprise_callbacks/llm_guard.py | 18 ++- .../proxy/common_utils/check_batch_cost.py | 4 +- .../proxy/hooks/managed_files.py | 27 ++-- .../proxy/hooks/managed_vector_stores.py | 6 +- .../management_endpoints/project_endpoints.py | 13 +- .../proxy/vector_stores/endpoints.py | 8 +- litellm/__init__.py | 16 ++- litellm/_lazy_imports.py | 4 +- litellm/a2a_protocol/cost_calculator.py | 20 ++- .../bedrock_agentcore/transformation.py | 27 ++-- .../watsonx_orchestrate/transformation.py | 40 ++++-- litellm/a2a_protocol/utils.py | 21 ++-- litellm/anthropic_beta_headers_manager.py | 13 +- .../exceptions/exception_mapping_utils.py | 11 +- litellm/assistants/main.py | 24 ++-- litellm/caching/_embedding_router.py | 6 +- litellm/caching/azure_blob_cache.py | 8 +- litellm/caching/base_cache.py | 2 +- litellm/caching/disk_cache.py | 8 +- litellm/cost_calculator.py | 44 ++++++- litellm/exceptions.py | 2 +- litellm/experimental_mcp_client/tools.py | 20 +-- litellm/files/main.py | 14 ++- litellm/files/streaming.py | 4 +- litellm/harness/handlers/base.py | 3 +- litellm/harness/handlers/cli_handler.py | 4 +- .../harness/handlers/deepagents_handler.py | 2 +- litellm/harness/runtime.py | 11 +- litellm/images/main.py | 11 +- .../SlackAlerting/hanging_request_check.py | 14 ++- .../SlackAlerting/user_spend_alerts.py | 5 +- litellm/integrations/agentops/agentops.py | 23 ++-- litellm/integrations/arize/arize.py | 2 +- litellm/integrations/braintrust_logging.py | 29 +++-- .../integrations/cloudzero/cz_stream_api.py | 7 +- litellm/integrations/custom_guardrail.py | 7 +- litellm/integrations/custom_secret_manager.py | 3 +- litellm/integrations/dotprompt/__init__.py | 4 +- .../dotprompt/dotprompt_manager.py | 15 ++- .../integrations/dotprompt/prompt_manager.py | 29 +++-- .../focus/destinations/factory.py | 5 +- litellm/integrations/focus/export_engine.py | 8 +- litellm/integrations/focus/focus_logger.py | 8 +- litellm/integrations/gcs_bucket/gcs_bucket.py | 8 +- .../gcs_bucket/gcs_bucket_mock_client.py | 3 +- litellm/integrations/gcs_pubsub/pub_sub.py | 8 +- litellm/integrations/langfuse/langfuse.py | 5 +- .../integrations/langfuse/langfuse_otel.py | 5 +- .../langfuse/langfuse_prompt_management.py | 12 +- litellm/integrations/langsmith.py | 11 +- litellm/integrations/langtrace.py | 2 +- litellm/integrations/levo/levo.py | 2 +- litellm/integrations/logfire_logger.py | 5 +- litellm/integrations/newrelic/newrelic.py | 19 +-- .../integrations/newrelic/newrelic_metrics.py | 2 +- .../opik_payload_builder/payload_builders.py | 4 +- litellm/integrations/otel/model/config.py | 3 +- litellm/integrations/otel/model/metadata.py | 6 +- litellm/integrations/otel/model/payloads.py | 15 ++- litellm/integrations/otel/plumbing/metrics.py | 8 +- litellm/integrations/sqs.py | 5 +- .../handler.py | 5 +- litellm/litellm_core_utils/app_crypto.py | 9 +- .../cloud_storage_security.py | 6 +- litellm/litellm_core_utils/core_helpers.py | 47 +++++-- litellm/litellm_core_utils/cost_discount.py | 2 +- .../litellm_core_utils/default_encoding.py | 2 +- .../dot_notation_indexing.py | 2 +- .../exception_mapping_utils.py | 4 +- .../litellm_core_utils/get_model_cost_map.py | 6 +- litellm/litellm_core_utils/hidden_params.py | 3 +- .../initialize_dynamic_callback_params.py | 2 +- litellm/litellm_core_utils/litellm_logging.py | 63 ++++++---- .../llm_cost_calc/tool_call_cost_tracking.py | 2 +- litellm/litellm_core_utils/llm_judge.py | 5 +- .../logging_callback_manager.py | 8 +- .../prompt_templates/common_utils.py | 20 +-- .../prompt_templates/factory.py | 32 +++-- .../mid_conversation_system.py | 2 +- .../sensitive_data_masker.py | 8 +- .../streaming_chunk_builder_utils.py | 3 +- .../litellm_core_utils/streaming_handler.py | 86 ++++++++----- litellm/litellm_core_utils/token_counter.py | 4 +- litellm/litellm_core_utils/url_utils.py | 59 +++++---- litellm/llms/__init__.py | 3 +- litellm/llms/a2a/common_utils.py | 6 +- litellm/llms/ai21/chat/transformation.py | 4 +- .../llms/amazon_nova/chat/transformation.py | 4 +- litellm/llms/anthropic/batches/handler.py | 2 +- .../chat/guardrail_translation/handler.py | 9 +- litellm/llms/anthropic/chat/transformation.py | 7 +- .../anthropic/completion/transformation.py | 3 +- litellm/llms/anthropic/files/handler.py | 5 +- .../adapters/streaming_iterator.py | 8 +- .../editors/clear_tool_uses.py | 15 ++- .../context_management/placeholders.py | 4 +- .../pass_through/context_management/result.py | 6 +- .../pass_through/convertible_blocks.py | 17 ++- .../messages/fake_stream_iterator.py | 12 +- .../pass_through/messages/transformation.py | 2 +- .../responses_adapters/transformation.py | 4 +- litellm/llms/azure/audio_transcriptions.py | 4 +- litellm/llms/azure/azure.py | 10 +- litellm/llms/azure/exception_mapping.py | 8 +- .../llms/azure/responses/transformation.py | 2 +- litellm/llms/azure_ai/chat/transformation.py | 2 +- litellm/llms/base_llm/chat/transformation.py | 5 +- .../guardrail_translation/base_translation.py | 2 +- .../base_llm/guardrail_translation/utils.py | 4 +- .../base_llm/image_edit/transformation.py | 5 +- .../base_llm/interactions/transformation.py | 6 +- .../base_managed_resource.py | 4 +- .../base_llm/passthrough/transformation.py | 3 +- .../llms/base_llm/rerank/transformation.py | 2 +- .../base_llm/text_to_speech/transformation.py | 6 +- .../base_llm/vector_store/transformation.py | 4 +- litellm/llms/baseten/chat.py | 3 +- litellm/llms/bedrock/batches/handler.py | 3 +- .../bedrock/chat/agentcore/transformation.py | 15 ++- .../bedrock/chat/converse_transformation.py | 26 +++- .../amazon_ai21_transformation.py | 3 +- .../amazon_cohere_transformation.py | 3 +- .../amazon_llama_transformation.py | 3 +- .../amazon_mistral_transformation.py | 3 +- .../amazon_titan_transformation.py | 3 +- .../anthropic_claude2_transformation.py | 3 +- .../bedrock/chat/mantle/transformation.py | 5 +- litellm/llms/bedrock/common_utils.py | 7 +- .../embed/amazon_titan_g1_transformation.py | 3 +- .../embed/amazon_titan_v2_transformation.py | 3 +- litellm/llms/bedrock/files/handler.py | 4 +- litellm/llms/bedrock/image_edit/handler.py | 5 +- .../amazon_stability3_transformation.py | 4 +- .../amazon_titan_transformation.py | 6 +- .../anthropic_claude3_transformation.py | 4 +- .../bedrock/messages/mantle_transformation.py | 4 +- litellm/llms/bedrock/model_listing.py | 5 +- .../guardrail_translation/handler.py | 6 +- .../image_edit/transformation.py | 3 +- .../image_generation/transformation.py | 3 +- litellm/llms/cerebras/chat.py | 3 +- litellm/llms/chatgpt/chat/streaming_utils.py | 7 +- litellm/llms/clarifai/chat/transformation.py | 3 +- .../claude_code/harness/transformation.py | 5 +- litellm/llms/codestral/completion/handler.py | 4 +- litellm/llms/codex/harness/transformation.py | 7 +- litellm/llms/cohere/chat/transformation.py | 5 +- litellm/llms/cohere/rerank/transformation.py | 3 +- .../llms/cohere/rerank_v2/transformation.py | 4 +- litellm/llms/cometapi/chat/transformation.py | 4 +- .../llms/compactifai/chat/transformation.py | 44 +++++-- .../llms/custom_httpx/async_client_cleanup.py | 2 +- litellm/llms/custom_httpx/llm_http_handler.py | 50 +++++--- litellm/llms/custom_httpx/mock_transport.py | 13 +- litellm/llms/dashscope/cost_calculator.py | 13 +- .../image_generation/transformation.py | 3 +- .../llms/databricks/embed/transformation.py | 6 +- .../databricks/messages/transformation.py | 4 +- .../deepagents/harness/sandbox_backend.py | 4 +- .../llms/deepagents/harness/transformation.py | 7 +- .../audio_transcription/transformation.py | 41 +++++- .../llms/deepinfra/rerank/transformation.py | 5 +- .../llms/deepseek/messages/transformation.py | 36 ++++-- litellm/llms/deprecated_providers/palm.py | 6 +- .../llms/edenai/embedding/transformation.py | 18 ++- .../edenai/image_generation/transformation.py | 18 ++- .../audio_transcription/transformation.py | 35 +++--- .../flux_pro_v11_ultra_transformation.py | 3 +- .../recraft_v3_transformation.py | 3 +- .../stable_diffusion_transformation.py | 3 +- .../featherless_ai/chat/transformation.py | 4 +- .../fireworks_ai/rerank/transformation.py | 3 +- litellm/llms/gdc/chat/transformation.py | 8 +- litellm/llms/gemini/agents/transformation.py | 52 +++++--- .../audio_transcription/transformation.py | 4 +- litellm/llms/gemini/common_utils.py | 12 +- .../guardrail_translation/handler.py | 10 +- .../gemini/google_genai/transformation.py | 16 ++- .../llms/gemini/image_edit/transformation.py | 28 +++-- .../gemini/image_generation/transformation.py | 7 +- .../llms/gemini/realtime/transformation.py | 12 +- litellm/llms/gemini/videos/transformation.py | 5 +- litellm/llms/gigachat/chat/transformation.py | 6 +- litellm/llms/gigachat/file_handler.py | 18 +-- .../gigachat/passthrough/transformation.py | 8 +- .../github_copilot/chat/transformation.py | 11 +- .../github_copilot/messages/transformation.py | 5 +- .../responses/transformation.py | 28 +++-- .../llms/gradient_ai/chat/transformation.py | 4 +- litellm/llms/groq/chat/transformation.py | 3 +- litellm/llms/groq/cost_calculator.py | 5 +- litellm/llms/groq/stt/transformation.py | 6 +- .../llms/hosted_vllm/rerank/transformation.py | 9 +- .../llms/hosted_vllm/videos/transformation.py | 4 +- litellm/llms/huggingface/embedding/handler.py | 18 +-- .../huggingface/embedding/transformation.py | 7 +- .../llms/huggingface/rerank/transformation.py | 2 +- litellm/llms/langflow/chat/transformation.py | 11 +- litellm/llms/langgraph/chat/sse_iterator.py | 5 +- litellm/llms/langgraph/chat/transformation.py | 10 +- litellm/llms/lemonade/chat/transformation.py | 7 +- .../litellm_proxy/skills/prompt_injection.py | 34 ++--- .../litellm_proxy/skills/sandbox_executor.py | 12 +- .../llms/lm_studio/embed/transformation.py | 6 +- litellm/llms/maritalk.py | 3 +- .../milvus/vector_stores/transformation.py | 14 ++- .../llms/minimax/messages/transformation.py | 4 +- .../image_generation/transformation.py | 3 +- litellm/llms/nadir/chat/transformation.py | 9 +- litellm/llms/nvidia_nim/embed.py | 6 +- .../audio_transcription/handler.py | 31 +++-- .../audio_transcription/transformation.py | 17 ++- litellm/llms/oci/chat/transformation.py | 3 +- litellm/llms/oci/embed/transformation.py | 3 +- litellm/llms/ollama/common_utils.py | 3 +- .../llms/ollama/completion/transformation.py | 2 +- litellm/llms/openai/common_utils.py | 17 ++- litellm/llms/openai/evals/transformation.py | 8 +- .../responses/count_tokens/token_counter.py | 4 +- .../responses/count_tokens/transformation.py | 42 +++++-- .../guardrail_translation/handler.py | 11 +- .../openai_like/messages/transformation.py | 6 +- .../llms/opencode/harness/transformation.py | 26 +++- .../openrouter/image_edit/transformation.py | 6 +- .../image_generation/transformation.py | 3 +- .../opensandbox/sandbox/transformation.py | 12 +- .../guardrail_translation/handler.py | 48 +++++-- .../perplexity/responses/transformation.py | 5 +- .../llms/petals/completion/transformation.py | 4 +- .../pg_vector/vector_stores/transformation.py | 2 +- .../image_generation/transformation.py | 3 +- .../image_generation/transformation.py | 40 ++++-- .../runwayml/text_to_speech/transformation.py | 17 ++- .../llms/runwayml/videos/transformation.py | 2 +- litellm/llms/sagemaker/common_utils.py | 11 +- .../embedding/cohere_transformation.py | 6 +- .../embedding/openai_transformation.py | 5 +- .../sagemaker/embedding/transformation.py | 6 +- litellm/llms/sap/chat/transformation.py | 8 +- litellm/llms/sap/credentials.py | 33 +++-- litellm/llms/sap/embed/transformation.py | 8 +- .../stability/image_edit/transformations.py | 11 +- .../llms/tencent/messages/transformation.py | 4 +- .../llms/triton/completion/transformation.py | 7 +- .../llms/triton/embedding/transformation.py | 16 ++- .../vercel_ai_gateway/chat/transformation.py | 14 ++- .../gemini_transcribe_transformation.py | 4 +- .../llms/vertex_ai/batches/transformation.py | 2 +- litellm/llms/vertex_ai/files/handler.py | 4 +- .../llms/vertex_ai/files/transformation.py | 15 ++- litellm/llms/vertex_ai/fine_tuning/handler.py | 13 +- litellm/llms/vertex_ai/gemini/long_decode.py | 3 +- .../batch_embed_content_handler.py | 10 +- .../vertex_gemini_transformation.py | 2 +- .../vertex_imagen_transformation.py | 7 +- .../image_generation_handler.py | 4 +- .../vertex_gemini_transformation.py | 3 +- .../multimodal_embeddings/transformation.py | 2 +- .../vertex_ai/rag_engine/transformation.py | 18 ++- .../llms/vertex_ai/rerank/transformation.py | 16 ++- .../ai21/transformation.py | 6 +- .../transformation.py | 5 +- .../vertex_embeddings/transformation.py | 6 +- .../llms/volcengine/chat/transformation.py | 3 +- .../volcengine/responses/transformation.py | 11 +- litellm/llms/voyage/rerank/transformation.py | 3 +- .../audio_transcription/transformation.py | 3 +- litellm/llms/watsonx/rerank/transformation.py | 3 +- litellm/llms/xai/chat/transformation.py | 13 +- litellm/llms/xai/cost_calculator.py | 9 +- litellm/main.py | 21 ++-- litellm/models/organization_membership.py | 10 +- .../mcp_server/auth/user_api_key_auth_mcp.py | 3 +- .../proxy/_experimental/mcp_server/catalog.py | 7 +- litellm/proxy/_experimental/mcp_server/db.py | 18 +-- .../mcp_server/discoverable_endpoints.py | 2 +- .../mcp_server/faults/classify.py | 5 +- .../mcp_server/faults/list_outcomes.py | 2 +- .../mcp_server/mcp_server_manager.py | 8 +- .../mcp_server/oauth2_token_cache.py | 5 +- .../mcp_server/oauth_identity_binding.py | 6 +- .../token_exchange_provider.py | 4 +- .../mcp_server/result_conversion.py | 13 +- .../mcp_server/semantic_tool_filter.py | 14 ++- .../proxy/_experimental/mcp_server/server.py | 20 ++- .../_experimental/mcp_server/toolset_db.py | 6 +- .../proxy/_experimental/mcp_server/utils.py | 5 +- litellm/proxy/_lazy_features.py | 20 +-- litellm/proxy/_lazy_openapi_snapshot.py | 45 ++++--- litellm/proxy/a2a/discovery.py | 14 ++- .../proxy/agent_endpoints/a2a_endpoints.py | 15 ++- litellm/proxy/agent_endpoints/a2a_routing.py | 6 +- .../proxy/agent_endpoints/databricks_oauth.py | 3 +- .../proxy/agent_endpoints/managed_identity.py | 6 +- .../claude_code_marketplace.py | 3 +- litellm/proxy/auth/auth_checks.py | 4 +- litellm/proxy/auth/auth_utils.py | 16 ++- litellm/proxy/auth/ip_address_utils.py | 4 +- litellm/proxy/auth/rds_iam_token.py | 8 +- litellm/proxy/auth/route_checks.py | 2 +- litellm/proxy/auth/user_api_key_auth.py | 8 +- litellm/proxy/caching_routes.py | 7 +- litellm/proxy/client/chat.py | 5 +- .../proxy/client/cli/commands/encryption.py | 30 +++-- .../client/cli/commands/statusline_script.py | 26 +++- litellm/proxy/client/cli/interface.py | 2 +- litellm/proxy/client/cli/main.py | 6 +- litellm/proxy/client/credentials.py | 6 +- litellm/proxy/client/http_client.py | 7 +- litellm/proxy/client/model_groups.py | 4 +- litellm/proxy/client/models.py | 2 +- .../auth_cache_invalidation_pubsub.py | 15 ++- litellm/proxy/common_utils/callback_utils.py | 18 ++- litellm/proxy/common_utils/debug_utils.py | 7 +- .../proxy/common_utils/http_parsing_utils.py | 2 +- .../proxy/common_utils/load_config_utils.py | 3 +- .../common_utils/proxy_rate_limit_error.py | 4 +- .../container_endpoints/handler_factory.py | 5 +- .../proxy/container_endpoints/ownership.py | 25 ++-- litellm/proxy/db/autorouter_session_rollup.py | 2 +- litellm/proxy/db/db_spend_update_writer.py | 69 +++++++++-- .../db_transaction_queue/pod_lock_manager.py | 6 +- .../redis_update_buffer.py | 4 +- litellm/proxy/db/prisma_query_span.py | 5 +- litellm/proxy/db/proxy_worker_heartbeat.py | 5 +- litellm/proxy/db/routing_prisma_wrapper.py | 2 +- litellm/proxy/db/spend_log_tool_index.py | 4 +- .../agent_skills_archive.py | 4 +- .../agent_skills_endpoints.py | 7 +- .../proxy/example_config_yaml/custom_auth.py | 2 +- .../example_config_yaml/custom_callbacks.py | 12 +- litellm/proxy/guardrails/_content_utils.py | 15 +-- .../proxy/guardrails/guardrail_endpoints.py | 3 +- .../guardrail_hooks/agent_365/agent_365.py | 5 +- .../guardrails/guardrail_hooks/airia/airia.py | 13 +- .../block_code_execution/__init__.py | 3 +- .../cato_networks/cato_networks.py | 7 +- .../cisco_ai_defense/cisco_ai_defense.py | 6 +- .../guardrail_hooks/compresr/__init__.py | 3 +- .../guardrail_hooks/compresr/compresr.py | 4 +- .../crowdstrike_aidr/crowdstrike_aidr.py | 6 +- .../guardrail_hooks/custom_code/__init__.py | 2 +- .../generic_guardrail_api.py | 12 +- .../guardrail_hooks/grayswan/grayswan.py | 5 +- .../guardrail_hooks/javelin/javelin.py | 7 +- .../guardrail_hooks/lakera_ai_v2.py | 10 +- .../guardrails/guardrail_hooks/lasso/lasso.py | 43 ++++--- .../competitor_intent/airline.py | 8 +- .../litellm_content_filter/content_filter.py | 8 +- .../llm_shield_proxy/llm_shield_proxy.py | 53 ++++---- .../llm_shield_proxy/stream_restorers.py | 18 ++- .../mcp_end_user_permission/__init__.py | 15 ++- .../mcp_jwt_signer/mcp_jwt_signer.py | 19 +-- .../mcp_security/mcp_security_guardrail.py | 7 +- .../microsoft_purview/__init__.py | 5 +- .../guardrail_hooks/noma/noma_v2.py | 24 ++-- .../guardrails/guardrail_hooks/onyx/onyx.py | 5 +- .../guardrail_hooks/openai/__init__.py | 7 +- .../guardrail_hooks/openai/moderations.py | 7 +- .../panw_prisma_airs/panw_prisma_airs.py | 45 ++++--- .../guardrails/guardrail_hooks/presidio.py | 6 +- .../guardrail_hooks/qualifire/qualifire.py | 4 +- .../semantic_guard/route_loader.py | 4 +- .../semantic_guard/semantic_guard.py | 27 ++-- .../tool_policy/tool_policy_guardrail.py | 2 +- .../unified_guardrail/unified_guardrail.py | 20 +-- .../proxy/guardrails/guardrail_registry.py | 18 +-- .../proxy/guardrails/tool_name_extraction.py | 8 +- litellm/proxy/guardrails/usage_endpoints.py | 50 +++++--- .../proxy/hooks/key_management_event_hooks.py | 2 +- .../proxy/hooks/parallel_request_limiter.py | 2 +- .../hooks/parallel_request_limiter_v3.py | 2 +- litellm/proxy/hooks/rate_limiter_utils.py | 5 +- .../access_group_endpoints.py | 29 +++-- .../auto_router_endpoints.py | 31 +++-- .../credential_migration.py | 3 +- .../key_management_endpoints.py | 46 ++++--- .../management_v1/spend_logs.py | 22 +++- .../mcp_management_endpoints.py | 12 +- .../model_management_endpoints.py | 11 +- .../router_settings_endpoints.py | 5 +- .../management_endpoints/team_endpoints.py | 12 +- litellm/proxy/management_endpoints/ui_sso.py | 30 +++-- .../object_permission_utils.py | 2 +- litellm/proxy/memory/memory_endpoints.py | 4 +- litellm/proxy/moyai_endpoints.py | 23 ++-- .../batch_file_validation.py | 7 +- .../storage_backend_service.py | 12 +- .../llm_passthrough_endpoints.py | 2 +- ...end_medical_passthrough_logging_handler.py | 8 +- .../tinyfish_passthrough_logging_handler.py | 5 +- .../transcribe_passthrough_logging_handler.py | 9 +- .../upstream_usage_headers.py | 3 +- .../policy_engine/attachment_registry.py | 26 ++-- .../proxy/policy_engine/pipeline_executor.py | 14 ++- .../proxy/policy_engine/policy_endpoints.py | 2 +- litellm/proxy/proxy_server.py | 116 ++++++++++++----- .../public_endpoints/public_v1/model_hub.py | 6 +- .../spend_tracking/budget_reservation.py | 4 +- .../spend_tracking/ptu_flat_cost_rollup.py | 7 +- .../spend_management_endpoints.py | 4 +- litellm/proxy/utils.py | 117 ++++++++++++------ litellm/proxy/video_endpoints/endpoints.py | 6 +- litellm/proxy/video_endpoints/utils.py | 2 +- litellm/rag/ingestion/bedrock_ingestion.py | 4 +- litellm/realtime_api/main.py | 14 ++- litellm/repositories/base_repository.py | 3 +- litellm/repositories/team_repository.py | 3 +- litellm/rerank_api/main.py | 4 +- litellm/rerank_api/rerank_utils.py | 4 +- .../responses/file_search/emulated_handler.py | 4 +- .../custom_tools.py | 6 +- .../streaming_iterator.py | 11 +- litellm/responses/main.py | 9 +- .../responses/mcp/chat_completions_handler.py | 5 +- litellm/responses/sse_output_recovery.py | 5 +- litellm/responses/streaming_iterator.py | 5 +- litellm/responses/utils.py | 104 +++++++++------- litellm/router.py | 4 +- .../quality_router/quality_router.py | 6 +- litellm/sandbox/__init__.py | 5 +- .../secret_managers/base_secret_manager.py | 4 +- .../cyberark_secret_manager.py | 4 +- litellm/secret_managers/main.py | 3 +- litellm/utils.py | 23 ++-- litellm/vector_stores/main.py | 14 ++- .../code_coverage_tests/recursive_detector.py | 2 +- tests/unit/a2a_protocol/test_utils.py | 14 +++ .../test_container_proxy_ownership.py | 19 +++ .../proxy/hooks/test_managed_files.py | 15 +++ .../dotprompt/test_prompt_manager.py | 38 ++++++ .../integrations/newrelic/test_newrelic.py | 13 ++ .../litellm_core_utils/attribute_forwarder.py | 6 + .../unit/llms/compactifai/test_compactifai.py | 37 +++++- ...gram_audio_transcription_transformation.py | 22 ++++ .../test_edenai_embedding_transformation.py | 6 + ..._edenai_image_generation_transformation.py | 6 + .../test_google_genai_transformation.py | 20 +++ .../test_gemini_image_edit_transformation.py | 20 +++ .../llms/gemini/test_gemini_common_utils.py | 19 +++ ...test_openai_count_tokens_transformation.py | 18 +++ ...test_openai_responses_guardrail_handler.py | 17 +++ .../runwayml/image_generation/__init__.py | 0 ..._runway_image_generation_transformation.py | 87 +++++++++++++ .../unit/proxy/auth/test_mcp_ip_filtering.py | 14 +++ .../client/cli/test_encryption_commands.py | 19 +++ .../common_utils/test_check_batch_cost.py | 11 ++ .../semantic_guard/test_semantic_guard.py | 14 +++ .../guardrail_hooks/test_llm_shield_proxy.py | 17 +++ .../test_storage_backend_service.py | 37 +++++- .../test_openapi_customization.py | 12 ++ 455 files changed, 3670 insertions(+), 1546 deletions(-) create mode 100644 tests/unit/litellm_core_utils/attribute_forwarder.py create mode 100644 tests/unit/llms/runwayml/image_generation/__init__.py create mode 100644 tests/unit/llms/runwayml/image_generation/test_runway_image_generation_transformation.py diff --git a/enterprise/enterprise_hooks/aporia_ai.py b/enterprise/enterprise_hooks/aporia_ai.py index 28b49bfce21..1e840f8dcc5 100644 --- a/enterprise/enterprise_hooks/aporia_ai.py +++ b/enterprise/enterprise_hooks/aporia_ai.py @@ -15,9 +15,10 @@ sys.path.insert( ) # Adds the parent directory to the system path import json import sys -from typing import Any, List, Literal, Optional +from typing import Final, List, Literal, Optional from fastapi import HTTPException +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -29,11 +30,13 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) +from litellm.llms.openai.common_utils import decoded_json_object from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata from litellm.types.guardrails import GuardrailEventHooks GUARDRAIL_NAME = "aporia" +_DECODED_JSON: Final = TypeAdapter(object) class AporiaGuardrail(CustomGuardrail): @@ -68,7 +71,7 @@ class AporiaGuardrail(CustomGuardrail): async def prepare_aporia_request( self, new_messages: List[dict], response_string: Optional[str] = None ) -> dict: - data: dict[str, Any] = {} + data: dict[str, object] = {} if new_messages is not None: data["messages"] = new_messages if response_string is not None: @@ -122,8 +125,10 @@ class AporiaGuardrail(CustomGuardrail): verbose_proxy_logger.debug("Aporia AI response: %s", response.text) if response.status_code == 200: # check if the response was flagged - _json_response = response.json() - action: str = _json_response.get( + _json_response: Final = decoded_json_object( + _DECODED_JSON.validate_python(response.json()) + ) + action: Final = _json_response.get( "action" ) # possible values are modify, passthrough, block, rephrase if action == "block": diff --git a/enterprise/enterprise_hooks/blocked_user_list.py b/enterprise/enterprise_hooks/blocked_user_list.py index f6998d58cd5..7e4d2514880 100644 --- a/enterprise/enterprise_hooks/blocked_user_list.py +++ b/enterprise/enterprise_hooks/blocked_user_list.py @@ -7,9 +7,10 @@ ## This accepts a list of user id's for whom calls will be rejected -from typing import Literal, Optional +from typing import Final, Literal, Optional from fastapi import HTTPException +from pydantic import TypeAdapter import litellm from litellm._internal_context import with_service_target @@ -19,6 +20,9 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import LiteLLM_EndUserTable, UserAPIKeyAuth from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET from litellm.proxy.utils import PrismaClient +from litellm.repositories.table_repositories import EndUserRepository + +_REQUEST_BODY_VALUE: Final = TypeAdapter(object) class ENTERPRISE_BlockedUserList(CustomLogger): @@ -94,15 +98,14 @@ class ENTERPRISE_BlockedUserList(CustomLogger): ) if end_user_cache_obj is None and self.prisma_client is not None: # check db - end_user_obj = ( - await self.prisma_client.db.litellm_endusertable.find_unique( - where={"user_id": user} - ) - ) + end_user_id: Final = _REQUEST_BODY_VALUE.validate_python(user) + end_user_obj = await EndUserRepository( + self.prisma_client + ).table.find_unique(where={"user_id": end_user_id}) if end_user_obj is None: # user not in db - assume not blocked end_user_obj = LiteLLM_EndUserTable(user_id=user, blocked=False) cache.set_cache(key=cache_key, value=end_user_obj, ttl=60) - if end_user_obj is not None and end_user_obj.blocked is True: + if end_user_obj.blocked is True: raise HTTPException( status_code=400, detail={ diff --git a/enterprise/enterprise_hooks/google_text_moderation.py b/enterprise/enterprise_hooks/google_text_moderation.py index b3ef5a3d496..7dbbc9ed6ce 100644 --- a/enterprise/enterprise_hooks/google_text_moderation.py +++ b/enterprise/enterprise_hooks/google_text_moderation.py @@ -6,7 +6,10 @@ # +-----------------------------------------------+ # Thank you users! We ❤️ you! - Krrish & Ishaan +from typing import Final + from fastapi import HTTPException +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -15,6 +18,8 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails._content_utils import iter_message_text from litellm.types.utils import CallTypesLiteral +_THRESHOLD_VALUE: Final = TypeAdapter(object) + class ENTERPRISE_GoogleTextModeration(CustomLogger): user_api_key_cache = None @@ -69,9 +74,11 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger): f"{category}_confidence_threshold", default_confidence_threshold, ) - set_confidence_value = getattr( - self, - f"{category}_confidence_threshold", + set_confidence_value = _THRESHOLD_VALUE.validate_python( + getattr( + self, + f"{category}_confidence_threshold", + ) ) verbose_proxy_logger.info( f"Google Text Moderation: {category}_confidence_threshold: {set_confidence_value}" diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/example_logging_api.py b/enterprise/litellm_enterprise/enterprise_callbacks/example_logging_api.py index 14d34f5d1e8..e0190ea8da0 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/example_logging_api.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/example_logging_api.py @@ -1,15 +1,20 @@ # this is an example endpoint to receive data from litellm +from typing import Final + from fastapi import FastAPI, HTTPException, Request +from pydantic import TypeAdapter app = FastAPI() +_LOGGED_EVENT: Final = TypeAdapter(object) + @app.post("/log-event") async def log_event(request: Request): try: print("Received /log-event request") # noqa # Assuming the incoming request has JSON data - data = await request.json() + data: Final = _LOGGED_EVENT.validate_python(await request.json()) print("Received request data:") # noqa print(data) # noqa diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py b/enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py index 3422e8969b0..e12539eac53 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py @@ -8,25 +8,30 @@ ## This provides an LLM Guard Integration for content moderation on the proxy import asyncio -from typing import Final, Optional +from collections.abc import Mapping +from typing import Final import aiohttp from fastapi import HTTPException +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.openai.common_utils import decoded_json_object from litellm.proxy._types import UserAPIKeyAuth from litellm.secret_managers.main import get_secret_str from litellm.types.utils import CallTypesLiteral +_DECODED_JSON: Final = TypeAdapter(object) + class _ENTERPRISE_LLMGuard(CustomLogger): # Class variables or attributes def __init__( self, mock_testing: bool = False, - mock_redacted_text: Optional[dict] = None, + mock_redacted_text: Mapping[str, object] | None = None, ): self.mock_redacted_text = mock_redacted_text self.llm_guard_mode = litellm.llm_guard_mode @@ -66,7 +71,9 @@ class _ENTERPRISE_LLMGuard(CustomLogger): async with session.post( analyze_url, json={"prompt": text} ) as response: - redacted_text = await response.json() + redacted_text = _DECODED_JSON.validate_python( + await response.json() + ) verbose_proxy_logger.debug( f"LLM Guard: Received response - {redacted_text}" ) @@ -77,12 +84,13 @@ class _ENTERPRISE_LLMGuard(CustomLogger): "error": f"Invalid content moderation response: {redacted_text}" }, ) - if redacted_text.get("is_valid", None) is False: + redacted_response: Final = decoded_json_object(redacted_text) + if redacted_response.get("is_valid", None) is False: raise HTTPException( status_code=400, detail={"error": "Violated content safety policy"}, ) - sanitized_prompt = redacted_text.get("sanitized_prompt") + sanitized_prompt: Final = redacted_response.get("sanitized_prompt") return sanitized_prompt if isinstance(sanitized_prompt, str) else text except Exception as e: verbose_proxy_logger.exception( diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index df1f263a947..f3b1240f733 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -1040,8 +1040,8 @@ class CheckBatchCost: from litellm.proxy.proxy_server import proxy_config from litellm.secret_managers.main import str_to_bool - api_key: Final = getattr(job, "api_key", None) - team_id: Final = getattr(job, "team_id", None) + api_key: Final = job.api_key if hasattr(job, "api_key") else None + team_id: Final = job.team_id if hasattr(job, "team_id") else None has_key_row: Final = api_key is not None and is_valid_sha256_hash(api_key) try: key_row: Final = ( diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index bc35c4a0aba..48cbc0ab8ff 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -44,6 +44,7 @@ from litellm.constants import ( ) from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import require_attributes from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.litellm_core_utils.internal_call_metadata import requests_background from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -133,7 +134,7 @@ if TYPE_CHECKING: from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache from litellm.proxy.utils import PrismaClient as _PrismaClient - Span = Union[_Span, Any] + Span = _Span InternalUsageCache = _InternalUsageCache PrismaClient = _PrismaClient else: @@ -490,6 +491,14 @@ _PROVIDER_FILE_ID_KEYS: Final = tuple(dict.fromkeys(chain.from_iterable(_PROVIDE _PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS: Final = (0.5, 1.0, 2.0) +class _SupportsKeyedGet(Protocol): + def get(self, key: str, default: object, /) -> object: ... + + +def _keyed_get(source: object, key: str, default: object = None) -> object: + return require_attributes(source, _SupportsKeyedGet, "get").get(key, default) + + class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): # Class variables or attributes def __init__( @@ -1611,7 +1620,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return holders if holders or "input_file_id" in request else healthy_deployments async def async_pre_call_deployment_hook( - self, kwargs: Dict[str, Any], call_type: Optional[CallTypes] + self, kwargs: dict[str, object], call_type: Optional[CallTypes] ) -> Optional[dict]: """ Allow modifying the request just before it's sent to the deployment. @@ -1623,11 +1632,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping")) # model_info may be at top-level or nested under litellm_metadata # (batch/file operations use litellm_metadata) - model_id = cast(Optional[str], kwargs.get("model_info", {}).get("id", None)) + model_id = cast(Optional[str], _keyed_get(kwargs.get("model_info", {}), "id")) if model_id is None: model_id = cast( Optional[str], - kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None), + _keyed_get(_keyed_get(kwargs.get("litellm_metadata", {}), "model_info", {}), "id"), ) for accessor_key in accessor_keys: input_file_id = cast(Optional[str], kwargs.get(accessor_key)) @@ -1824,7 +1833,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ), ) - async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span) -> dict: + async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span | None) -> dict: """ Get model-specific file IDs for a list of proxy file IDs. Returns a dictionary mapping litellm_proxy/ file_id -> model_id -> model_file_id @@ -1866,7 +1875,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): llm_router: Optional[Router], _create_file_request: CreateFileRequest, target_model_names_list: List[str], - litellm_parent_otel_span: Span, + litellm_parent_otel_span: Span | None, ) -> List[OpenAIFileObject]: if llm_router is None: raise Exception("LLM Router not initialized. Ensure models added to proxy.") @@ -1882,7 +1891,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): create_file_request: CreateFileRequest, llm_router: Router, target_model_names_list: List[str], - litellm_parent_otel_span: Span, + litellm_parent_otel_span: Span | None, user_api_key_dict: UserAPIKeyAuth, ) -> OpenAIFileObject: responses = await self.create_file_for_each_model( @@ -1949,7 +1958,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): file_objects: List[OpenAIFileObject], create_file_request: CreateFileRequest, internal_usage_cache: InternalUsageCache, - litellm_parent_otel_span: Span, + litellm_parent_otel_span: Span | None, target_model_names_list: List[str], ) -> OpenAIFileObject: ## GET THE FILE TYPE FROM THE CREATE FILE REQUEST @@ -2652,7 +2661,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): # Continue with other files even if one fails continue - def _get_content_type_from_file_object(self, file_object: Optional[Any]) -> str: + def _get_content_type_from_file_object(self, file_object: object) -> str: """ Determine content type from file object. diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py b/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py index 0e9657aba96..fcaf58099b9 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py @@ -29,7 +29,7 @@ if TYPE_CHECKING: from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache from litellm.proxy.utils import PrismaClient as _PrismaClient - Span = Union[_Span, Any] + Span = _Span InternalUsageCache = _InternalUsageCache PrismaClient = _PrismaClient else: @@ -107,7 +107,7 @@ class _PROXY_LiteLLMManagedVectorStores( llm_router: Router, model: str, request_data: dict[str, object] | VectorStoreCreateOptionalRequestParams, - litellm_parent_otel_span: Span, + litellm_parent_otel_span: Span | None, ) -> VectorStoreCreateResponse: """ Create a vector store for a specific model. @@ -134,7 +134,7 @@ class _PROXY_LiteLLMManagedVectorStores( create_request: VectorStoreCreateOptionalRequestParams, llm_router: Router, target_model_names_list: List[str], - litellm_parent_otel_span: Span, + litellm_parent_otel_span: Span | None, user_api_key_dict: UserAPIKeyAuth, ) -> VectorStoreCreateResponse: """ diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py index db419fde1e6..b91b4a970e8 100644 --- a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py @@ -36,6 +36,7 @@ from litellm.repositories.budget_repository import BudgetRepository from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.project_repository import ProjectRepository +from litellm.repositories.table_repositories import PrismaTableRepository from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository from litellm.repositories.verification_token_repository import VerificationTokenRepository @@ -73,6 +74,16 @@ def _verification_token_table( return VerificationTokenRepository(prisma_client).table +class _WriterVerificationTokenRepository(PrismaTableRepository["prisma_models.LiteLLM_VerificationToken"]): + table_name = "litellm_verificationtoken" + + +def _writer_verification_token_table( + prisma_client: PrismaClient, +) -> TableActions["prisma_models.LiteLLM_VerificationToken"]: + return _WriterVerificationTokenRepository(prisma_client, use_writer=True).table + + def _budget_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_BudgetTable"]: return BudgetRepository(prisma_client).table @@ -849,7 +860,7 @@ async def update_project( else None ) if current_project is not None and data.team_id != current_project.team_id: - mismatched_key_count: Final = await prisma_client.writer_db.litellm_verificationtoken.count( + mismatched_key_count: Final = await _writer_verification_token_table(prisma_client).count( where={ "project_id": data.project_id, "OR": [{"team_id": {"not": data.team_id}}, {"team_id": None}], diff --git a/enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py b/enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py index 9b87f4f1a7f..19d4988452b 100644 --- a/enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py +++ b/enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py @@ -14,6 +14,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Final, List, Optional, Protocol from fastapi import APIRouter, Depends, HTTPException +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -37,6 +38,7 @@ if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient router = APIRouter() +_DECODED_JSON: Final = TypeAdapter(object) class ManagedVectorStoreRow(Protocol): @@ -348,11 +350,11 @@ async def update_vector_store( raise HTTPException(status_code=500, detail="Database not connected") try: - update_data = data.model_dump(exclude_unset=True) - vector_store_id: Final[str] = update_data.pop("vector_store_id") + update_data = data.model_dump(exclude_unset=True, exclude={"vector_store_id"}) + vector_store_id: Final = data.vector_store_id if update_data.get("vector_store_metadata") is not None: update_data["vector_store_metadata"] = safe_dumps( - update_data["vector_store_metadata"] + _DECODED_JSON.validate_python(update_data["vector_store_metadata"]) ) updated = await managed_vector_store_table(prisma_client).update( diff --git a/litellm/__init__.py b/litellm/__init__.py index 1784deeb797..be57c8f3f16 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -175,7 +175,7 @@ argilla_batch_size: Optional[int] = None datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload. gcs_pub_sub_use_v1: Optional[bool] = False # if you want to use v1 gcs pubsub logged payload generic_api_use_v1: Optional[bool] = False # if you want to use v1 generic api logged payload -argilla_transformation_object: Optional[Dict[str, Any]] = None +argilla_transformation_object: Mapping[str, str] | None = None _async_input_callback: List[Union[str, Callable[..., object], "CustomLogger"]] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. @@ -333,7 +333,7 @@ in_memory_llm_clients_cache: "LLMClientCache" safe_memory_mode: bool = False enable_azure_ad_token_refresh: Optional[bool] = False # Proxy Authentication - auto-obtain/refresh OAuth2/JWT tokens for LiteLLM Proxy -proxy_auth: Optional[Any] = None +proxy_auth: _ProxyAuthHandlerType | None = None ### DEFAULT AZURE API VERSION ### AZURE_DEFAULT_API_VERSION = "2025-02-01-preview" # this is updated to the latest ### DEFAULT WATSONX API VERSION ### @@ -503,7 +503,7 @@ skill_search_embedding_model: Optional[str] = None # Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]]) # New format: { "displayName": { "url": "...", "index": 0 } } # Old format: { "displayName": "url" } (for backward compatibility) -public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {} +public_model_groups_links: dict[str, str | dict[str, object]] = {} #### REQUEST PRIORITIZATION ####### priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]]] = None # priority_reservation_settings is lazy-loaded via __getattr__ @@ -2289,6 +2289,10 @@ if TYPE_CHECKING: LiteLLM_UpperboundKeyGenerateParams, ) from litellm.utils import ModelResponseListIterator as _ModelResponseListIterator + from collections.abc import Awaitable + from litellm.types.utils import ModelResponse as _ModelResponseType + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper as _CustomStreamWrapperType + from litellm.proxy_auth import ProxyAuthHandler as _ProxyAuthHandlerType # Cost calculator functions cost_per_token: Callable[..., Tuple[float, float]] @@ -2312,7 +2316,7 @@ if TYPE_CHECKING: supports_audio_output: Callable[..., bool] supports_system_messages: Callable[..., bool] supports_reasoning: Callable[..., bool] - acreate: Callable[..., Any] + acreate: Callable[..., Awaitable[_ModelResponseType | _CustomStreamWrapperType]] get_max_tokens: Callable[..., int] get_model_info: Callable[..., _ModelInfoType] register_prompt_template: Callable[..., None] @@ -2610,8 +2614,10 @@ def __getattr__(name: str) -> Any: cancel_run, delete_run, ) + from types import MappingProxyType - return locals()[name] + evals_exports: Final[Mapping[str, object]] = MappingProxyType(locals()) + return evals_exports[name] raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index 46126c444a6..1a7dc343745 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -19,7 +19,7 @@ import importlib import sys from collections.abc import Callable, Mapping from types import ModuleType -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final from typing_extensions import ReadOnly, TypedDict @@ -456,7 +456,7 @@ def _lazy_import_http_handlers(name: str) -> object: params: Final = {"timeout": async_timeout, "client_alias": "module level aclient"} # Create the client instance - provider_id: Final = cast(Any, "litellm_module_level_client") + provider_id: Final = "litellm_module_level_client" async_client: Final = get_async_httpx_client( llm_provider=provider_id, params=params, diff --git a/litellm/a2a_protocol/cost_calculator.py b/litellm/a2a_protocol/cost_calculator.py index 31ca81c44dd..9a1431d6457 100644 --- a/litellm/a2a_protocol/cost_calculator.py +++ b/litellm/a2a_protocol/cost_calculator.py @@ -5,7 +5,10 @@ Supports dynamic cost parameters that allow platform owners to define custom costs per agent query or per token. """ -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Final, SupportsFloat, SupportsIndex + +from typing_extensions import Buffer if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import ( @@ -15,6 +18,15 @@ else: LitellmLoggingObject = Any +def _custom_response_cost(model_call_details: Mapping[str, object]) -> float | None: + response_cost: Final = model_call_details.get("response_cost", None) + if response_cost is None: + return None + if isinstance(response_cost, (str, Buffer, SupportsFloat, SupportsIndex)): + return float(response_cost) + raise TypeError(f"float() argument must be a string or a real number, not {type(response_cost).__name__!r}") + + class A2ACostCalculator: @staticmethod def calculate_a2a_cost( @@ -45,9 +57,9 @@ class A2ACostCalculator: model_call_details: Final = litellm_logging_obj.model_call_details # Check if user set a custom response cost (backward compatibility) - response_cost: Final = model_call_details.get("response_cost", None) - if response_cost is not None: - return float(response_cost) + custom_response_cost: Final = _custom_response_cost(model_call_details) + if custom_response_cost is not None: + return custom_response_cost # Get litellm_params for cost parameters litellm_params: Final = model_call_details.get("litellm_params", {}) or {} diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py index 404bcca712f..1cea0194715 100644 --- a/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py @@ -84,14 +84,23 @@ def _filter_reserved_headers( return filtered or None +def _calling_key_hash(litellm_params: Mapping[str, object]) -> str | None: + principal: Final = litellm_params.get(A2A_USER_API_KEY_HASH_PARAM) + if principal is None or isinstance(principal, str): + return principal + if not principal: + return None + raise AttributeError(f"{type(principal).__name__!r} object has no attribute 'encode'", name="encode", obj=principal) + + def _request_scoped_runtime_session_id( params: Mapping[str, object], - litellm_params: Mapping[str, Any], + litellm_params: Mapping[str, object], ) -> str | None: context_id: Final = get_session_id_from_a2a_params(params) if not isinstance(context_id, str) or not context_id: return None - return scope_session_to_principal(context_id, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM)) + return scope_session_to_principal(context_id, _calling_key_hash(litellm_params)) def _validate_runtime_session_id(session_id: str, model: str) -> str: @@ -109,6 +118,13 @@ def _validate_runtime_session_id(session_id: str, model: str) -> str: ) +def _agentcore_model_name(litellm_params: Mapping[str, object]) -> str: + model: Final = litellm_params.get("model", "") + if isinstance(model, str): + return model + raise AttributeError(f"{type(model).__name__!r} object has no attribute 'startswith'", name="startswith", obj=model) + + class BedrockAgentCoreA2ATransformation: """ Request/response transformation for Bedrock AgentCore A2A agents. @@ -152,11 +168,8 @@ class BedrockAgentCoreA2ATransformation: """ # Extract model and strip the "bedrock/" prefix # "bedrock/agentcore/arn:aws:..." → "agentcore/arn:aws:..." - model: Final = litellm_params.get("model", "") - if model.startswith("bedrock/"): - agentcore_model = model[len("bedrock/") :] - else: - agentcore_model = model + model: Final = _agentcore_model_name(litellm_params) + agentcore_model: Final = model.removeprefix("bedrock/") # Build optional_params from litellm_params (everything except model and custom_llm_provider) optional_params: Final = {k: v for k, v in litellm_params.items() if k not in ("model", "custom_llm_provider")} diff --git a/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py b/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py index 57c4a4677d0..c2d581321cd 100644 --- a/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py +++ b/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py @@ -8,11 +8,31 @@ WXO uses a REST API (not A2A/JSON-RPC) with an async-poll execution model: """ import asyncio -from collections.abc import AsyncIterator, Mapping -from typing import Any, Final +from collections.abc import AsyncIterator, Iterable, Mapping, Sequence +from itertools import chain +from typing import Final from uuid import uuid4 from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items +from litellm.llms.openai.common_utils import decoded_json_object + + +def _a2a_text_part(part: object) -> Sequence[object]: + part_fields: Final = as_str_mapping(part) + if not isinstance(part_fields, dict): + return () + if part_fields.get("kind") in (None, "", "text") and part_fields.get("text"): + return (part_fields["text"],) + return () + + +def _join_with_spaces(texts: Sequence[object]) -> str: + strings: Final = [text for text in texts if isinstance(text, str)] + if len(strings) == len(texts): + return " ".join(strings) + first_bad: Final = next(position for position, text in enumerate(texts) if not isinstance(text, str)) + raise TypeError(f"sequence item {first_bad}: expected str instance, {type(texts[first_bad]).__name__} found") class WatsonxOrchestrateTransformation: @@ -29,22 +49,16 @@ class WatsonxOrchestrateTransformation: return f"{cp4d_host.rstrip('/')}/orchestrate/cpd/instances/{instance_id}" @staticmethod - def extract_text_from_a2a_params(params: dict[str, Any]) -> str: + def extract_text_from_a2a_params(params: Mapping[str, object]) -> str: """ Extract user message text from A2A MessageSendParams. A2A format: params.message.parts[*] where part.kind == "text" """ - message: Final = params.get("message", {}) - parts: Final = message.get("parts", []) - texts: Final = [] - for part in parts: - if not isinstance(part, dict): - continue - kind = part.get("kind") - if kind in (None, "", "text") and part.get("text"): - texts.append(part["text"]) - return " ".join(texts) or "" + message: Final = decoded_json_object(params.get("message", {})) + parts: Final[Iterable[object]] = iter_items(message.get("parts", [])) + texts: Final = list(chain.from_iterable(map(_a2a_text_part, parts))) + return _join_with_spaces(texts) or "" @staticmethod def build_wxo_run_body( diff --git a/litellm/a2a_protocol/utils.py b/litellm/a2a_protocol/utils.py index 7400844bf28..17d1b86a609 100644 --- a/litellm/a2a_protocol/utils.py +++ b/litellm/a2a_protocol/utils.py @@ -4,15 +4,20 @@ Utility functions for A2A protocol. import hashlib from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final, Protocol import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import has_attributes if TYPE_CHECKING: from a2a.types import SendMessageRequest, SendStreamingMessageRequest +class _SupportsModelDump(Protocol): + def model_dump(self, *, mode: str) -> object: ... + + class A2ARequestUtils: """Utility class for A2A request/response processing.""" @@ -74,7 +79,7 @@ class A2ARequestUtils: @staticmethod def get_input_message_from_request( request: "SendMessageRequest | SendStreamingMessageRequest", - ) -> Any: + ) -> object: """ Extract the input message from an A2A request. @@ -84,7 +89,7 @@ class A2ARequestUtils: Returns: The message object/dict or None """ - params: Final = getattr(request, "params", None) + params: Final[object] = getattr(request, "params", None) if params is None: return None return getattr(params, "message", None) @@ -127,10 +132,12 @@ class A2ARequestUtils: # the dict branch — request-side parts are a2a-sdk Part RootModels whose # kind/text live on part.root, which the object branch cannot read. This # mirrors how the response side already works (it operates on model_dump). - input_message = A2ARequestUtils.get_input_message_from_request(request) - if input_message is not None and hasattr(input_message, "model_dump"): - input_message = input_message.model_dump(mode="json") - input_text: Final = A2ARequestUtils.extract_text_from_message(input_message) + input_message: Final = A2ARequestUtils.get_input_message_from_request(request) + input_text: Final = A2ARequestUtils.extract_text_from_message( + input_message.model_dump(mode="json") + if has_attributes(input_message, _SupportsModelDump, "model_dump") + else input_message + ) prompt_tokens: Final = A2ARequestUtils.count_tokens(input_text) # Count output tokens diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index 61e36bd1e96..fec1d8e4465 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -45,6 +45,7 @@ _BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_na BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]] _BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig) +_DECODED_JSON: Final = TypeAdapter(object) class GetAnthropicBetaHeadersConfig: @@ -76,7 +77,7 @@ class GetAnthropicBetaHeadersConfig: } @staticmethod - def _check_is_valid_dict(fetched_config: dict) -> bool: + def _check_is_valid_dict(fetched_config: object) -> bool: """Check if fetched config is a non-empty dict with expected structure.""" if not isinstance(fetched_config, dict): verbose_logger.warning( @@ -110,7 +111,7 @@ class GetAnthropicBetaHeadersConfig: return True @classmethod - def validate_beta_headers_config(cls, fetched_config: dict) -> bool: + def validate_beta_headers_config(cls, fetched_config: object) -> bool: """ Validate the integrity of a fetched beta headers config. @@ -119,7 +120,7 @@ class GetAnthropicBetaHeadersConfig: return cls._check_is_valid_dict(fetched_config) @staticmethod - def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> dict: + def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> object: """ Fetch the beta headers config from a remote URL. @@ -128,7 +129,7 @@ class GetAnthropicBetaHeadersConfig: """ response: Final = httpx.get(url, timeout=timeout) response.raise_for_status() - return response.json() + return _DECODED_JSON.validate_python(response.json()) def get_beta_headers_config(url: str) -> dict: @@ -161,7 +162,9 @@ def get_beta_headers_config(url: str) -> dict: return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config() # Validate the fetched config - if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content): + if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content) or not isinstance( + content, dict + ): verbose_logger.warning( "LiteLLM: Fetched beta headers config failed integrity check. Using local backup instead. url=%s", url, diff --git a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py index eb3ec8aaee2..d8198e41239 100644 --- a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py +++ b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py @@ -7,10 +7,14 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop import json from typing import Final +from pydantic import TypeAdapter + from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicErrorType +_PARSED_ERROR_BODY: Final = TypeAdapter(object) + # HTTP status code -> Anthropic error type # Source: https://docs.anthropic.com/en/api/errors ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = { @@ -79,7 +83,7 @@ class AnthropicExceptionMapping: - Generic: {"message": "..."} - Plain strings """ - parsed: Final = safe_json_loads(raw_message) + parsed: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message)) if isinstance(parsed, dict): # Bedrock format if "detail" in parsed and isinstance(parsed["detail"], dict): @@ -145,9 +149,8 @@ class AnthropicExceptionMapping: AnthropicErrorResponse dict """ # Try to parse as JSON once - parsed: dict | None = safe_json_loads(raw_message) - if not isinstance(parsed, dict): - parsed = None + decoded: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message)) + parsed: Final[dict[str, object] | None] = decoded if isinstance(decoded, dict) else None # If parsed and already in Anthropic format - passthrough if parsed is not None and AnthropicExceptionMapping._is_anthropic_error_dict(parsed): diff --git a/litellm/assistants/main.py b/litellm/assistants/main.py index a3428261636..a5721c6f047 100644 --- a/litellm/assistants/main.py +++ b/litellm/assistants/main.py @@ -77,7 +77,7 @@ def get_assistants( api_base: str | None = None, api_version: str | None = None, **kwargs, -) -> SyncCursorPage[Assistant]: +) -> SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]]: aget_assistants: Final[bool | None] = kwargs.pop("aget_assistants", None) if aget_assistants is not None and not isinstance(aget_assistants, bool): raise Exception("Invalid value passed in for aget_assistants. Only bool or None allowed") @@ -100,7 +100,7 @@ def get_assistants( elif timeout is None: timeout = 600.0 - response: SyncCursorPage[Assistant] | None = None + response: SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]] | None = None if custom_llm_provider == "openai": api_base = ( optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there @@ -564,7 +564,7 @@ def create_thread( tool_resources: OpenAICreateThreadParamsToolResources | None = None, client: OpenAI | None = None, **kwargs, -) -> Thread: +) -> Thread | Coroutine[None, None, Thread]: """ - get the llm provider - if openai - route it there @@ -610,7 +610,7 @@ def create_thread( api_base: str | None = None api_key: str | None = None - response: Thread | None = None + response: Thread | Coroutine[None, None, Thread] | None = None if custom_llm_provider == "openai": api_base = ( optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there @@ -735,7 +735,7 @@ def get_thread( thread_id: str, client=None, **kwargs, -) -> Thread: +) -> Thread | Coroutine[None, None, Thread]: """Get the thread object, given a thread_id""" aget_thread: Final = kwargs.pop("aget_thread", None) optional_params: Final = GenericLiteLLMParams(**kwargs) @@ -757,7 +757,7 @@ def get_thread( timeout = 600.0 api_base: str | None = None api_key: str | None = None - response: Thread | None = None + response: Thread | Coroutine[None, None, Thread] | None = None if custom_llm_provider == "openai": api_base = ( optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there @@ -903,7 +903,7 @@ def add_message( metadata: dict | None = None, client=None, **kwargs, -) -> OpenAIMessage: +) -> OpenAIMessage | Coroutine[None, None, OpenAIMessage]: ### COMMON OBJECTS ### a_add_message: Final = kwargs.pop("a_add_message", None) _message_data: Final = MessageData(role=role, content=content, attachments=attachments, metadata=metadata) @@ -935,7 +935,7 @@ def add_message( timeout = 600.0 api_key: str | None = None api_base: str | None = None - response: OpenAIMessage | None = None + response: OpenAIMessage | Coroutine[None, None, OpenAIMessage] | None = None if custom_llm_provider == "openai": api_base = ( optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there @@ -1065,7 +1065,7 @@ def get_messages( thread_id: str, client: Any | None = None, **kwargs, -) -> SyncCursorPage[OpenAIMessage]: +) -> SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]: aget_messages: Final = kwargs.pop("aget_messages", None) optional_params: Final = GenericLiteLLMParams(**kwargs) litellm_params_dict: Final = get_litellm_params(**kwargs) @@ -1086,7 +1086,7 @@ def get_messages( elif timeout is None: timeout = 600.0 - response: SyncCursorPage[OpenAIMessage] | None = None + response: SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]] | None = None api_key: str | None = None api_base: str | None = None if custom_llm_provider == "openai": @@ -1256,7 +1256,7 @@ def run_thread( client: object | None = None, event_handler: AssistantEventHandler | None = None, # for stream=True calls **kwargs, -) -> Run: +) -> Run | Coroutine[None, None, Run]: """Run a given thread + assistant.""" arun_thread: Final = kwargs.pop("arun_thread", None) optional_params: Final = GenericLiteLLMParams(**kwargs) @@ -1278,7 +1278,7 @@ def run_thread( elif timeout is None: timeout = 600.0 - response: Run | None = None + response: Run | Coroutine[None, None, Run] | None = None if custom_llm_provider == "openai": api_base = ( optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there diff --git a/litellm/caching/_embedding_router.py b/litellm/caching/_embedding_router.py index d82e58ecc91..0e7589d88ab 100644 --- a/litellm/caching/_embedding_router.py +++ b/litellm/caching/_embedding_router.py @@ -13,7 +13,7 @@ This module is dependency-injected: callers pass the proxy ``llm_router`` and from __future__ import annotations from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final import litellm from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS @@ -25,12 +25,12 @@ if TYPE_CHECKING: def resolve_embedding_router( embedding_model: str, llm_router: Router | None, - llm_model_list: list[dict[str, Any]] | None, + llm_model_list: Sequence[Mapping[str, object]] | None, ) -> Router | None: """Return ``llm_router`` iff it serves ``embedding_model`` as a deployment.""" if llm_router is None: return None - router_model_names: Final[list[str]] = ( + router_model_names: Final[Sequence[object]] = ( [m["model_name"] for m in llm_model_list if "model_name" in m] if llm_model_list is not None else [] ) if embedding_model in router_model_names: diff --git a/litellm/caching/azure_blob_cache.py b/litellm/caching/azure_blob_cache.py index bf0e15421bc..5b9071dc920 100644 --- a/litellm/caching/azure_blob_cache.py +++ b/litellm/caching/azure_blob_cache.py @@ -13,10 +13,14 @@ import json from contextlib import suppress from typing import Final +from pydantic import TypeAdapter + from litellm._logging import print_verbose, verbose_logger from .base_cache import BaseCache +_DECODED_BLOB: Final = TypeAdapter(object) + class AzureBlobCache(BaseCache): def __init__(self, account_url, container) -> None: @@ -65,7 +69,7 @@ class AzureBlobCache(BaseCache): print_verbose(f"Get Azure Blob Cache: key: {key}") as_bytes: Final = self.container_client.download_blob(key).readall() as_str: Final = as_bytes.decode("utf-8") - cached_response: Final[object] = json.loads(as_str) + cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str)) verbose_logger.debug( "Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s", @@ -86,7 +90,7 @@ class AzureBlobCache(BaseCache): blob: Final = await self.async_container_client.download_blob(key) as_bytes: Final = await blob.readall() as_str: Final = as_bytes.decode("utf-8") - cached_response: Final[object] = json.loads(as_str) + cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str)) verbose_logger.debug( "Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s", key, diff --git a/litellm/caching/base_cache.py b/litellm/caching/base_cache.py index 51c169ba796..fb91d13a8c9 100644 --- a/litellm/caching/base_cache.py +++ b/litellm/caching/base_cache.py @@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Any, Final if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - Span = _Span | Any + Span = _Span else: Span = Any diff --git a/litellm/caching/disk_cache.py b/litellm/caching/disk_cache.py index 50939ad51ca..9470f23570a 100644 --- a/litellm/caching/disk_cache.py +++ b/litellm/caching/disk_cache.py @@ -1,15 +1,19 @@ import json from typing import TYPE_CHECKING, Any, Final +from pydantic import TypeAdapter + from .base_cache import BaseCache if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - Span = _Span | Any + Span = _Span else: Span = Any +_DECODED_DISK_VALUE: Final = TypeAdapter(object) + class DiskCache(BaseCache): def __init__(self, disk_cache_dir: str | None = None): @@ -44,7 +48,7 @@ class DiskCache(BaseCache): original_cached_response: Final = self.disk_cache.get(key) if original_cached_response: try: - cached_response = json.loads(original_cached_response) + cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response)) except Exception: cached_response = original_cached_response return cached_response diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 98f6f1409f2..a0da0f6ddca 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1200,12 +1200,16 @@ def get_usage_object( elif isinstance(usage_obj, dict): return Usage(**usage_obj) elif isinstance(usage_obj, BaseModel): - return Usage(**usage_obj.model_dump()) + return _usage_from_model(usage_obj) else: verbose_logger.debug("Unknown usage object type: %s, usage_obj: %s", type(usage_obj), usage_obj) return None +def _usage_from_model(usage_model: BaseModel) -> Usage: + return Usage(**usage_model.model_dump()) + + def _decisions_prompt_tokens_details(usage: DecisionsUsage | OpenAIDecisionUsage) -> PromptTokensDetailsWrapper: match usage: case DecisionsUsage(): @@ -1686,7 +1690,7 @@ def completion_cost( setattr( completion_response, "usage", - litellm.Usage(**_usage_for_dump.model_dump()), + _usage_from_model(_usage_for_dump), ) if isinstance(usage_obj, (DecisionsUsage, OpenAIDecisionUsage)): _usage = _decisions_usage(usage_obj).model_dump() @@ -3035,8 +3039,11 @@ def _batch_prompt_cost( ) +_ATTRIBUTE_VALUE: Final = TypeAdapter(object) + + def _attribute_value(obj: object, name: str) -> object: - return getattr(obj, name) + return _ATTRIBUTE_VALUE.validate_python(getattr(obj, name)) def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]: @@ -3671,7 +3678,26 @@ def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo | return None -def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...]) -> float | None: +_TranscriptionRateKey: TypeAlias = Literal[ + "input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token" +] + + +def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None: + match key: + case "input_cost_per_second": + return info.get("input_cost_per_second") + case "input_cost_per_audio_token": + return info.get("input_cost_per_audio_token") + case "input_cost_per_token": + return info.get("input_cost_per_token") + case "output_cost_per_token": + return info.get("output_cost_per_token") + case _: + assert_never(key) + + +def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None: """First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry because ``get_model_info`` synthesizes zero token rates for entries that omit them.""" if info is None: @@ -3680,12 +3706,18 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...]) if declared is None: return None return next( - (float(value) for key in keys if declared.get(key) is not None and (value := info.get(key)) is not None), + ( + float(value) + for key in keys + if declared.get(key) is not None and (value := _model_info_rate(info, key)) is not None + ), None, ) -def _transcription_rate(keys: tuple[str, ...], override: ModelInfo | None, base: ModelInfo | None) -> float: +def _transcription_rate( + keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None +) -> float: rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base)) return next((rate for rate in rates if rate is not None), 0.0) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 733f1aa72d5..b8e933bf046 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -1273,7 +1273,7 @@ class MidStreamFallbackError(ServiceUnavailableError): self.message = _saved_message self.args = (_saved_message,) - original_headers: Final = getattr(original_exception, "litellm_response_headers", None) + original_headers: Final[object] = getattr(original_exception, "litellm_response_headers", None) if original_headers is not None: self.litellm_response_headers = original_headers diff --git a/litellm/experimental_mcp_client/tools.py b/litellm/experimental_mcp_client/tools.py index 8015b065899..4f204b3629e 100644 --- a/litellm/experimental_mcp_client/tools.py +++ b/litellm/experimental_mcp_client/tools.py @@ -10,6 +10,7 @@ from mcp.types import Tool as MCPTool from openai.types.chat import ChatCompletionToolParam from openai.types.responses.function_tool_param import FunctionToolParam from openai.types.shared_params.function_definition import FunctionDefinition +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.constants import ( @@ -20,6 +21,8 @@ from litellm.constants import ( from litellm.types.llms.anthropic import AnthropicMessagesTool from litellm.types.utils import ChatCompletionMessageToolCall +_DECODED_JSON: Final = TypeAdapter(object) + ######################################################## # List MCP Tool functions @@ -200,15 +203,18 @@ async def call_mcp_tool( return tool_result +def _decode_json_arguments(arguments: str) -> object: + try: + return _DECODED_JSON.validate_python(json.loads(arguments)) + except json.JSONDecodeError: + return {} + + def _get_function_arguments(function: FunctionDefinition) -> dict: """Helper to safely get and parse function arguments.""" - arguments = function.get("arguments", {}) - if isinstance(arguments, str): - try: - arguments = json.loads(arguments) - except json.JSONDecodeError: - arguments = {} - return arguments if isinstance(arguments, dict) else {} + arguments: Final = function.get("arguments", {}) + decoded: Final = _decode_json_arguments(arguments) if isinstance(arguments, str) else arguments + return decoded if isinstance(decoded, dict) else {} def transform_openai_tool_call_request_to_mcp_tool_call_request( diff --git a/litellm/files/main.py b/litellm/files/main.py index 16ff5b02f9c..c5172bc01ae 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -9,7 +9,7 @@ import asyncio import contextvars import time import uuid as uuid_module -from collections.abc import Coroutine +from collections.abc import Awaitable, Coroutine from functools import partial from typing import Final, Literal, cast @@ -293,6 +293,14 @@ def create_file( raise e +async def _await_retrieved_file(pending: Awaitable[FileObject | OpenAIFileObject]) -> FileObject | OpenAIFileObject: + return await pending + + +async def _await_deleted_file(pending: Awaitable[object]) -> object: + return await pending + + @client async def afile_retrieve( file_id: str, @@ -325,7 +333,7 @@ async def afile_retrieve( func_with_context: Final = partial(ctx.run, func) init_response: Final = await loop.run_in_executor(None, func_with_context) if asyncio.iscoroutine(init_response): - response = await init_response + response = await _await_retrieved_file(init_response) else: response = init_response @@ -505,7 +513,7 @@ async def afile_delete( func_with_context: Final = partial(ctx.run, func) init_response: Final = await loop.run_in_executor(None, func_with_context) if asyncio.iscoroutine(init_response): - response = await init_response + response = await _await_deleted_file(init_response) else: response = init_response diff --git a/litellm/files/streaming.py b/litellm/files/streaming.py index 460050872f4..e9a030e0e32 100644 --- a/litellm/files/streaming.py +++ b/litellm/files/streaming.py @@ -1,7 +1,7 @@ import datetime import traceback from collections.abc import AsyncIterator, Iterator -from typing import TYPE_CHECKING, Any, Final, Optional, cast +from typing import TYPE_CHECKING, Final, Optional, cast import anyio @@ -163,7 +163,7 @@ class FileContentStreamingResponse: merged_hidden_params: Final = cast( "StandardLoggingHiddenParams", { - **cast(dict[str, Any], payload.get("hidden_params") or {}), + **(payload.get("hidden_params") or {}), **self._hidden_params, }, ) diff --git a/litellm/harness/handlers/base.py b/litellm/harness/handlers/base.py index 18b7391e988..4721e9d6ff2 100644 --- a/litellm/harness/handlers/base.py +++ b/litellm/harness/handlers/base.py @@ -4,7 +4,6 @@ from __future__ import annotations from abc import ABC, abstractmethod from collections.abc import AsyncIterator -from typing import Any from litellm.harness.context import SessionContext from litellm.harness.errors import CapabilityUnsupported @@ -38,5 +37,5 @@ class BaseHarnessHandler(ABC): async def resume(self, ctx: SessionContext, native_session_id: str) -> None: """Continue the runtime's own session on the next turn.""" - async def history(self, ctx: SessionContext) -> list[dict[str, Any]]: # mutable-ok: public history() API shape + async def history(self, ctx: SessionContext) -> list[dict[str, object]]: # mutable-ok: public history() API shape raise CapabilityUnsupported(f"Harness.{self.config.harness.name} does not expose history") diff --git a/litellm/harness/handlers/cli_handler.py b/litellm/harness/handlers/cli_handler.py index 6320a877e6b..b01f449a67d 100644 --- a/litellm/harness/handlers/cli_handler.py +++ b/litellm/harness/handlers/cli_handler.py @@ -70,10 +70,10 @@ async def send_stdin(proc: Process, data: str) -> None: async def private_dir_for(sandbox: Sandbox) -> str: - tempdir = getattr(sandbox, "tempdir", None) + tempdir: Final = sandbox.tempdir if hasattr(sandbox, "tempdir") else None if tempdir is None: raise SandboxError(f"{type(sandbox).__name__} has no tempdir(); CLI harnesses need a private config dir") - path: str = await tempdir() + path: Final = await tempdir() return path diff --git a/litellm/harness/handlers/deepagents_handler.py b/litellm/harness/handlers/deepagents_handler.py index 4a36b1a08d1..f6785323292 100644 --- a/litellm/harness/handlers/deepagents_handler.py +++ b/litellm/harness/handlers/deepagents_handler.py @@ -161,7 +161,7 @@ class DeepAgentsHandler(BaseHarnessHandler): async def history( self, ctx: SessionContext - ) -> list[dict[str, Any]]: # mutable-ok: BaseHarnessHandler.history API returns OpenAI message dicts + ) -> list[dict[str, object]]: # mutable-ok: BaseHarnessHandler.history API returns OpenAI message dicts agent, deps = self._require_agent() snapshot = await agent.aget_state(self._run_config(ctx, None)) messages = (snapshot.values or MappingProxyType({})).get("messages") or () diff --git a/litellm/harness/runtime.py b/litellm/harness/runtime.py index dc24a1f0ef7..1d946521bfa 100644 --- a/litellm/harness/runtime.py +++ b/litellm/harness/runtime.py @@ -12,11 +12,10 @@ import inspect import logging import os import uuid -from collections.abc import AsyncIterator, Callable, Coroutine, Generator, Mapping, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Generator, Mapping, Sequence from dataclasses import dataclass, field from types import MappingProxyType from typing import ( - Any, Final, get_args, ) @@ -331,11 +330,15 @@ async def _put_end(queue: asyncio.Queue[Event | _End], end: _End) -> None: queue.put_nowait(end) +async def _await_decision(pending: Awaitable[object]) -> object: + return await pending + + async def call_approval_handler(handler: ApprovalHandler, approval: Approval) -> None: """Run on_approval (sync in a worker thread, or async) and resolve approval.""" try: if inspect.iscoroutinefunction(handler): - decision: object = await handler(approval) + decision: object = await _await_decision(handler(approval)) else: decision = await asyncio.to_thread(handler, approval) if inspect.isawaitable(decision): @@ -749,7 +752,7 @@ class AsyncSession: async def history( self, - ) -> list[dict[str, Any]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler + ) -> list[dict[str, object]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler if not self.harness_config.capabilities.history: raise CapabilityUnsupported(f"Harness.{self.config.harness.name} does not expose history") await self._ensure_ready() diff --git a/litellm/images/main.py b/litellm/images/main.py index 7f94835d59b..29bac3ffe05 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -3,7 +3,7 @@ import contextvars import importlib from collections.abc import Coroutine from functools import partial -from typing import TYPE_CHECKING, Final, Literal, Optional, cast, overload +from typing import TYPE_CHECKING, Final, Literal, cast, overload from pydantic import TypeAdapter, ValidationError @@ -70,16 +70,17 @@ from litellm.utils import ( _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints) # Cache for ImageEditRequestUtils to avoid repeated __getattr__ calls -_ImageEditRequestUtils_cache: Optional["ImageEditRequestUtils"] = None +_ImageEditRequestUtils_cache: type["ImageEditRequestUtils"] | None = None -def _get_ImageEditRequestUtils() -> "ImageEditRequestUtils": +def _get_ImageEditRequestUtils() -> type["ImageEditRequestUtils"]: """Get ImageEditRequestUtils, loading it lazily if needed.""" global _ImageEditRequestUtils_cache if _ImageEditRequestUtils_cache is None: # Access via module to trigger __getattr__ if not cached - module: Final = importlib.import_module(__name__) - _ImageEditRequestUtils_cache = module.ImageEditRequestUtils + from litellm.images import main as images_main + + _ImageEditRequestUtils_cache = images_main.ImageEditRequestUtils assert _ImageEditRequestUtils_cache is not None # Type narrowing for type checker return _ImageEditRequestUtils_cache diff --git a/litellm/integrations/SlackAlerting/hanging_request_check.py b/litellm/integrations/SlackAlerting/hanging_request_check.py index 6f986144d4c..87baee91780 100644 --- a/litellm/integrations/SlackAlerting/hanging_request_check.py +++ b/litellm/integrations/SlackAlerting/hanging_request_check.py @@ -115,14 +115,16 @@ class AlertingHangingRequestCheck: if hanging_request_data.alerted: continue - request_status = await proxy_logging_obj.internal_usage_cache.async_get_cache( - key=f"request_status:{hanging_request_data.request_id}", - litellm_parent_otel_span=None, - local_only=True, - ) # this means the request status was either success or fail # and is not hanging - if request_status is not None: + if ( + await proxy_logging_obj.internal_usage_cache.async_get_cache( + key=f"request_status:{hanging_request_data.request_id}", + litellm_parent_otel_span=None, + local_only=True, + ) + is not None + ): # clear this request from hanging request cache since the request was either success or failed self.hanging_request_cache._remove_key( key=request_id, diff --git a/litellm/integrations/SlackAlerting/user_spend_alerts.py b/litellm/integrations/SlackAlerting/user_spend_alerts.py index 38794735c1b..08799a2f59f 100644 --- a/litellm/integrations/SlackAlerting/user_spend_alerts.py +++ b/litellm/integrations/SlackAlerting/user_spend_alerts.py @@ -56,8 +56,9 @@ async def fetch_user_spend_rows( today_str: Final = today.strftime("%Y-%m-%d") month_start_str: Final = today.replace(day=1).strftime("%Y-%m-%d") baseline_start_str: Final = (today - datetime.timedelta(days=max(baseline_days, 1))).strftime("%Y-%m-%d") - raw: Final = await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str) - return USER_SPEND_ROWS_ADAPTER.validate_python(raw) + return USER_SPEND_ROWS_ADAPTER.validate_python( + await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str) + ) def _daily_threshold_event(row: UserSpendRow, args: SlackAlertingArgs, today_str: str) -> UserSpendAlertEvent | None: diff --git a/litellm/integrations/agentops/agentops.py b/litellm/integrations/agentops/agentops.py index 9043ab1853a..3364520a8ae 100644 --- a/litellm/integrations/agentops/agentops.py +++ b/litellm/integrations/agentops/agentops.py @@ -4,11 +4,16 @@ AgentOps integration for LiteLLM - Provides OpenTelemetry tracing for LLM calls import os from dataclasses import dataclass -from typing import Any, Final +from typing import Final + +from pydantic import TypeAdapter from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.custom_httpx.http_handler import get_httpx_client +_DECODED_JSON: Final = TypeAdapter(object) + @dataclass class AgentOpsConfig: @@ -58,9 +63,10 @@ class AgentOps(OpenTelemetry): project_id = None if config.api_key: try: - response: Final = self._fetch_auth_token(config.api_key, config.auth_endpoint) - jwt_token = response.get("token") - project_id = response.get("project_id") + response: Final = as_str_mapping(self._fetch_auth_token(config.api_key, config.auth_endpoint)) + if response is not None: + jwt_token = response.get("token") + project_id = response.get("project_id") except Exception: pass @@ -78,12 +84,9 @@ class AgentOps(OpenTelemetry): "telemetry.sdk.name": "agentops", } - if project_id: - resource_attrs["project.id"] = project_id + self.resource_attributes = {**resource_attrs, "project.id": project_id} if project_id else resource_attrs - self.resource_attributes = resource_attrs - - def _fetch_auth_token(self, api_key: str, auth_endpoint: str) -> dict[str, Any]: + def _fetch_auth_token(self, api_key: str, auth_endpoint: str) -> object: """ Fetch JWT authentication token from AgentOps API @@ -111,6 +114,6 @@ class AgentOps(OpenTelemetry): if response.status_code != 200: raise Exception(f"Failed to fetch auth token: {response.text}") - return response.json() + return _DECODED_JSON.validate_python(response.json()) finally: client.close() diff --git a/litellm/integrations/arize/arize.py b/litellm/integrations/arize/arize.py index 24279b443aa..5a03b91d470 100644 --- a/litellm/integrations/arize/arize.py +++ b/litellm/integrations/arize/arize.py @@ -166,7 +166,7 @@ class ArizeLogger(OpenTelemetry): primary ``otel`` callback which handles proxy-level parent spans. """ - def set_attributes(self, span: Span, kwargs, response_obj: Any | None): + def set_attributes(self, span: Span, kwargs, response_obj: object): ArizeLogger.set_arize_attributes(span, kwargs, response_obj) @staticmethod diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index 501f5749ea4..f72b29a6ad8 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -3,9 +3,10 @@ import os from datetime import datetime -from typing import Final +from typing import Final, Protocol import httpx +from typing_extensions import ReadOnly, TypedDict import litellm from litellm import verbose_logger @@ -24,6 +25,18 @@ from litellm.utils import print_verbose API_BASE: Final = "https://api.braintrustdata.com/v1" +class _BraintrustProjectBody(TypedDict): + id: ReadOnly[str] + + +class _BraintrustProjectResponse(Protocol): + def json(self) -> _BraintrustProjectBody: ... + + +def _project_id(response: _BraintrustProjectResponse) -> str: + return response.json()["id"] + + def get_utc_datetime(): import datetime as dt @@ -80,8 +93,7 @@ class BraintrustLogger(CustomLogger): headers=self.headers, json={"name": project_name}, ) - project_dict: Final = response.json() - project_id: Final = project_dict["id"] + project_id: Final = _project_id(response) self._project_id_cache[project_name] = project_id return project_id except httpx.HTTPStatusError as e: @@ -100,8 +112,7 @@ class BraintrustLogger(CustomLogger): headers=self.headers, json={"name": project_name}, ) - project_dict: Final = response.json() - project_id: Final = project_dict["id"] + project_id: Final = _project_id(response) self._project_id_cache[project_name] = project_id return project_id except httpx.HTTPStatusError as e: @@ -112,18 +123,14 @@ class BraintrustLogger(CustomLogger): f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"} ) - project_dict: Final = project.json() - - self.default_project_id = project_dict["id"] + self.default_project_id = _project_id(project) def create_sync_default_project_and_experiment(self): project: Final = self.global_braintrust_sync_http_handler.post( f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"} ) - project_dict: Final = project.json() - - self.default_project_id = project_dict["id"] + self.default_project_id = _project_id(project) def log_success_event(self, kwargs, response_obj, start_time, end_time): verbose_logger.debug("REACHES BRAINTRUST SUCCESS") diff --git a/litellm/integrations/cloudzero/cz_stream_api.py b/litellm/integrations/cloudzero/cz_stream_api.py index 1e0413e8576..d0b25ea83e8 100644 --- a/litellm/integrations/cloudzero/cz_stream_api.py +++ b/litellm/integrations/cloudzero/cz_stream_api.py @@ -19,8 +19,9 @@ """Output modules for writing CBF data to various destinations.""" import zoneinfo +from collections.abc import Mapping from datetime import datetime, timezone -from typing import Any, Final +from typing import Final import httpx import polars as pl @@ -188,7 +189,7 @@ class CloudZeroStreamer: ) raise - def _prepare_batch_payload(self, batch_date: str, batch_data: pl.DataFrame, operation: str) -> dict[str, Any]: + def _prepare_batch_payload(self, batch_date: str, batch_data: pl.DataFrame, operation: str) -> Mapping[str, object]: """Prepare batch payload according to CloudZero AnyCost API format.""" # Convert batch_date to month for the API (YYYY-MM format) try: @@ -205,7 +206,7 @@ class CloudZeroStreamer: if record: data_records.append(record) - payload: Final = {"month": month_str, "operation": operation, "data": data_records} + payload: Final[Mapping[str, object]] = {"month": month_str, "operation": operation, "data": data_records} return payload diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 1c12ac395c7..9e2dc902d55 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -352,7 +352,7 @@ class CustomGuardrail(CustomLogger): self, violation_message: str, request_data: dict[str, Any], - detection_info: dict[str, Any] | None = None, + detection_info: Mapping[str, object] | None = None, original_response: object = None, ) -> None: """ @@ -1368,7 +1368,10 @@ class CustomGuardrail(CustomLogger): if event_type is not None: guardrail_mode = event_type elif isinstance(self.event_hook, Mode): - guardrail_mode = GuardrailMode(**dict(self.event_hook.model_dump())) + guardrail_mode = GuardrailMode( + tags=copy.deepcopy(self.event_hook.tags), + default=copy.deepcopy(self.event_hook.default), + ) else: guardrail_mode = self.event_hook diff --git a/litellm/integrations/custom_secret_manager.py b/litellm/integrations/custom_secret_manager.py index e59842d409a..8fc5bc551c8 100644 --- a/litellm/integrations/custom_secret_manager.py +++ b/litellm/integrations/custom_secret_manager.py @@ -37,7 +37,6 @@ Usage: """ from abc import abstractmethod -from typing import Any import httpx @@ -154,7 +153,7 @@ class CustomSecretManager(BaseSecretManager): optional_params: dict | None = None, timeout: float | httpx.Timeout | None = None, tags: dict | list | None = None, - ) -> dict[str, Any]: + ) -> dict[str, object]: """ Asynchronously write a secret to your custom secret manager. diff --git a/litellm/integrations/dotprompt/__init__.py b/litellm/integrations/dotprompt/__init__.py index 4cf6c0de2df..0a2921db6c7 100644 --- a/litellm/integrations/dotprompt/__init__.py +++ b/litellm/integrations/dotprompt/__init__.py @@ -61,7 +61,9 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom prompt_file: Final = getattr(litellm_params, "prompt_file", None) # Handle dotprompt_content from database - dotprompt_content: Final = getattr(litellm_params, "dotprompt_content", None) + dotprompt_content: Final = ( + litellm_params.dotprompt_content if hasattr(litellm_params, "dotprompt_content") else None + ) if dotprompt_content and not prompt_data and not prompt_file: prompt_data = get_prompt_data_from_dotprompt_content(dotprompt_content) diff --git a/litellm/integrations/dotprompt/dotprompt_manager.py b/litellm/integrations/dotprompt/dotprompt_manager.py index c646dbf4e2e..77d70dd761c 100644 --- a/litellm/integrations/dotprompt/dotprompt_manager.py +++ b/litellm/integrations/dotprompt/dotprompt_manager.py @@ -7,6 +7,8 @@ import json from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final +from typing_extensions import ReadOnly, TypedDict + from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.integrations.prompt_management_base import PromptManagementClient from litellm.types.llms.openai import AllMessageValues @@ -18,7 +20,12 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any -from .prompt_manager import PromptManager, PromptTemplate +from .prompt_manager import PromptJsonEntry, PromptManager, PromptTemplate + + +class _PromptJsonData(TypedDict, total=False): + content: ReadOnly[str] + metadata: ReadOnly[dict[str, object]] class DotpromptManager(CustomPromptManagement): @@ -342,17 +349,17 @@ class DotpromptManager(CustomPromptManagement): if self._prompt_manager: self._prompt_manager.reload_prompts() - def add_prompt_from_json(self, prompt_id: str, json_data: dict[str, Any]) -> None: + def add_prompt_from_json(self, prompt_id: str, json_data: _PromptJsonData) -> None: """Add a prompt from JSON data.""" content: Final = json_data.get("content", "") metadata: Final = json_data.get("metadata", {}) self.prompt_manager.add_prompt(prompt_id, content, metadata) - def load_prompts_from_json(self, prompts_data: dict[str, dict[str, object]]) -> None: + def load_prompts_from_json(self, prompts_data: Mapping[str, object]) -> None: """Load multiple prompts from JSON data.""" self.prompt_manager.load_prompts_from_json_data(prompts_data) - def get_prompts_as_json(self) -> dict[str, dict[str, object]]: + def get_prompts_as_json(self) -> dict[str, PromptJsonEntry]: """Get all prompts in JSON format.""" return self.prompt_manager.get_all_prompts_as_json() diff --git a/litellm/integrations/dotprompt/prompt_manager.py b/litellm/integrations/dotprompt/prompt_manager.py index 0eda266740c..0e52ca0b62f 100644 --- a/litellm/integrations/dotprompt/prompt_manager.py +++ b/litellm/integrations/dotprompt/prompt_manager.py @@ -12,6 +12,8 @@ from jinja2 import DictLoader, select_autoescape from jinja2.sandbox import ImmutableSandboxedEnvironment from typing_extensions import NotRequired, ReadOnly, TypedDict +from litellm.litellm_core_utils.core_helpers import as_str_mapping + class _PromptFileJson(TypedDict): """JSON form of a .prompt file: rendered template text plus its frontmatter.""" @@ -20,6 +22,11 @@ class _PromptFileJson(TypedDict): metadata: ReadOnly[NotRequired[dict[str, object]]] +class PromptJsonEntry(TypedDict): + content: ReadOnly[str] + metadata: ReadOnly[dict[str, object]] + + def strip_version_suffix(prompt_id: str) -> str | None: base, separator, version = prompt_id.rpartition(".v") if separator and base and version.isdigit(): @@ -358,20 +365,26 @@ class PromptManager: return f"---\n{frontmatter_yaml}---\n{content}" - def get_all_prompts_as_json(self) -> dict[str, dict[str, Any]]: + def get_all_prompts_as_json(self) -> dict[str, PromptJsonEntry]: """Get all loaded prompts in JSON format. Returns: Dictionary mapping prompt_id to prompt data """ - result: Final = {} - for prompt_id, template in self.prompts.items(): - result[prompt_id] = { + return { + prompt_id: { "content": template.content, "metadata": template.metadata, } - return result + for prompt_id, template in self.prompts.items() + } - def load_prompts_from_json_data(self, prompt_data: dict[str, dict[str, object]]) -> None: - """Load additional prompts from JSON data (merges with existing prompts).""" - self._load_prompts_from_json(prompt_data) + def load_prompts_from_json_data(self, prompt_data: Mapping[str, object]) -> None: + """Load additional prompts from JSON data (merges with existing prompts). Entries that are not + mappings are skipped, the same as a malformed entry inside the loader.""" + entries: Final = { + prompt_id: dict(entry) + for prompt_id, raw in prompt_data.items() + if (entry := as_str_mapping(raw)) is not None + } + self._load_prompts_from_json(entries) diff --git a/litellm/integrations/focus/destinations/factory.py b/litellm/integrations/focus/destinations/factory.py index 0ecf04326b3..cf871013265 100644 --- a/litellm/integrations/focus/destinations/factory.py +++ b/litellm/integrations/focus/destinations/factory.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +from collections.abc import Mapping from typing import Any, Final from .base import FocusDestination @@ -20,7 +21,7 @@ class FocusDestinationFactory: *, provider: str, prefix: str, - config: dict[str, Any] | None = None, + config: Mapping[str, object] | None = None, ) -> FocusDestination: """Return a destination implementation for the requested provider.""" provider_lower: Final = provider.lower() @@ -39,7 +40,7 @@ class FocusDestinationFactory: def _resolve_config( *, provider: str, - overrides: dict[str, object], + overrides: Mapping[str, object], ) -> dict[str, Any]: if provider == "s3": resolved = { diff --git a/litellm/integrations/focus/export_engine.py b/litellm/integrations/focus/export_engine.py index 615aba72954..5c345630984 100644 --- a/litellm/integrations/focus/export_engine.py +++ b/litellm/integrations/focus/export_engine.py @@ -2,8 +2,8 @@ from __future__ import annotations -from collections.abc import Callable -from typing import Any, Final, SupportsFloat, SupportsIndex, SupportsInt, cast +from collections.abc import Callable, Mapping +from typing import Final, SupportsFloat, SupportsIndex, SupportsInt, cast import polars as pl @@ -24,7 +24,7 @@ class FocusExportEngine: provider: str, export_format: str, prefix: str, - destination_config: dict[str, Any] | None = None, + destination_config: Mapping[str, object] | None = None, ) -> None: self.provider = provider self.export_format = export_format @@ -77,7 +77,7 @@ class FocusExportEngine: return FocusParquetSerializer() raise NotImplementedError(f"Export format '{self.export_format}' not supported. Use 'parquet' or 'csv'.") - async def dry_run_export_usage_data(self, limit: int | None) -> dict[str, Any]: + async def dry_run_export_usage_data(self, limit: int | None) -> dict[str, object]: data: Final = await self.database.get_usage_data(limit=limit) normalized: Final = self.transformer.transform(data) diff --git a/litellm/integrations/focus/focus_logger.py b/litellm/integrations/focus/focus_logger.py index e961b53a240..71b61296cfc 100644 --- a/litellm/integrations/focus/focus_logger.py +++ b/litellm/integrations/focus/focus_logger.py @@ -6,6 +6,8 @@ import os from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, ClassVar, Final, cast +from typing_extensions import TypedDict, Unpack + import litellm from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger @@ -25,6 +27,10 @@ FOCUS_USAGE_DATA_JOB_NAME: Final = "focus_export_usage_data" DEFAULT_DRY_RUN_LIMIT: Final = 500 +class _CustomLoggerOptions(TypedDict, total=False, extra_items=object): + pass + + class FocusLogger(CustomLogger): """Coordinates Focus export jobs across transformer/serializer/destination layers.""" @@ -40,7 +46,7 @@ class FocusLogger(CustomLogger): interval_seconds: int | None = None, prefix: str | None = None, destination_config: dict[str, Any] | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomLoggerOptions], ) -> None: super().__init__(**kwargs) self.provider = (provider or os.getenv("FOCUS_PROVIDER") or "s3").lower() diff --git a/litellm/integrations/gcs_bucket/gcs_bucket.py b/litellm/integrations/gcs_bucket/gcs_bucket.py index 5bc7247e02a..99cd8b31b63 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket.py @@ -3,6 +3,7 @@ import hashlib import json import os import time +from collections.abc import Mapping from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, Final @@ -15,6 +16,7 @@ from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase from litellm.litellm_core_utils.cloud_storage_security import ( sanitize_cloud_object_component, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus from litellm.types.integrations.gcs_bucket import * from litellm.types.utils import StandardBatchLineItemsPayload, StandardLoggingPayload @@ -134,7 +136,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): """ return f"{date_str}/batch-{batch_id}.ndjson" - def _get_config_key(self, kwargs: dict[str, Any]) -> str: + def _get_config_key(self, kwargs: Mapping[str, object]) -> str: """ Extract a synchronous grouping key from kwargs to group items by GCS config. This allows us to batch items with the same bucket/credentials together. @@ -143,7 +145,9 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): This key may contain sensitive information (bucket names, paths) - use _sanitize_config_key() for logging purposes. """ - standard_callback_dynamic_params: Final = kwargs.get("standard_callback_dynamic_params", None) or {} + standard_callback_dynamic_params: Final = ( + as_str_mapping(kwargs.get("standard_callback_dynamic_params", None)) or {} + ) bucket_name = standard_callback_dynamic_params.get("gcs_bucket_name", None) or self.BUCKET_NAME or "default" path_service_account: Final = ( diff --git a/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py b/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py index 9dfd75e5559..3f8cf93e11c 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py @@ -9,6 +9,7 @@ Usage: """ import asyncio +import os from collections.abc import AsyncIterable, Iterable from typing import Final @@ -40,7 +41,7 @@ _mocks_initialized = False # Default mock latency in seconds (simulates network round-trip) # Typical GCS API calls take 100-300ms for uploads, 50-150ms for GET/DELETE -_MOCK_LATENCY_SECONDS: Final = float(__import__("os").getenv("GCS_MOCK_LATENCY_MS", "150")) / 1000.0 +_MOCK_LATENCY_SECONDS: Final = float(os.getenv("GCS_MOCK_LATENCY_MS", "150")) / 1000.0 async def _mock_async_handler_get(self, url, params=None, headers=None, follow_redirects=None): diff --git a/litellm/integrations/gcs_pubsub/pub_sub.py b/litellm/integrations/gcs_pubsub/pub_sub.py index c799245c215..cbf6898d978 100644 --- a/litellm/integrations/gcs_pubsub/pub_sub.py +++ b/litellm/integrations/gcs_pubsub/pub_sub.py @@ -12,6 +12,8 @@ import os import traceback from typing import TYPE_CHECKING, Any, Final +from pydantic import TypeAdapter + from litellm.types.utils import StandardLoggingPayload if TYPE_CHECKING: @@ -27,6 +29,8 @@ from litellm.llms.custom_httpx.http_handler import ( httpxSpecialProvider, ) +_DECODED_JSON: Final = TypeAdapter(object) + class GcsPubSubLogger(CustomBatchLogger): def __init__( @@ -152,7 +156,7 @@ class GcsPubSubLogger(CustomBatchLogger): finally: self.log_queue.clear() - async def publish_message(self, message: SpendLogsPayload | StandardLoggingPayload) -> dict[str, Any] | None: + async def publish_message(self, message: SpendLogsPayload | StandardLoggingPayload) -> object: """ Publish message to Google Cloud Pub/Sub using REST API @@ -188,7 +192,7 @@ class GcsPubSubLogger(CustomBatchLogger): raise Exception(f"Failed to publish message: {response.text}") verbose_logger.debug("Pub/Sub response: %s", response.text) - return response.json() + return _DECODED_JSON.validate_python(response.json()) except Exception as e: verbose_logger.error("Pub/Sub publish error: %s", str(e)) diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 86f80047e52..b8c6e91d0d7 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -20,6 +20,7 @@ from litellm.integrations.langfuse.langfuse_mock_client import ( should_use_langfuse_mock, ) from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, filter_exceptions_from_params, reconstruct_model_name, safe_deep_copy, @@ -1268,11 +1269,11 @@ def log_provider_specific_information_as_span( *, tracing: "LangfuseTracing", parent: "LangfuseObservation", - enrichments: Mapping[str, Any], + enrichments: Mapping[str, object], ): """Logs provider-specific information as spans under the generation.""" - _hidden_params: Final[Mapping[str, object] | None] = enrichments.get("hidden_params", None) + _hidden_params: Final = as_str_mapping(enrichments.get("hidden_params", None)) if _hidden_params is None: return diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index f195cd97e3c..9364a3d4216 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -12,6 +12,7 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import ( serialize_langfuse_datetime, ) from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.types.integrations.langfuse_otel import ( LangfuseSpanAttributes, @@ -222,7 +223,7 @@ class LangfuseOtelLogger(OpenTelemetry): ) @staticmethod - def _set_langfuse_specific_attributes(span: Span, kwargs: Mapping[str, Any], response_obj): + def _set_langfuse_specific_attributes(span: Span, kwargs: Mapping[str, object], response_obj): """ Sets Langfuse specific metadata attributes onto the OTEL span. @@ -234,7 +235,7 @@ class LangfuseOtelLogger(OpenTelemetry): from litellm.integrations.arize._utils import safe_set_attribute from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - dynamic_params: Final = kwargs.get("standard_callback_dynamic_params") + dynamic_params: Final = as_str_mapping(kwargs.get("standard_callback_dynamic_params")) langfuse_environment: Final = ( dynamic_params.get("langfuse_environment") if dynamic_params else None ) or os.environ.get("LANGFUSE_TRACING_ENVIRONMENT") diff --git a/litellm/integrations/langfuse/langfuse_prompt_management.py b/litellm/integrations/langfuse/langfuse_prompt_management.py index 3786087ba91..50ff7002068 100644 --- a/litellm/integrations/langfuse/langfuse_prompt_management.py +++ b/litellm/integrations/langfuse/langfuse_prompt_management.py @@ -2,6 +2,7 @@ Call Hook for LiteLLM Proxy which allows Langfuse prompt management. """ +from collections.abc import Mapping from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast @@ -110,6 +111,10 @@ def _remember_trace_id(litellm_call_id: object, logged: LangfuseLoggedEvent) -> in_memory_trace_id_cache.set_cache(litellm_call_id=litellm_call_id, service_name="langfuse", trace_id=trace_id) +def _params_without_model(config: Mapping[str, object]) -> Mapping[str, object]: + return {k: v for k, v in config.items() if k != "model"} + + class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogger): def __init__( self, @@ -182,12 +187,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge return compiled_prompt def _get_optional_params_from_langfuse(self, langfuse_prompt_client: PROMPT_CLIENT) -> dict: - config: Final = langfuse_prompt_client.config - optional_params: Final = {} - for k, v in config.items(): - if k != "model": - optional_params[k] = v - return optional_params + return dict(_params_without_model(langfuse_prompt_client.config)) async def async_get_chat_completion_prompt( self, diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 01812149498..ff1f3568685 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -7,10 +7,11 @@ import random import traceback import types from datetime import datetime, timezone -from typing import Any, Final +from typing import Final import httpx from pydantic import BaseModel, TypeAdapter +from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -31,6 +32,10 @@ from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPa _RUN_RESPONSE: Final = TypeAdapter(object) +class _DynamicParamsKwargs(TypedDict, total=False): + standard_callback_dynamic_params: ReadOnly[StandardCallbackDynamicParams | None] + + def is_serializable(value): non_serializable_types: Final = ( types.CoroutineType, @@ -474,7 +479,7 @@ class LangsmithLogger(CustomBatchLogger): return log_queue_by_credentials - def _get_sampling_rate_to_use_for_request(self, kwargs: dict[str, Any]) -> float: + def _get_sampling_rate_to_use_for_request(self, kwargs: _DynamicParamsKwargs) -> float: standard_callback_dynamic_params: Final[StandardCallbackDynamicParams | None] = kwargs.get( "standard_callback_dynamic_params", None ) @@ -485,7 +490,7 @@ class LangsmithLogger(CustomBatchLogger): sampling_rate = float(_sampling_rate) return sampling_rate - def _get_credentials_to_use_for_request(self, kwargs: dict[str, Any]) -> LangsmithCredentialsObject: + def _get_credentials_to_use_for_request(self, kwargs: _DynamicParamsKwargs) -> LangsmithCredentialsObject: """ Handles key/team based logging diff --git a/litellm/integrations/langtrace.py b/litellm/integrations/langtrace.py index eeff76e7063..ea411baee7e 100644 --- a/litellm/integrations/langtrace.py +++ b/litellm/integrations/langtrace.py @@ -6,7 +6,7 @@ from litellm.types.integrations.otel_span_attributes import SpanAttributes if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - Span = _Span | Any + Span = _Span else: Span = Any diff --git a/litellm/integrations/levo/levo.py b/litellm/integrations/levo/levo.py index 4d2b4edf3cd..6273f39e09c 100644 --- a/litellm/integrations/levo/levo.py +++ b/litellm/integrations/levo/levo.py @@ -13,7 +13,7 @@ if TYPE_CHECKING: Protocol = _Protocol OpenTelemetryConfig = _OpenTelemetryConfig - Span = _Span | Any + Span = _Span else: Protocol = Any OpenTelemetryConfig = Any diff --git a/litellm/integrations/logfire_logger.py b/litellm/integrations/logfire_logger.py index 848369a1cbb..862893fa883 100644 --- a/litellm/integrations/logfire_logger.py +++ b/litellm/integrations/logfire_logger.py @@ -3,8 +3,9 @@ import os import traceback +from collections.abc import Mapping from enum import Enum -from typing import Any, Final, NamedTuple +from typing import Final, NamedTuple from typing_extensions import LiteralString @@ -15,7 +16,7 @@ from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info class SpanConfig(NamedTuple): message_template: LiteralString - span_data: dict[str, Any] + span_data: Mapping[str, object] class LogfireLevel(str, Enum): diff --git a/litellm/integrations/newrelic/newrelic.py b/litellm/integrations/newrelic/newrelic.py index 9829ef4e18f..3623335f599 100644 --- a/litellm/integrations/newrelic/newrelic.py +++ b/litellm/integrations/newrelic/newrelic.py @@ -49,11 +49,12 @@ import time import uuid from collections.abc import Mapping, Sequence from datetime import datetime -from typing import Any, Final +from typing import Final, Protocol import litellm from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import has_attributes from litellm.litellm_core_utils.redact_messages import should_redact_message_logging from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus from litellm.types.integrations.newrelic import NewRelicInitParams @@ -65,6 +66,10 @@ except ImportError: _newrelic_agent = None +class _SupportsTimestamp(Protocol): + def timestamp(self) -> float: ... + + class NewRelicLogger(CustomLogger): """ New Relic logger for LiteLLM to send AI monitoring events. @@ -126,13 +131,13 @@ class NewRelicLogger(CustomLogger): verbose_logger.error("Failed to initialize New Relic agent: %s. Integration will be disabled.", e) self.enabled = False - def _get_newrelic_params(self) -> dict: + def _get_newrelic_params(self) -> Mapping[str, object]: """ Get the newrelic_params from litellm.newrelic_params These are params specific to initializing the NewRelicLogger e.g. turn_off_message_logging """ - dict_newrelic_params: dict = {} + dict_newrelic_params: Mapping[str, object] = {} if litellm.newrelic_params is not None: if isinstance(litellm.newrelic_params, NewRelicInitParams): dict_newrelic_params = litellm.newrelic_params.model_dump() @@ -401,9 +406,9 @@ class NewRelicLogger(CustomLogger): return choices[0].get("finish_reason") or "unknown" return "unknown" - def _to_epoch_ms(self, t: Any) -> float: + def _to_epoch_ms(self, t: datetime | float) -> float: """Convert a datetime or float timestamp to epoch milliseconds.""" - if hasattr(t, "timestamp"): + if has_attributes(t, _SupportsTimestamp, "timestamp"): return t.timestamp() * 1000.0 return float(t) * 1000.0 @@ -518,13 +523,13 @@ class NewRelicLogger(CustomLogger): sequence = 0 # Extract timestamps, preferring StandardLoggingPayload - start_time = None + start_time: datetime | float | None = None if standard_logging_object: start_time = standard_logging_object.get("startTime") if not start_time: start_time = kwargs.get("start_time") - end_time = None + end_time: datetime | float | None = None if standard_logging_object: end_time = standard_logging_object.get("endTime") if not end_time: diff --git a/litellm/integrations/newrelic/newrelic_metrics.py b/litellm/integrations/newrelic/newrelic_metrics.py index a2cedeba0f5..641d1d86188 100644 --- a/litellm/integrations/newrelic/newrelic_metrics.py +++ b/litellm/integrations/newrelic/newrelic_metrics.py @@ -254,7 +254,7 @@ class NewRelicMetricsLogger(CustomBatchLogger): if not self.log_queue: return if _pass < NEWRELIC_METRICS_MAX_DRAIN_PASSES - 1: - await asyncio.sleep(2**_pass) + await asyncio.sleep(1 << _pass) async with self.flush_lock: tried_ids: Final = frozenset(id(record) for record in attempted) survivors: Final = tuple(record for record in self.log_queue if id(record) not in tried_ids) diff --git a/litellm/integrations/opik/opik_payload_builder/payload_builders.py b/litellm/integrations/opik/opik_payload_builder/payload_builders.py index 3aaf5bfc162..2909ab9d63c 100644 --- a/litellm/integrations/opik/opik_payload_builder/payload_builders.py +++ b/litellm/integrations/opik/opik_payload_builder/payload_builders.py @@ -57,9 +57,7 @@ def build_span_payload( span_id: Final = utils.create_uuid7() model: Final[str] = response_obj.get("model", "unknown-model") - obj_type: Final[str] = response_obj.get("object", "unknown-object") - created: Final[int] = response_obj.get("created", 0) - span_name: Final = f"{model}_{obj_type}_{created}" + span_name: Final = f"{model}_{response_obj.get('object', 'unknown-object')}_{response_obj.get('created', 0)}" _logging.verbose_logger.debug("OpikLogger creating span with id %s for trace %s", span_id, trace_id) diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index cca12b885c1..dd150082252 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -130,7 +130,8 @@ class _EnvWithoutBareExcludedServices(PydanticBaseSettingsSource): return self._env_settings.get_field_value(field, field_name) def __call__(self) -> dict[str, object]: - return {key: value for key, value in self._env_settings().items() if key != "excluded_services"} + settings: Final = self._env_settings() + return {key: settings[key] for key in settings if key != "excluded_services"} class OpenTelemetryV2Config(BaseSettings): diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index 5d37894bb8f..a69c334a1fb 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -332,7 +332,7 @@ def caller_session_id(kwargs: Mapping[str, object], trace: TraceControls) -> str return (None if echoes_trace_id else explicit) or trace.session_id or None -def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None: +def time_to_first_chunk_seconds(kwargs: Mapping[str, object]) -> float | None: """Seconds from the upstream request being issued (``api_call_start_time``) to the first streamed chunk (``completion_start_time``); ``None`` for non-streaming calls, where ``completion_start_time`` is backfilled with the @@ -340,8 +340,8 @@ def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None: optional_params: Final = cast(Mapping[str, object], kwargs.get("optional_params") or {}) if not optional_params.get("stream"): return None - api_call_start: Final = to_seconds(kwargs.get("api_call_start_time")) - completion_start: Final = to_seconds(kwargs.get("completion_start_time")) + api_call_start: Final = _epoch_seconds(kwargs.get("api_call_start_time")) + completion_start: Final = _epoch_seconds(kwargs.get("completion_start_time")) if api_call_start is None or completion_start is None: return None return completion_start - api_call_start diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index d11505ce026..9c62c45e489 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -7,7 +7,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from enum import Enum from types import MappingProxyType -from typing import TYPE_CHECKING, ClassVar, Final, Literal, cast +from typing import TYPE_CHECKING, ClassVar, Final, Literal, Protocol, cast from urllib.parse import urlsplit from typing_extensions import ReadOnly, TypedDict @@ -716,6 +716,15 @@ def _json_or_none(value: object) -> str | None: return None +class _EnumValue(Protocol): + @property + def value(self) -> object: ... + + +def _enum_value(item: _EnumValue) -> object: + return item.value + + def _guardrail_mode_str(value: object) -> str | None: """Normalize ``guardrail_mode`` to a stable string. @@ -732,12 +741,12 @@ def _guardrail_mode_str(value: object) -> str | None: for item in value: if item is None: continue - part = as_str(item.value) if isinstance(item, Enum) else as_str(item) + part = as_str(_enum_value(item)) if isinstance(item, Enum) else as_str(item) if part: parts.append(part) return ",".join(parts) or None if isinstance(value, Enum): - return as_str(value.value) + return as_str(_enum_value(value)) return as_str(value) diff --git a/litellm/integrations/otel/plumbing/metrics.py b/litellm/integrations/otel/plumbing/metrics.py index 7ec3711ee65..72d51471ba2 100644 --- a/litellm/integrations/otel/plumbing/metrics.py +++ b/litellm/integrations/otel/plumbing/metrics.py @@ -173,6 +173,7 @@ class _MetricKwargs(TypedDict, total=False): response_cost: ReadOnly[float | None] completion_start_time: ReadOnly[datetime | float | str | None] api_call_start_time: ReadOnly[datetime | float | str | None] + end_time: ReadOnly[datetime | float | str | None] def resolve_error_type(kwargs: Mapping[str, Any]) -> str: @@ -414,15 +415,12 @@ class GenAIMetricRecorder: if generation_time > 0: self._metrics.time_per_output_token.record(generation_time / completion_tokens, attributes=common_attrs) - def _record_response_duration(self, kwargs: Mapping[str, Any], end_time: datetime, common_attrs: dict) -> None: + def _record_response_duration(self, kwargs: _MetricKwargs, end_time: datetime, common_attrs: dict) -> None: api_call_start_time: Final = kwargs.get("api_call_start_time") if api_call_start_time is None: return - _end_time = kwargs.get("end_time") or end_time - if _end_time is None: - _end_time = datetime.now() api_call_start: Final = to_seconds(api_call_start_time) - end_ts: Final = to_seconds(_end_time) + end_ts: Final = to_seconds(kwargs.get("end_time") or end_time) if api_call_start is None or end_ts is None: return duration: Final = end_ts - api_call_start diff --git a/litellm/integrations/sqs.py b/litellm/integrations/sqs.py index d787375ca3c..83f85232182 100644 --- a/litellm/integrations/sqs.py +++ b/litellm/integrations/sqs.py @@ -13,6 +13,8 @@ import re import traceback from typing import Final +from pydantic import TypeAdapter + import litellm from litellm._logging import print_verbose, verbose_logger from litellm.constants import ( @@ -32,6 +34,7 @@ from litellm.types.utils import StandardLoggingPayload from .custom_batch_logger import CustomBatchLogger +_DECODED_JSON: Final = TypeAdapter(object) _BASE64_INLINE_PATTERN: Final = re.compile( r"data:(?:application|image|audio|video)/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=\s]+", re.MULTILINE, @@ -270,7 +273,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): if self.sqs_queue_url is None: raise ValueError("sqs_queue_url not set") - json_data: Final = json.loads(safe_dumps(payload)) + json_data: Final = _DECODED_JSON.validate_python(json.loads(safe_dumps(payload))) if self.app_crypto: aad_bytes: Final = self.sqs_app_encryption_aad.encode("utf-8") if self.sqs_app_encryption_aad else None encrypted: Final = self.app_crypto.encrypt_json(json_data, aad=aad_bytes) diff --git a/litellm/interactions/litellm_responses_transformation/handler.py b/litellm/interactions/litellm_responses_transformation/handler.py index 8fee0fcd1b5..d84f0b5ffbd 100644 --- a/litellm/interactions/litellm_responses_transformation/handler.py +++ b/litellm/interactions/litellm_responses_transformation/handler.py @@ -116,10 +116,7 @@ class LiteLLMResponsesInteractionsHandler: """Async handler for interactions API requests.""" # Call litellm.aresponses() # Note: litellm.aresponses() returns Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator] - aresponses_fn: Final[ - Callable[..., Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]] - ] = vars(litellm)["aresponses"] - responses_response: Final = await aresponses_fn( + responses_response: Final = await litellm.aresponses( **responses_request, ) diff --git a/litellm/litellm_core_utils/app_crypto.py b/litellm/litellm_core_utils/app_crypto.py index 4ef3f34f3b8..da844620dce 100644 --- a/litellm/litellm_core_utils/app_crypto.py +++ b/litellm/litellm_core_utils/app_crypto.py @@ -4,6 +4,9 @@ import os from typing import Final from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from pydantic import TypeAdapter + +_DECODED_JSON: Final = TypeAdapter(object) class AppCrypto: @@ -12,7 +15,7 @@ class AppCrypto: raise ValueError("Master key must be 32 bytes for AES-256-GCM") self.key = master_key - def encrypt_json(self, data: dict, aad: bytes | None = None) -> dict: + def encrypt_json(self, data: object, aad: bytes | None = None) -> dict: aes: Final = AESGCM(self.key) nonce: Final = os.urandom(12) plaintext: Final = json.dumps(data).encode("utf-8") @@ -24,10 +27,10 @@ class AppCrypto: "tag": base64.b64encode(tag).decode(), } - def decrypt_json(self, enc: dict, aad: bytes | None = None) -> dict: + def decrypt_json(self, enc: dict, aad: bytes | None = None) -> object: aes: Final = AESGCM(self.key) nonce: Final = base64.b64decode(enc["nonce"]) ct: Final = base64.b64decode(enc["ciphertext"]) tag: Final = base64.b64decode(enc["tag"]) data: Final = aes.decrypt(nonce, ct + tag, aad) - return json.loads(data.decode()) + return _DECODED_JSON.validate_python(json.loads(data.decode())) diff --git a/litellm/litellm_core_utils/cloud_storage_security.py b/litellm/litellm_core_utils/cloud_storage_security.py index e9689bd7d8a..8e949694881 100644 --- a/litellm/litellm_core_utils/cloud_storage_security.py +++ b/litellm/litellm_core_utils/cloud_storage_security.py @@ -3,10 +3,11 @@ import re from collections.abc import Mapping, Sequence from itertools import accumulate, repeat from types import MappingProxyType -from typing import Any, Final, cast +from typing import Final from urllib.parse import quote, unquote from litellm._uuid import uuid +from litellm.litellm_core_utils.core_helpers import as_str_mapping VERTEX_AI_MANAGED_GCS_PREFIX: Final = "litellm-vertex-files/" BEDROCK_MANAGED_S3_BATCH_PREFIX: Final = "litellm-bedrock-files-" @@ -141,7 +142,8 @@ def should_allow_legacy_cloud_file_ids( if isinstance(litellm_params, Mapping): trusted_model_credentials: Final = litellm_params.get("_litellm_internal_model_credentials") if isinstance(trusted_model_credentials, _MAPPING_PROXY_TYPE): - value = cast(Mapping[str, Any], trusted_model_credentials).get("allow_legacy_cloud_file_ids") + trusted_mapping: Final = as_str_mapping(trusted_model_credentials) + value = trusted_mapping.get("allow_legacy_cloud_file_ids") if trusted_mapping is not None else None if isinstance(value, bool): return value diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 1d61ef8519a..4be5cabac04 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -5,10 +5,11 @@ import logging import re from collections.abc import Collection, Iterable, Iterator, Mapping from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_checkable +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable import httpx from pydantic import TypeAdapter, ValidationError +from typing_extensions import TypeIs # noqa: TID251 # narrows a duck-typed value to the Protocol its caller reads import litellm from litellm._logging import verbose_logger @@ -58,6 +59,13 @@ def as_str_mapping(value: object) -> Mapping[str, object] | None: return value if isinstance(value, Mapping) else None +def decoded_json_object(value: object) -> Mapping[str, object]: + mapping: Final = as_str_mapping(value) + if mapping is not None: + return mapping + raise AttributeError(f"{type(value).__name__!r} object has no attribute 'get'", name="get", obj=value) + + @runtime_checkable class _SupportsIter(Protocol): def __iter__(self) -> Iterator[object]: ... @@ -72,6 +80,24 @@ def attribute_or_none(source: object, name: str) -> object: return getattr(source, name, None) +_Shape: Final = TypeVar("_Shape") + + +def has_attributes( + value: object, + shape: type[_Shape], + *names: str, +) -> TypeIs[_Shape]: # guard-ok: hasattr sees what __getattr__ serves, which Protocol isinstance skips on 3.12+ + return all(hasattr(value, name) for name in names) + + +def require_attributes(value: object, shape: type[_Shape], *names: str) -> _Shape: + if has_attributes(value, shape, *names): + return value + missing: Final = next(name for name in names if not hasattr(value, name)) + raise AttributeError(f"'{type(value).__name__}' object has no attribute '{missing}'", name=missing, obj=value) + + def iter_items(value: object) -> Iterable[object]: if isinstance(value, _SupportsIter): return value @@ -634,11 +660,14 @@ def preserve_upstream_non_openai_attributes( for key in candidate_keys ) if all(value is None or isinstance(value, (str, int, float, bool)) for _, value in candidate_values): - for key, value in candidate_values: - setattr(model_response, key, value) + _assign_attributes(model_response, candidate_values) return - for key, value in original_chunk.model_dump(include=frozenset(candidate_keys)).items(): - setattr(model_response, key, value) + _assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items()) + + +def _assign_attributes(target: object, values: Iterable[tuple[str, object]]) -> None: + for key, value in values: + setattr(target, key, value) def safe_deep_copy(data): @@ -764,6 +793,10 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any: Filtered data structure with Exception and callable objects removed, or None if the entire input was an Exception or callable """ + return _filter_exceptions_from_params(data, max_depth) + + +def _filter_exceptions_from_params(data: object, max_depth: int) -> object: if max_depth <= 0: return data @@ -785,7 +818,7 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any: if isinstance(v, Exception) or (callable(v) and not isinstance(v, type)): continue try: - filtered = filter_exceptions_from_params(v, max_depth - 1) + filtered = _filter_exceptions_from_params(v, max_depth - 1) if filtered is not None: result[k] = filtered except Exception: @@ -799,7 +832,7 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any: if isinstance(item, Exception) or (callable(item) and not isinstance(item, type)): continue try: - filtered = filter_exceptions_from_params(item, max_depth - 1) + filtered = _filter_exceptions_from_params(item, max_depth - 1) if filtered is not None: result_list.append(filtered) except Exception: diff --git a/litellm/litellm_core_utils/cost_discount.py b/litellm/litellm_core_utils/cost_discount.py index a64ad0b20f7..bfcee7928fb 100644 --- a/litellm/litellm_core_utils/cost_discount.py +++ b/litellm/litellm_core_utils/cost_discount.py @@ -23,7 +23,7 @@ def parse_cost_discount_key(key: str) -> CostDiscountKey: def _literal_length(pattern: str) -> int: - return sum(1 for token in _GLOB_TOKEN.findall(pattern) if len(token) == 1 and token not in "*?") + return sum(1 for match in _GLOB_TOKEN.finditer(pattern) if len(match.group()) == 1 and match.group() not in "*?") def _model_names(model: str, provider_prefix: str) -> tuple[str, ...]: diff --git a/litellm/litellm_core_utils/default_encoding.py b/litellm/litellm_core_utils/default_encoding.py index c3b6a008411..970dd512a40 100644 --- a/litellm/litellm_core_utils/default_encoding.py +++ b/litellm/litellm_core_utils/default_encoding.py @@ -48,5 +48,5 @@ for attempt in range(_max_retries): # Last attempt, re-raise the exception raise # Exponential backoff with jitter to reduce collision probability - delay = _retry_delay * (2**attempt) + random.uniform(0, 0.1) + delay = _retry_delay * (2.0**attempt) + random.uniform(0, 0.1) time.sleep(delay) diff --git a/litellm/litellm_core_utils/dot_notation_indexing.py b/litellm/litellm_core_utils/dot_notation_indexing.py index 1dac67ecbf6..09720fa413c 100644 --- a/litellm/litellm_core_utils/dot_notation_indexing.py +++ b/litellm/litellm_core_utils/dot_notation_indexing.py @@ -181,7 +181,7 @@ def _delete_nested_value_custom( def delete_nested_value( - data: dict[str, Any], + data: dict[str, object], path: str, depth: int = 0, max_depth: int = 20, diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index e8da59cd33a..1c816b22ef1 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -16,7 +16,7 @@ from litellm.litellm_core_utils.bug_report import ( build_bug_report, should_report_bug, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string from litellm.types.utils import LlmProviders @@ -169,7 +169,7 @@ def get_error_message(error_obj) -> str | None: return None if hasattr(error_obj, "body"): - _error_obj_body: Final = getattr(error_obj, "body") + _error_obj_body: Final = attribute_or_none(error_obj, "body") if isinstance(_error_obj_body, dict): # OpenAI-style: {"message": "...", "type": "...", ...} if _error_obj_body.get("message"): diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 1e2a511d439..7fc6eab6c60 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -43,6 +43,7 @@ from litellm.litellm_core_utils.offline_mode import is_offline_mode FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations" _CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) +_DECODED_JSON: Final = TypeAdapter(object) _BUNDLED_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) _CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"}) @@ -314,7 +315,7 @@ def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) - def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome: try: - parsed: Final[object] = json.loads(content) + parsed: Final = _DECODED_JSON.validate_python(json.loads(content)) except ValueError as e: return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}") if not isinstance(parsed, dict): @@ -383,7 +384,8 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp ) if response.is_error: return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}") - return _classify_fetch_content(response.content, url, response.headers.get("etag")) + etag_values: Final = response.headers.get_list("etag") + return _classify_fetch_content(response.content, url, ", ".join(etag_values) if etag_values else None) def _next_retry_wait( diff --git a/litellm/litellm_core_utils/hidden_params.py b/litellm/litellm_core_utils/hidden_params.py index f8d44d44cdc..a74337d2549 100644 --- a/litellm/litellm_core_utils/hidden_params.py +++ b/litellm/litellm_core_utils/hidden_params.py @@ -1,4 +1,5 @@ from collections.abc import Iterator, Mapping, MutableMapping +from types import MappingProxyType from typing import Final, Protocol, cast from pydantic import BaseModel @@ -52,7 +53,7 @@ class HiddenParamsModelView(MutableMapping[str, object]): def stored_attribute(obj: object, name: str) -> object: if not isinstance(obj, BaseModel): return getattr(obj, name, None) - values: Final[Mapping[str, object]] = vars(obj) + values: Final[Mapping[str, object]] = MappingProxyType[str, object](vars(obj)) if name in values: return values[name] extras: Final[Mapping[str, object]] = obj.__pydantic_extra__ or {} diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index 5a533054f0b..c3769686e45 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -28,7 +28,7 @@ def inherit_message_logging_privacy(disabled: bool) -> Generator[None]: def iter_client_callback_metadata_dicts( - kwargs: dict[str, Any], + kwargs: Mapping[str, object], ) -> Iterator[tuple[str, dict[str, Any]]]: litellm_params: Final = kwargs.get("litellm_params") if isinstance(litellm_params, dict): diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 41f13817d75..fc8d1ecc8c1 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -556,6 +556,7 @@ def _resolve_mantle_region_for_cost( _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"}) _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object]) +_REDACTED_RESULT: Final = TypeAdapter(object) def _provider_response_id(source: object) -> str | None: @@ -1171,7 +1172,7 @@ class Logging(LiteLLMLoggingBaseClass): automatically extracts metadata/litellm_metadata from kwargs, so callers don't need to manually plumb them into litellm_params. """ - base_litellm_params: Final[dict[str, Any]] = {} + base_litellm_params: Final[dict[str, object]] = {} if isinstance(kwargs.get("metadata"), dict): base_litellm_params["metadata"] = kwargs["metadata"].copy() @@ -1188,10 +1189,11 @@ class Logging(LiteLLMLoggingBaseClass): lp_metadata: Final = litellm_params.pop("metadata", None) base_litellm_params.update(litellm_params) if lp_metadata and isinstance(lp_metadata, dict): - base_litellm_params.setdefault("metadata", {}) - for k, v in lp_metadata.items(): - if k not in base_litellm_params["metadata"]: - base_litellm_params["metadata"][k] = v + merged_metadata: Final = base_litellm_params.setdefault("metadata", {}) + if isinstance(merged_metadata, dict): + for k, v in lp_metadata.items(): + if k not in merged_metadata: + merged_metadata[k] = v self.update_environment_variables( litellm_params=base_litellm_params, @@ -1793,7 +1795,7 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.exception( "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e ) - original_response = redact_message_input_output_from_logging( + redact_message_input_output_from_logging( model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), result=original_response, ) @@ -2023,10 +2025,9 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details.pop("response_cost_failure_debug_information", None) - if cache_hit is None: - cache_hit = self.model_call_details.get("cache_hit", False) + resolved_cache_hit: Final = self.model_call_details.get("cache_hit", False) if cache_hit is None else cache_hit - if cache_hit is True: + if resolved_cache_hit is True: return 0.0 if is_unbilled_non_inference_call( @@ -2078,7 +2079,7 @@ class Logging(LiteLLMLoggingBaseClass): model_value: Final = litellm_model_name or self.model cost_cache_hit_value: Final = ( - self.model_call_details.get("cache_hit", False) if cache_hit is None else cache_hit + self.model_call_details.get("cache_hit", False) if resolved_cache_hit is None else resolved_cache_hit ) cost_cache_hit: Final[bool | None] = cast( # cast-ok: callback metadata is caller-provided bool | None, cost_cache_hit_value @@ -2512,8 +2513,10 @@ class Logging(LiteLLMLoggingBaseClass): def carry_failed_attempt_usage(self) -> None: """Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its own usage while the request's failure log still bills every attempt.""" - usage: Final = self.model_call_details.pop("combined_usage_object", None) - response_cost: Final = self.model_call_details.pop("response_cost", None) + usage: Final = self._model_call_details_view.get("combined_usage_object") + response_cost: Final = self._model_call_details_view.get("response_cost") + self.model_call_details.pop("combined_usage_object", None) + self.model_call_details.pop("response_cost", None) if isinstance(usage, Usage): self.model_call_details["current_attempt_api_call_start_time"] = None self.model_call_details["failed_attempts_usage"] = ( @@ -3302,7 +3305,7 @@ class Logging(LiteLLMLoggingBaseClass): if callback_logging_payload is not None or shared_logging_payload is None else shared_logging_payload.copy() ) - call_details: dict[str, Any] # mutable-ok: legacy loggers take a plain dict + call_details: dict[str, object] # mutable-ok: legacy loggers take a plain dict call_details = ( # rebind-ok: logging hooks may return a new dict self.model_call_details if not for_async_request @@ -3323,9 +3326,11 @@ class Logging(LiteLLMLoggingBaseClass): ) ## REDACT MESSAGES ## - result = redact_message_input_output_from_logging( - model_call_details=call_details, - result=result, + result = _REDACTED_RESULT.validate_python( + redact_message_input_output_from_logging( + model_call_details=call_details, + result=result, + ) ) ## LOGGING HOOK ## hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks))) @@ -3591,7 +3596,7 @@ class Logging(LiteLLMLoggingBaseClass): print_verbose("S3Logger Logger: Got Stream Event - Completed Stream Response") s3Logger.log_event( kwargs=call_details, - response_obj=call_details["complete_streaming_response"], + response_obj=call_details_view["complete_streaming_response"], start_time=start_time, end_time=end_time, print_verbose=print_verbose, @@ -3914,9 +3919,11 @@ class Logging(LiteLLMLoggingBaseClass): self._get_completed_success_callbacks(asynchronous=True) if not self.stream else callbacks ) - result = redact_message_input_output_from_logging( - model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), - result=result, + result = _REDACTED_RESULT.validate_python( + redact_message_input_output_from_logging( + model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), + result=result, + ) ) ## LOGGING HOOK ## @@ -4261,9 +4268,11 @@ class Logging(LiteLLMLoggingBaseClass): result: object = None # result sent to all loggers, init this to None incase it's not created - result = redact_message_input_output_from_logging( - model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), - result=result, + result = _REDACTED_RESULT.validate_python( + redact_message_input_output_from_logging( + model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), + result=result, + ) ) self.mark_logging_complete(event_type="sync_failure") for callback in callbacks: @@ -4952,7 +4961,7 @@ class Logging(LiteLLMLoggingBaseClass): """ import httpx - httpx_response: Final = self.model_call_details.get("httpx_response", None) + httpx_response: Final[Response | None] = self.model_call_details.get("httpx_response", None) if httpx_response is None: raise ValueError("Google GenAI Generate Content: httpx_response is None") dict_result: Final = httpx_response.json() @@ -4982,7 +4991,7 @@ class Logging(LiteLLMLoggingBaseClass): The response object with usage added if available """ # Get usage from model_call_details (set by asend_message) - usage: Final = self.model_call_details.get("usage") + usage: Final[Usage | None] = self.model_call_details.get("usage") if usage is None: return result @@ -5815,7 +5824,9 @@ def build_custom_logger_compatible_class( return callback # Get global BitBucket config - bitbucket_config: Final = getattr(litellm, "global_bitbucket_config", None) + bitbucket_config: Final = ( + litellm.global_bitbucket_config if hasattr(litellm, "global_bitbucket_config") else None + ) if bitbucket_config is None: raise ValueError("BitBucket configuration not found. Please set litellm.global_bitbucket_config first.") diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 04510c20db0..045869e4547 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -450,7 +450,7 @@ class StandardBuiltInToolCostTracking: def _get_code_interpreter_cost( model_info: ModelInfo | None, custom_llm_provider: str | None, - standard_built_in_tools_params: StandardBuiltInToolsParams, + standard_built_in_tools_params: Mapping[str, object], ) -> float: """Calculate code interpreter cost.""" code_interpreter_sessions: Final = standard_built_in_tools_params.get("code_interpreter_sessions", None) diff --git a/litellm/litellm_core_utils/llm_judge.py b/litellm/litellm_core_utils/llm_judge.py index 0c2d7767455..307e000119b 100644 --- a/litellm/litellm_core_utils/llm_judge.py +++ b/litellm/litellm_core_utils/llm_judge.py @@ -21,6 +21,7 @@ if TYPE_CHECKING: JSON_FENCE_RE: Final = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE) _OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) def default_router_provider() -> Router | None: @@ -40,13 +41,13 @@ def parse_json_verdict(raw: str) -> dict[str, object]: # mutable-ok: plain pars text = fenced.group(1).strip() # rebind-ok: progressively narrowed to the JSON payload parsed: object try: - parsed = json.loads(text) + parsed = _DECODED_JSON.validate_python(json.loads(text)) except json.JSONDecodeError: start: Final = text.find("{") end: Final = text.rfind("}") if start == -1 or end <= start: raise - parsed = json.loads(text[start : end + 1]) + parsed = _DECODED_JSON.validate_python(json.loads(text[start : end + 1])) if not isinstance(parsed, dict): raise ValueError("judge response is not a JSON object") return {str(k): v for k, v in parsed.items()} diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index 307353d081b..56c6436eae6 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -1,5 +1,5 @@ import threading -from collections.abc import Callable, Iterable, Iterator +from collections.abc import Callable, Iterable, Iterator, Mapping from types import MethodType from typing import TYPE_CHECKING, Final @@ -20,6 +20,10 @@ _generic_api_logger_cache: Final[dict[str, GenericAPILogger]] = {} _callback_lists_lock: Final = threading.RLock() +def _instance_attributes(target: object) -> Mapping[str, object]: + return vars(target) + + class LoggingCallbackManager: """ A centralized class that allows easy add / remove callbacks for litellm. @@ -341,7 +345,7 @@ class LoggingCallbackManager: key_parts: Final = [type(custom_logger).__name__] # Add only fundamental type instance variables to the key - for attr_name, attr_value in vars(custom_logger).items(): + for attr_name, attr_value in _instance_attributes(custom_logger).items(): if not attr_name.startswith("_"): # Skip private attributes if isinstance(attr_value, (str, bool, int)): key_parts.append(f"{attr_name}={attr_value}") diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 60be4d44706..59882dcbf2f 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -20,6 +20,7 @@ from pydantic import TypeAdapter import litellm from litellm import verbose_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.router_utils.batch_utils import InMemoryFile from litellm.types.llms.openai import ( AllMessageValues, @@ -1527,7 +1528,7 @@ def _grammar_branches(node: Mapping[str, object]) -> int: if properties is None: return 0 optional: Final = len(_optional_property_names(node)) - return 2**optional * (len(properties) - optional + 1) + return (1 << optional) * (len(properties) - optional + 1) def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]: @@ -2839,7 +2840,7 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, object]]: break try: - obj, end_idx = decoder.raw_decode(raw, idx) + decoded = decoder.raw_decode(raw, idx) except json.JSONDecodeError as e: verbose_logger.warning( "split_concatenated_json_objects: discarding unparseable tool-call " @@ -2849,13 +2850,14 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, object]]: e, ) break - if isinstance(obj, dict): - results.append(obj) + obj = as_str_mapping(_DECODED_JSON.validate_python(decoded[0])) + if obj is not None: + results.append(dict(obj)) else: # Non-dict JSON value – wrap in empty dict (Bedrock requires # toolUse.input to be an object). results.append({}) - idx = end_idx + idx = decoded[1] return results @@ -2887,14 +2889,16 @@ def salvage_concatenated_tool_arguments(raw: str) -> tuple[dict[str, object], .. if idx >= length: break try: - obj, end_idx = decoder.raw_decode(stripped, idx) + decoded = decoder.raw_decode(stripped, idx) except json.JSONDecodeError: return () - if not isinstance(obj, dict): + decoded_object = as_str_mapping(_DECODED_JSON.validate_python(decoded[0])) + if decoded_object is None: return () - idx = end_idx + idx = decoded[1] if exceeded: continue + obj = dict(decoded_object) count += 1 if not kept: kept = (obj,) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 3d865bd1819..cfb6e3ab45f 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -21,6 +21,7 @@ import litellm.types.llms from litellm import verbose_logger from litellm._uuid import uuid from litellm.constants import REDACTED_BY_LITELLM +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( anthropic_system_messages, find_anthropic_server_tool_result, @@ -1021,11 +1022,12 @@ def convert_to_anthropic_tool_invoke_xml(tool_calls: list) -> str: tool_function = get_attribute_or_key(tool, "function") tool_name = get_attribute_or_key(tool_function, "name") tool_arguments = get_attribute_or_key(tool_function, "arguments") - parsed_args = parse_tool_call_arguments( - tool_arguments, tool_name=tool_name, context="Anthropic XML tool invoke" + parsed_args = _DECODED_JSON.validate_python( + parse_tool_call_arguments(tool_arguments, tool_name=tool_name, context="Anthropic XML tool invoke") ) - if isinstance(parsed_args, dict): - parameters = "".join(f"<{param}>{val}\n" for param, val in parsed_args.items()) + parsed_mapping = as_str_mapping(parsed_args) + if parsed_mapping is not None: + parameters = "".join(f"<{param}>{val}\n" for param, val in parsed_mapping.items()) else: parameters = f"{parsed_args}\n" invokes += f"\n{tool_name}\n\n{parameters}\n\n" @@ -2342,7 +2344,7 @@ def _add_missing_tool_results( else: tool_name = getattr(function, "name", "unknown_tool") else: - function = getattr(tool_call, "function", None) + function = attribute_or_none(tool_call, "function") if function: tool_name = getattr(function, "name", "unknown_tool") break @@ -3184,10 +3186,14 @@ def anthropic_messages_pt( def extract_between_tags(tag: str, string: str, strip: bool = False) -> list[str]: ext_list = re.findall(f"<{tag}>(.+?)", string, re.DOTALL) if strip: - ext_list = [e.strip() for e in ext_list] + ext_list = list(_stripped(ext_list)) return ext_list +def _stripped(values: Iterable[str]) -> Iterator[str]: + return (value.strip() for value in values) + + def contains_tag(tag: str, string: str) -> bool: return bool(re.search(f"<{tag}>(.+?)", string, re.DOTALL)) @@ -4061,13 +4067,15 @@ def _convert_to_bedrock_tool_call_invoke( arguments_dict = {} else: try: - arguments_dict = json.loads(arguments) + decoded_arguments = as_str_mapping(_DECODED_JSON.validate_python(json.loads(arguments))) # Ensure arguments_dict is always a dict # (Bedrock requires toolUse.input to be an object). # Some providers return arguments: '""' which # json.loads decodes to a bare string. - if not isinstance(arguments_dict, dict): + if decoded_arguments is None: arguments_dict = {} + else: + arguments_dict = dict(decoded_arguments) except json.JSONDecodeError: # The model may return multiple JSON objects # concatenated in a single arguments string, e.g. @@ -6028,7 +6036,11 @@ def _parse_tool_call_arguments(raw: object, tool_name: str | None, context: str) return ({},) normalized_raw: Final = "{}" if raw == REDACTED_BY_LITELLM else raw try: - parsed: Final = parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context) + parsed: Final = as_str_mapping( + _DECODED_JSON.validate_python( + parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context) + ) + ) except ValueError as e: salvaged: Final = salvage_concatenated_tool_arguments(normalized_raw) if salvaged: @@ -6041,7 +6053,7 @@ def _parse_tool_call_arguments(raw: object, tool_name: str | None, context: str) return salvaged verbose_logger.warning("Failed to parse tool call arguments: %s", e) return ({},) - return (parsed,) if isinstance(parsed, dict) else ({},) + return (dict(parsed),) if parsed is not None else ({},) def _choice_tool_calls(choice: object) -> tuple[object, ...]: diff --git a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py index ecf419695d0..eb3400aee8b 100644 --- a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py +++ b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py @@ -482,7 +482,7 @@ def _runs_by_anchor( ) -> Mapping[int, tuple[tuple[int, ...], ...]]: anchored: Final = tuple(sorted(((anchor, run) for run, anchor in anchors if anchor is not None), key=itemgetter(0))) return MappingProxyType( - {anchor: tuple(run for _, run in group) for anchor, group in groupby(anchored, key=itemgetter(0))} + {anchor: tuple(run for _, run in group) for anchor, group in groupby(anchored, key=lambda entry: entry[0])} ) diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 52758e617d8..8c402fe9b44 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping, Sequence +from collections.abc import ItemsView, Mapping, Sequence from collections.abc import Set as AbstractSet from dataclasses import dataclass, field from typing import Any, Final @@ -30,6 +30,10 @@ _DEFAULT_SENSITIVE_PATTERNS: Final = frozenset( ) +def _object_items(data: Mapping[str, object]) -> ItemsView[str, object]: + return data.items() + + class SensitiveDataMasker: def __init__( self, @@ -127,7 +131,7 @@ class SensitiveDataMasker: return data masked_data: Final[dict[str, object]] = {} - for k, v in data.items(): + for k, v in _object_items(data): try: key_is_sensitive = self.is_sensitive_key(k, excluded_keys) if isinstance(v, Mapping): diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 39f675099d7..50789406055 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -425,7 +425,7 @@ class ChunkProcessor: if hasattr(chunk["choices"][0], "finish_reason"): chunk_finish_reason = chunk["choices"][0].finish_reason elif "finish_reason" in chunk["choices"][0]: - chunk_finish_reason = chunk["choices"][0]["finish_reason"] + chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"]) if chunk_finish_reason is not None: finish_reason = chunk_finish_reason @@ -1226,6 +1226,7 @@ def concatenate_base64_list(base64_strings: Sequence[str]) -> str: _CHOICE_DICTS: Final = TypeAdapter(list[dict[str, object]]) +_CHOICE_FIELD_VALUE: Final = TypeAdapter(object) def _text_completion_chunk_from_mapping(chunk: Mapping[str, object]) -> TextCompletionResponse: diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index f994baa52b1..fdcdca151be 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -37,7 +37,6 @@ from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.llms.openai import OpenAIChatCompletionChunk from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( CacheCreationTokenDetails, @@ -111,14 +110,15 @@ def _prepend_upstream_choice_extras( choice.__pydantic_fields_set__.update(upstream_extras) -def _holds_only_scalars(model: BaseModel) -> bool: - fields: Final[Iterator[tuple[str, object]]] = iter(model) +def _holds_only_scalars(fields: Iterable[tuple[str, object]]) -> bool: return all(value is None or isinstance(value, (str, int, float, bool)) for _, value in fields) -def _delta_from_upstream(upstream_delta: BaseModel | None) -> Delta | None: +def _delta_from_upstream(upstream_delta: "BaseModel | _UpstreamDeltaLike | None") -> Delta | None: if upstream_delta is None: return Delta() + if not isinstance(upstream_delta, BaseModel): + return None try: return Delta(**dict(iter(upstream_delta))) except Exception: @@ -189,6 +189,33 @@ class _DeltaDumpDict(TypedDict): tool_calls: NotRequired[Sequence[Mapping[str, object]]] +class _UpstreamFunctionCallLike(Protocol): + arguments: str | None + + +class _UpstreamToolCallLike(Protocol): + function: _UpstreamFunctionCallLike + + +class _UpstreamDeltaLike(Protocol): + function_call: _UpstreamFunctionCallLike | None + tool_calls: Sequence[_UpstreamToolCallLike] | None + + def __iter__(self) -> Iterator[tuple[str, object]]: ... + + def model_dump(self) -> _DeltaDumpDict: ... + + +class _UpstreamChoiceLike(Protocol): + delta: _UpstreamDeltaLike | None + + +class _UpstreamChunkLike(Protocol): + id: str + choices: Sequence[_UpstreamChoiceLike] + system_fingerprint: str | None + + class _TextCompletionChoiceLike(Protocol): text: str finish_reason: str | None @@ -920,7 +947,7 @@ class CustomStreamWrapper: def copy_model_response_level_provider_specific_fields( self, - original_chunk: ModelResponseStream | OpenAIChatCompletionChunk, + original_chunk: object, model_response: ModelResponseStream, ) -> ModelResponseStream: """ @@ -949,8 +976,8 @@ class CustomStreamWrapper: or ("function_call" in completion_obj and completion_obj["function_call"] is not None) or ( "tool_calls" in model_response.choices[0].delta - and model_response.choices[0].delta["tool_calls"] is not None - and len(model_response.choices[0].delta["tool_calls"]) > 0 + and model_response.choices[0].delta.tool_calls is not None + and len(model_response.choices[0].delta.tool_calls) > 0 ) or ( "function_call" in model_response.choices[0].delta @@ -1115,7 +1142,8 @@ class CustomStreamWrapper: ## check if openai/azure chunk original_chunk: Final = response_obj.get("original_chunk", None) if original_chunk: - if len(original_chunk.choices) > 0: + upstream_chunk: Final[_UpstreamChunkLike] = original_chunk + if len(upstream_chunk.choices) > 0: original_chunk_for_reuse: Final[object] = cast( # cast-ok: runtime type-check before access object, original_chunk, @@ -1151,7 +1179,7 @@ class CustomStreamWrapper: ) else: choices: Final = [] - for choice in original_chunk.choices: + for choice in upstream_chunk.choices: try: if isinstance(choice, BaseModel): choice_json = choice.model_dump() @@ -1164,11 +1192,11 @@ class CustomStreamWrapper: setattr(model_response, "choices", choices) else: return - model_response.system_fingerprint = original_chunk.system_fingerprint + model_response.system_fingerprint = upstream_chunk.system_fingerprint setattr( model_response, "citations", - stored_attribute(original_chunk, "citations"), + stored_attribute(upstream_chunk, "citations"), ) preserve_upstream_non_openai_attributes( model_response=model_response, @@ -1611,7 +1639,9 @@ class CustomStreamWrapper: model_response.model = self.model ## FUNCTION CALL PARSING - original_chunk: Final = response_obj.get("original_chunk") if response_obj is not None else None + original_chunk: Final[_UpstreamChunkLike | None] = ( + response_obj.get("original_chunk") if response_obj is not None else None + ) if ( original_chunk is not None ): # function / tool calling branch - only set for openai/azure compatible endpoints @@ -1623,31 +1653,19 @@ class CustomStreamWrapper: original_chunk, model_response ) if original_chunk.choices and len(original_chunk.choices) > 0: - delta = original_chunk.choices[0].delta + delta: Final = original_chunk.choices[0].delta if delta is not None and (delta.function_call is not None or delta.tool_calls is not None): try: model_response.system_fingerprint = original_chunk.system_fingerprint ## AZURE - check if arguments is not None - if original_chunk.choices[0].delta.function_call is not None: - if ( - getattr( - original_chunk.choices[0].delta.function_call, - "arguments", - ) - is None - ): - original_chunk.choices[0].delta.function_call.arguments = "" - elif original_chunk.choices[0].delta.tool_calls is not None: - if isinstance(original_chunk.choices[0].delta.tool_calls, list): - for t in original_chunk.choices[0].delta.tool_calls: - if hasattr(t, "functions") and hasattr(t.functions, "arguments"): - if ( - getattr( - t.function, - "arguments", - ) - is None - ): + if delta.function_call is not None: + if delta.function_call.arguments is None: + delta.function_call.arguments = "" + elif delta.tool_calls is not None: + if isinstance(delta.tool_calls, list): + for t in delta.tool_calls: + if hasattr(t, "functions") and hasattr(getattr(t, "functions"), "arguments"): + if t.function.arguments is None: t.function.arguments = "" _json_delta: Final[_DeltaDumpDict] = delta.model_dump() if "role" not in _json_delta or _json_delta["role"] is None: @@ -1886,7 +1904,7 @@ class CustomStreamWrapper: else: asyncio.run(self.logging_obj.async_success_handler(processed_chunk, None, None, cache_hit)) ## SYNC LOGGING — only for sync SDK entrypoints; async proxy paths export via async_success_handler - litellm_params: Final = self.logging_obj.model_call_details.get("litellm_params", {}) + litellm_params: Final[Mapping[str, object]] = self.logging_obj.model_call_details.get("litellm_params", {}) if self.logging_obj.is_sync_litellm_request(litellm_params): self.logging_obj.success_handler(processed_chunk, None, None, cache_hit) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 8aded69aed2..90e79d4d853 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -44,7 +44,7 @@ from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEnc if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace from litellm.litellm_core_utils.url_utils import safe_get -from litellm.llms.custom_httpx.http_handler import get_httpx_client +from litellm.llms.custom_httpx.http_handler import get_httpx_client, header_value from litellm.rust_bridge.tokenizer import get_encoding from litellm.types.llms.anthropic import ( AnthropicContentParamSource, @@ -247,7 +247,7 @@ def get_image_dimensions( client: Final = get_httpx_client() response: Final[httpx.Response] = safe_get(client, data) max_bytes: Final = int(MAX_IMAGE_URL_DOWNLOAD_SIZE_MB * 1024 * 1024) - content_length: Final[str | None] = response.headers.get("Content-Length") + content_length: Final[str | None] = header_value(response.headers, "Content-Length") if content_length is not None and int(content_length) > max_bytes: pass # skip download; img_data stays None else: diff --git a/litellm/litellm_core_utils/url_utils.py b/litellm/litellm_core_utils/url_utils.py index 4a1eaa87d3a..0db6a41b503 100644 --- a/litellm/litellm_core_utils/url_utils.py +++ b/litellm/litellm_core_utils/url_utils.py @@ -22,11 +22,11 @@ Admins can opt out via two ``litellm`` globals (wired from proxy config): import asyncio import socket from ipaddress import ip_address, ip_network -from typing import Any, Final, Protocol +from typing import Final, Protocol from urllib.parse import quote, urlparse, urlsplit, urlunparse import httpx -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ReadOnly, TypedDict, Unpack import litellm @@ -50,6 +50,7 @@ class _UrlFetcher(Protocol): *, headers: dict[str, str] | None = None, follow_redirects: bool = False, + timeout: float | httpx.Timeout | None = None, ) -> httpx.Response: ... @@ -62,6 +63,8 @@ class _AsyncUrlFetcher(Protocol): *, headers: dict[str, str] | None = None, follow_redirects: bool = False, + timeout: float | httpx.Timeout | None = None, + max_response_bytes: int | None = None, ) -> httpx.Response: ... @@ -73,8 +76,14 @@ class _AsyncFetcherView(TypedDict): fetcher: ReadOnly[_AsyncUrlFetcher] -class _CallerHeadersView(TypedDict): +class _FetchOptions(TypedDict, total=False): headers: ReadOnly[dict[str, str]] + follow_redirects: ReadOnly[bool] + timeout: ReadOnly[float | httpx.Timeout | None] + + +class _AsyncFetchOptions(_FetchOptions, total=False): + max_response_bytes: ReadOnly[int | None] # Cloud-internal IPs that ``ip.is_global`` can report as public. Everything @@ -440,7 +449,7 @@ def _extract_redirect_url(response: httpx.Response, request_url: str) -> str: return str(httpx.URL(request_url).join(location)) -def safe_get(client: _UrlFetcher, url: str, **kwargs: Any) -> httpx.Response: +def safe_get(client: _UrlFetcher, url: str, **kwargs: Unpack[_FetchOptions]) -> httpx.Response: """ Fetch a user-supplied URL with SSRF protection on every redirect hop. @@ -460,21 +469,23 @@ def safe_get(client: _UrlFetcher, url: str, **kwargs: Any) -> httpx.Response: The final httpx.Response. """ if not getattr(litellm, "user_url_validation", True): - kwargs.setdefault("follow_redirects", True) - unvalidated: Final[_ResponseView] = {"response": client.get(url, **kwargs)} + unvalidated_options: Final[_FetchOptions] = { + **kwargs, + "follow_redirects": kwargs.get("follow_redirects", True), + } + unvalidated: Final[_ResponseView] = {"response": client.get(url, **unvalidated_options)} return unvalidated["response"] fetcher_view: Final[_FetcherView] = {"fetcher": client} fetcher: Final = fetcher_view["fetcher"] - kwargs.pop("follow_redirects", None) - headers_view: Final[_CallerHeadersView] = {"headers": kwargs.pop("headers", {})} + caller_headers: Final = kwargs.get("headers", {}) for _ in range(_MAX_REDIRECTS): validated_url, original_host = validate_url(url) - response = fetcher.get( - validated_url, - headers={**headers_view["headers"], "Host": original_host}, - follow_redirects=False, + hop_options: _FetchOptions = { **kwargs, - ) + "headers": {**caller_headers, "Host": original_host}, + "follow_redirects": False, + } + response = fetcher.get(validated_url, **hop_options) if not response.is_redirect: return response # Resolve the next hop against the ORIGINAL (pre-rewrite) URL so @@ -483,24 +494,26 @@ def safe_get(client: _UrlFetcher, url: str, **kwargs: Any) -> httpx.Response: raise SSRFError("Too many redirects") -async def async_safe_get(client: _AsyncUrlFetcher, url: str, **kwargs: Any) -> httpx.Response: +async def async_safe_get(client: _AsyncUrlFetcher, url: str, **kwargs: Unpack[_AsyncFetchOptions]) -> httpx.Response: """Async version of safe_get.""" if not getattr(litellm, "user_url_validation", True): - kwargs.setdefault("follow_redirects", True) - unvalidated: Final[_ResponseView] = {"response": await client.get(url, **kwargs)} + unvalidated_options: Final[_AsyncFetchOptions] = { + **kwargs, + "follow_redirects": kwargs.get("follow_redirects", True), + } + unvalidated: Final[_ResponseView] = {"response": await client.get(url, **unvalidated_options)} return unvalidated["response"] fetcher_view: Final[_AsyncFetcherView] = {"fetcher": client} fetcher: Final = fetcher_view["fetcher"] - kwargs.pop("follow_redirects", None) - headers_view: Final[_CallerHeadersView] = {"headers": kwargs.pop("headers", {})} + caller_headers: Final = kwargs.get("headers", {}) for _ in range(_MAX_REDIRECTS): validated_url, original_host = await asyncio.to_thread(validate_url, url) - response = await fetcher.get( - validated_url, - headers={**headers_view["headers"], "Host": original_host}, - follow_redirects=False, + hop_options: _AsyncFetchOptions = { **kwargs, - ) + "headers": {**caller_headers, "Host": original_host}, + "follow_redirects": False, + } + response = await fetcher.get(validated_url, **hop_options) if not response.is_redirect: return response # Resolve the next hop against the ORIGINAL (pre-rewrite) URL so diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index 7452a4a20ef..91f33635658 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -3,6 +3,7 @@ import os from typing import TYPE_CHECKING, Final from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import attribute_or_none from litellm.types.utils import CallTypes from . import * @@ -128,7 +129,7 @@ def discover_guardrail_translation_mappings() -> dict[CallTypes, type["BaseTrans # Check for guardrail_translation_mappings dictionary if hasattr(module, "guardrail_translation_mappings"): - mappings = getattr(module, "guardrail_translation_mappings") + mappings = attribute_or_none(module, "guardrail_translation_mappings") if isinstance(mappings, dict): discovered_mappings.update(mappings) verbose_logger.debug( diff --git a/litellm/llms/a2a/common_utils.py b/litellm/llms/a2a/common_utils.py index 1818b406823..633ea73bb16 100644 --- a/litellm/llms/a2a/common_utils.py +++ b/litellm/llms/a2a/common_utils.py @@ -31,6 +31,10 @@ class A2AError(BaseLLMException): ) +def _message_role(message: Mapping[str, object]) -> object: + return message.get("role", "user") + + def convert_messages_to_prompt(messages: list[AllMessageValues]) -> str: """ Convert OpenAI messages to a single prompt string for A2A agent. @@ -51,7 +55,7 @@ def convert_messages_to_prompt(messages: list[AllMessageValues]) -> str: # Get role if isinstance(msg, BaseModel): - role = msg.model_dump().get("role", "user") + role = _message_role(msg.model_dump()) elif isinstance(msg, dict): role = msg.get("role", "user") else: diff --git a/litellm/llms/ai21/chat/transformation.py b/litellm/llms/ai21/chat/transformation.py index 2b8c36d0085..e78cc77c0fc 100644 --- a/litellm/llms/ai21/chat/transformation.py +++ b/litellm/llms/ai21/chat/transformation.py @@ -4,6 +4,8 @@ AI21 Chat Completions API this is OpenAI compatible - no translation needed / occurs """ +from collections.abc import Mapping +from types import MappingProxyType from typing import Final from ...openai_like.chat.transformation import OpenAILikeChatConfig @@ -41,7 +43,7 @@ class AI21ChatConfig(OpenAILikeChatConfig): tool_choice: str | None = None, user: str | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/amazon_nova/chat/transformation.py b/litellm/llms/amazon_nova/chat/transformation.py index a93fbf1e933..4ef729bc76d 100644 --- a/litellm/llms/amazon_nova/chat/transformation.py +++ b/litellm/llms/amazon_nova/chat/transformation.py @@ -2,6 +2,8 @@ Translate from OpenAI's `/v1/chat/completions` to Amazon Nova's `/v1/chat/completions` """ +from collections.abc import Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Final import httpx @@ -38,7 +40,7 @@ class AmazonNovaChatConfig(OpenAILikeChatConfig): tools: list | None = None, reasoning_effort: list | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/anthropic/batches/handler.py b/litellm/llms/anthropic/batches/handler.py index 3f3716e0de6..a9d9f930b9e 100644 --- a/litellm/llms/anthropic/batches/handler.py +++ b/litellm/llms/anthropic/batches/handler.py @@ -132,7 +132,7 @@ class AnthropicBatchesHandler: max_retries: int | None, logging_obj: LiteLLMLoggingObj | None = None, litellm_params: dict | None = None, # mutable-ok: handed straight to validate_environment - ) -> LiteLLMBatch | Coroutine[Any, Any, LiteLLMBatch]: + ) -> LiteLLMBatch | Coroutine[object, object, LiteLLMBatch]: """ Retrieve a batch from Anthropic. diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 4c6bded68ea..3ab21099d3a 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -1697,14 +1697,13 @@ class AnthropicMessagesHandler(BaseTranslation): if not line.startswith("data:"): return line try: - data: Final[str | int | float | bool | None | Sequence[object] | Mapping[str, object]] = json.loads( - line[len("data:") :].strip() - ) + data: Final = _DECODED_SSE_DATA.validate_python(json.loads(line[len("data:") :].strip())) except json.JSONDecodeError: return line - if not isinstance(data, dict): + event: Final = as_str_mapping(data) + if not isinstance(event, dict): return line - rewritten: Final = _rewritten_event(_as_str_mapping(data), rewrite_event) + rewritten: Final = _rewritten_event(event, rewrite_event) return line if rewritten is data else "data: " + json.dumps(rewritten) def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 9a63afd5cc0..92e7c84031c 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -831,6 +831,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "else": "conditional (else): {}", "not": "must not match: {}", } + schema_fields: Final[Mapping[str, object]] = MappingProxyType(schema) min_items: Final = schema.get("minItems") enforced_min_items: Final = ( isinstance(min_items, int) and not isinstance(min_items, bool) and min_items in (0, 1) @@ -845,7 +846,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): for field, label in constraint_labels.items(): if field not in schema or field not in unsupported_fields: continue - value = schema[field] + value = schema_fields[field] # A falsy boolean constraint (e.g. ``uniqueItems: false``) imposes no # real requirement, so don't add a misleading advisory note for it. if isinstance(value, bool) and not value: @@ -869,7 +870,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): drops_conflicting_type: Final = _enum_conflicts_with_declared_type(schema) - for key, value in schema.items(): + for key, value in schema_fields.items(): if key in unsupported_fields: continue if key == "type" and drops_conflicting_type: @@ -2385,7 +2386,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): optional_params["tools"] = tools ## Load Config - config: Final = litellm.AnthropicConfig.get_config(model=model) + config: Final[Mapping[str, object]] = MappingProxyType(litellm.AnthropicConfig.get_config(model=model)) for k, v in config.items(): if ( k not in optional_params diff --git a/litellm/llms/anthropic/completion/transformation.py b/litellm/llms/anthropic/completion/transformation.py index 73e94e85f04..5cc4e955d00 100644 --- a/litellm/llms/anthropic/completion/transformation.py +++ b/litellm/llms/anthropic/completion/transformation.py @@ -7,6 +7,7 @@ Litellm provider slug: `anthropic_text/` import json import time from collections.abc import AsyncIterator, Iterator, Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Final import httpx @@ -112,7 +113,7 @@ class AnthropicTextConfig(BaseConfig): ) -> dict: prompt: Final = self._get_anthropic_text_prompt_from_messages(messages=messages, model=model) ## Load Config - config: Final = litellm.AnthropicTextConfig.get_config() + config: Final[Mapping[str, object]] = MappingProxyType(litellm.AnthropicTextConfig.get_config()) for k, v in config.items(): if ( k not in optional_params diff --git a/litellm/llms/anthropic/files/handler.py b/litellm/llms/anthropic/files/handler.py index 1a793e32317..279fb92cf6c 100644 --- a/litellm/llms/anthropic/files/handler.py +++ b/litellm/llms/anthropic/files/handler.py @@ -5,6 +5,7 @@ from collections.abc import Coroutine, Mapping from typing import Final import httpx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger @@ -24,6 +25,8 @@ from litellm.types.utils import CallTypes, LlmProviders, ModelResponse from ..chat.transformation import AnthropicConfig from ..common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP, AnthropicModelInfo +_DECODED_JSON: Final = TypeAdapter(object) + class AnthropicFilesHandler: """ @@ -194,7 +197,7 @@ class AnthropicFilesHandler: if not line.strip(): continue - anthropic_result: object = json.loads(line) + anthropic_result: object = _DECODED_JSON.validate_python(json.loads(line)) if not isinstance(anthropic_result, dict): raise TypeError( f"Anthropic batch result line is not a JSON object: {type(anthropic_result).__name__}" diff --git a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py index e9fd77ef441..dadd4d8558d 100644 --- a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py @@ -25,7 +25,7 @@ from typing_extensions import assert_never from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items from litellm.litellm_core_utils.hidden_params import set_hidden_params from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame @@ -1367,9 +1367,9 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): # open a block: the emitted {"type": "thinking", "thinking": ""} gets # replayed as history and Anthropic rejects it (LIT-6357). A signed # entry opens the block so the client receives the replay signature. - thinking_blocks: Final = getattr(delta, "thinking_blocks", None) - if thinking_blocks and any( - isinstance(b, dict) and not is_empty_unsigned_thinking_block(b) for b in thinking_blocks + if any( + isinstance(b, dict) and not is_empty_unsigned_thinking_block(b) + for b in attribute_items(delta, "thinking_blocks") ): return False return True diff --git a/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py index 935ce77ef87..d4430ca5ab9 100644 --- a/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py +++ b/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py @@ -15,7 +15,7 @@ from dataclasses import dataclass from functools import reduce from itertools import chain from types import MappingProxyType -from typing import Annotated, Any, Final, Literal, TypeVar, cast +from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeVar, cast from pydantic import BaseModel, ConfigDict, Field, StrictInt, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -32,6 +32,9 @@ from ..constants import ( from ..errors import AnthropicContextManagementError from ..placeholders import build_cleared_tool_result_content +if TYPE_CHECKING: + from litellm.types.llms.openai import ChatCompletionToolParam + T = TypeVar("T") # api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep _TINY_RESULT_BYTES: Final = 46 @@ -216,7 +219,7 @@ def _trigger_met( current_tokens: Final = litellm.token_counter( model=model, messages=messages, - tools=cast(Any, tools), + tools=cast("list[ChatCompletionToolParam] | None", tools), ) verbose_logger.debug("context_management polyfill: current_tokens: %s", current_tokens) verbose_logger.debug("context_management polyfill: threshold: %s", threshold) @@ -366,9 +369,13 @@ def apply_clear_tool_uses_20250919( tokens_before: Final = ( counted_tokens if counted_tokens is not None - else litellm.token_counter(model=model, messages=messages, tools=cast(Any, tools)) + else litellm.token_counter( + model=model, messages=messages, tools=cast("list[ChatCompletionToolParam] | None", tools) + ) + ) + tokens_after: Final = litellm.token_counter( + model=model, messages=final_messages, tools=cast("list[ChatCompletionToolParam] | None", tools) ) - tokens_after: Final = litellm.token_counter(model=model, messages=final_messages, tools=cast(Any, tools)) cleared_input_tokens: Final = max(tokens_before - tokens_after, 0) return final_messages, { diff --git a/litellm/llms/anthropic/pass_through/context_management/placeholders.py b/litellm/llms/anthropic/pass_through/context_management/placeholders.py index b3d3529f105..8a0eb6d48e1 100644 --- a/litellm/llms/anthropic/pass_through/context_management/placeholders.py +++ b/litellm/llms/anthropic/pass_through/context_management/placeholders.py @@ -1,12 +1,10 @@ """Placeholder content for cleared ``tool_result`` blocks (string or block list).""" -from typing import Any - from .constants import CLEARED_TOOL_RESULT_PLACEHOLDER def build_cleared_tool_result_content( - original_content: Any, + original_content: object, ) -> str | list[dict]: """Return a string or single text block list, matching ``original_content`` shape.""" if isinstance(original_content, list): diff --git a/litellm/llms/anthropic/pass_through/context_management/result.py b/litellm/llms/anthropic/pass_through/context_management/result.py index 5b88fe30871..4713b8d79c7 100644 --- a/litellm/llms/anthropic/pass_through/context_management/result.py +++ b/litellm/llms/anthropic/pass_through/context_management/result.py @@ -6,7 +6,7 @@ attach ``iterations`` to ``usage``. """ from dataclasses import dataclass, field -from typing import Any, Final +from typing import Final from litellm.types.llms.anthropic import ( AppliedEdit, @@ -19,8 +19,8 @@ from .constants import COMPACT_EDIT_TYPE @dataclass class PolyfillResult: - messages: list[dict[str, Any]] - system: str | list[dict[str, Any]] | None + messages: list[dict[str, object]] + system: str | list[dict[str, object]] | None applied_edits: list[AppliedEdit] = field(default_factory=list) compaction_block: CompactionBlock | None = None iterations_usage: list[UsageIteration] | None = None diff --git a/litellm/llms/anthropic/pass_through/convertible_blocks.py b/litellm/llms/anthropic/pass_through/convertible_blocks.py index fb17910fb97..9d651f8d721 100644 --- a/litellm/llms/anthropic/pass_through/convertible_blocks.py +++ b/litellm/llms/anthropic/pass_through/convertible_blocks.py @@ -3,7 +3,6 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from functools import partial from itertools import chain, groupby -from operator import itemgetter from types import MappingProxyType from typing import Final, TypeAlias @@ -279,6 +278,14 @@ def _with_kept_parts(block: object, tool_result: Mapping[str, object], parts: _K return {**tool_result, "content": [_restored_part(part, parts.get(index)) for index, part in enumerate(blocks)]} +def _entry_tool_use_id(entry: tuple[str, _KeptParts]) -> str: + return entry[0] + + +def _entry_kept_parts(entry: tuple[str, _KeptParts]) -> _KeptParts: + return entry[1] + + def _restored_block(block: object, kept: Mapping[str, Iterator[_KeptParts]]) -> object: match as_str_mapping(block): case {"type": "tool_result", "tool_use_id": str(tool_use_id)} as tool_result if ( @@ -293,13 +300,15 @@ def _restored_block(block: object, kept: Mapping[str, Iterator[_KeptParts]]) -> def keep_non_string_tool_result_texts(rewritten: Sequence[object], original: object) -> Sequence[object]: blocks: Final = chain.from_iterable(map(_user_content_blocks, _list_items(original))) - by_id: Final = tuple(sorted(chain.from_iterable(map(_tool_result_non_string_texts, blocks)), key=itemgetter(0))) + by_id: Final = tuple( + sorted(chain.from_iterable(map(_tool_result_non_string_texts, blocks)), key=_entry_tool_use_id) + ) if not any(parts for _, parts in by_id): return rewritten kept_in_order: Final = MappingProxyType( { - tool_use_id: iter(tuple(map(itemgetter(1), group))) - for tool_use_id, group in groupby(by_id, key=itemgetter(0)) + tool_use_id: iter(tuple(map(_entry_kept_parts, group))) + for tool_use_id, group in groupby(by_id, key=_entry_tool_use_id) } ) return [_with_user_blocks(message, partial(_restored_block, kept=kept_in_order)) for message in rewritten] diff --git a/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py b/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py index 8d1b358c8f1..2b6161d98f6 100644 --- a/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py +++ b/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py @@ -9,8 +9,8 @@ the LLM doesn't make a tool call, and we need to return a stream to the user. """ import json -from collections.abc import Mapping -from typing import Any, Final, cast +from collections.abc import Mapping, Sequence +from typing import Final, cast from litellm.llms.anthropic.pass_through.messages.utils import anthropic_message_delta from litellm.types.llms.anthropic_messages.anthropic_response import ( @@ -136,8 +136,7 @@ class FakeAnthropicMessagesStreamIterator: """Convert the non-streaming response to streaming chunks""" chunks: Final = [] - # Cast response to dict for easier access - response_dict: Final = cast(dict[str, Any], self.response) + response_dict: Final[Mapping[str, object]] = self.response # 1. message_start event usage: Final = self.response.get("usage") @@ -160,9 +159,8 @@ class FakeAnthropicMessagesStreamIterator: chunks.append(f"event: message_start\ndata: {json.dumps(message_start)}\n\n".encode()) # 2-4. For each content block, send start/delta/stop events - content_blocks: Final = response_dict.get("content", []) - for index, block in enumerate(content_blocks): - block_dict = cast(dict[str, Any], block) + content_blocks: Final = cast(Sequence[Mapping[str, object]], response_dict.get("content", [])) + for index, block_dict in enumerate(content_blocks): chunks.extend(self._create_content_block_chunks(block_dict, index)) # 5. message_delta event (with final usage and stop_reason) diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/pass_through/messages/transformation.py index dfe0788e50a..35db54147c5 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/pass_through/messages/transformation.py @@ -722,7 +722,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): logging_obj: LiteLLMLoggingObj, ) -> AnthropicMessagesResponse: try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final[AnthropicMessagesResponse] = raw_response.json() except Exception: raise AnthropicError(message=raw_response.text, status_code=raw_response.status_code) response: Final = AnthropicMessagesResponse(**raw_response_json) diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py index bd0f57c3ed9..48fc987a95a 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py @@ -242,7 +242,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: @staticmethod def _summary_part_text(part: object) -> str: if isinstance(part, Mapping): - mapping: Final = cast(Mapping[str, Any], part) # cast-ok: summary parts are untyped provider json + mapping: Final = cast(Mapping[str, object], part) # cast-ok: summary parts are untyped provider json return str(mapping.get("text") or "") return str(getattr(part, "text", None) or "") @@ -673,7 +673,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: output_config = anthropic_request.get("output_config") reasoning: Final = self.translate_thinking_to_reasoning( thinking, - output_config=cast(dict[str, Any] | None, output_config), + output_config=cast(dict[str, object] | None, output_config), auto_summary_level=auto_summary_level, drop_params=drop_params, ) diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index 99ebd2875a7..df8a0614665 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -1,5 +1,5 @@ from collections.abc import Coroutine -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final from openai import AsyncAzureOpenAI, AzureOpenAI from pydantic import BaseModel @@ -38,7 +38,7 @@ class AzureAudioTranscription(AzureChatCompletion): atranscription: bool = False, litellm_params: dict | None = None, custom_llm_provider: str = "azure", - ) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]: + ) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]: data: Final = {"model": model, "file": audio_file, **optional_params} if atranscription is True: diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 67b25bf7d12..02f1ba3472a 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -13,7 +13,7 @@ from openai import ( BadRequestError, OpenAI, ) -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter import litellm from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES @@ -59,6 +59,8 @@ from .image_generation import ( ) from .image_generation.http_utils import azure_deployment_image_generation_json_body +_DECODED_JSON: Final = TypeAdapter(object) + class AzureOpenAIAssistantsAPIConfig: """ @@ -961,7 +963,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): body=error_data, ) - result: Final = response.json()["result"] + result: Final = _DECODED_JSON.validate_python(response.json()["result"]) return httpx.Response( status_code=200, headers=response.headers, @@ -1078,7 +1080,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): body=error_data, ) - result: Final = response.json()["result"] + result: Final = _DECODED_JSON.validate_python(response.json()["result"]) return httpx.Response( status_code=200, headers=response.headers, @@ -1358,7 +1360,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): except AzureOpenAIError as e: raise e except Exception as e: - error_code: Final = getattr(e, "status_code", None) + error_code: Final[object] = getattr(e, "status_code", None) if error_code is not None: raise AzureOpenAIError(status_code=error_code, message=str(e)) else: diff --git a/litellm/llms/azure/exception_mapping.py b/litellm/llms/azure/exception_mapping.py index dcc3601cd8e..f7146a36384 100644 --- a/litellm/llms/azure/exception_mapping.py +++ b/litellm/llms/azure/exception_mapping.py @@ -1,4 +1,4 @@ -from typing import Any, Final +from typing import Final from litellm.exceptions import ContentPolicyViolationError @@ -54,7 +54,7 @@ class AzureOpenAIExceptionMapping: @staticmethod def _extract_azure_error( original_exception: Exception, - ) -> tuple[dict[str, object], dict | None]: + ) -> tuple[dict[str, object], object]: """Extract Azure OpenAI error payload and inner error details. Azure error formats can vary by endpoint/version. Common shapes: @@ -67,13 +67,13 @@ class AzureOpenAIExceptionMapping: return {}, None # Some SDKs place the payload under "error". - azure_error: dict[str, Any] + azure_error: dict[str, object] if isinstance(body_dict.get("error"), dict): azure_error = body_dict.get("error", {}) else: azure_error = body_dict - inner_error: Final = ( + inner_error: Final[object] = ( azure_error.get("inner_error") or azure_error.get("innererror") or body_dict.get("innererror") diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 8cae4bf284b..6d904503df8 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -95,7 +95,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): def get_stripped_model_name(self, model: str) -> str: return model.replace("responses/", "").replace("o_series/", "").replace("azure_ai/", "") - def _handle_reasoning_item(self, item: dict[str, Any]) -> dict[str, Any]: + def _handle_reasoning_item(self, item: dict[str, Any]) -> Mapping[str, object]: """ Handle reasoning items to filter out the status field. Issue: https://github.com/BerriAI/litellm/issues/13484 diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 83d00a621b4..3b541310d28 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -397,7 +397,7 @@ class AzureAIStudioConfig(OpenAIConfig): return data def _drop_tool_level_extra_fields(self, request_data: dict, error_text: str) -> dict: - fields_to_drop: Final = set(re.findall(r"tools\[\d+\]\.([\w-]+)", error_text)) + fields_to_drop: Final[Collection[str]] = set(re.findall(r"tools\[\d+\]\.([\w-]+)", error_text)) tools: Final = request_data.get("tools") if fields_to_drop and isinstance(tools, list): for tool in tools: diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index b2eb4cde5ce..91d0d52a629 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -94,9 +94,10 @@ class BaseConfig(ABC): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("_") and not isinstance( v, @@ -513,7 +514,7 @@ class BaseConfig(ABC): def apply_assembled_streaming_response_metadata( self, response: "ModelResponse", - chunks: list[Any], + chunks: list[object], ) -> None: """Hook for providers to merge chunk metadata into assembled streaming responses.""" return diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index 5fe81a7f1b7..6399ded9555 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -121,7 +121,7 @@ class BaseTranslation(ABC): @staticmethod def merge_user_api_key_metadata_into_request( - request_data: dict[str, Any], # mutable-ok: proxy hooks share and mutate the request payload dict in place + request_data: dict[str, object], # mutable-ok: proxy hooks share and mutate the request payload dict in place user_api_key_dict: Optional["UserAPIKeyAuth"], ) -> None: """ diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index 55a4b3da0b3..725146fe894 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -9,6 +9,8 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage +_DECODED_JSON: Final = TypeAdapter(object) + def _anthropic_stream_chunk_events(item: object) -> list[dict]: if isinstance(item, dict): @@ -30,7 +32,7 @@ def _anthropic_stream_chunk_events(item: object) -> list[dict]: if not payload or payload == "[DONE]": continue try: - parsed = json.loads(payload) + parsed = _DECODED_JSON.validate_python(json.loads(payload)) except json.JSONDecodeError: continue if isinstance(parsed, dict): diff --git a/litellm/llms/base_llm/image_edit/transformation.py b/litellm/llms/base_llm/image_edit/transformation.py index 4faf0aaaf30..35b4bb90025 100644 --- a/litellm/llms/base_llm/image_edit/transformation.py +++ b/litellm/llms/base_llm/image_edit/transformation.py @@ -1,7 +1,7 @@ import types from abc import ABC, abstractmethod from collections.abc import Mapping -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Final import httpx from httpx._types import RequestFiles @@ -32,9 +32,10 @@ class BaseImageEditConfig(ABC): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/base_llm/interactions/transformation.py b/litellm/llms/base_llm/interactions/transformation.py index b7bbf7bde73..39ca1d8f7d7 100644 --- a/litellm/llms/base_llm/interactions/transformation.py +++ b/litellm/llms/base_llm/interactions/transformation.py @@ -11,7 +11,8 @@ Per OpenAPI spec (https://ai.google.dev/static/api/interactions.openapi.json): import types from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Final import httpx @@ -60,9 +61,10 @@ class BaseInteractionsAPIConfig(ABC): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/base_llm/managed_resources/base_managed_resource.py b/litellm/llms/base_llm/managed_resources/base_managed_resource.py index d3b05716452..2663a3c42e7 100644 --- a/litellm/llms/base_llm/managed_resources/base_managed_resource.py +++ b/litellm/llms/base_llm/managed_resources/base_managed_resource.py @@ -141,7 +141,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]): self, llm_router: Router, model: str, - request_data: dict[str, Any], + request_data: dict[str, object], litellm_parent_otel_span: Span, ) -> ResourceObjectType: """ @@ -399,7 +399,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]): async def create_resource_for_each_model( self, llm_router: Router, - request_data: dict[str, Any], + request_data: dict[str, object], target_model_names_list: list[str], litellm_parent_otel_span: Span, ) -> list[ResourceObjectType]: diff --git a/litellm/llms/base_llm/passthrough/transformation.py b/litellm/llms/base_llm/passthrough/transformation.py index ac5032c08f1..7e58cc6e71b 100644 --- a/litellm/llms/base_llm/passthrough/transformation.py +++ b/litellm/llms/base_llm/passthrough/transformation.py @@ -28,6 +28,7 @@ if TYPE_CHECKING: RELAYED_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) class PassthroughMetadata(LiteLLMBaseModel): @@ -71,7 +72,7 @@ def relayed_json_object(httpx_response: Response) -> Mapping[str, object] | None def relayed_body(httpx_response: Response) -> str | dict: try: - body: Final[object] = httpx_response.json() + body: Final = _DECODED_JSON.validate_python(httpx_response.json()) except ValueError: return httpx_response.text return body if isinstance(body, dict) else httpx_response.text diff --git a/litellm/llms/base_llm/rerank/transformation.py b/litellm/llms/base_llm/rerank/transformation.py index 5d2f92b5e82..7f246bf33da 100644 --- a/litellm/llms/base_llm/rerank/transformation.py +++ b/litellm/llms/base_llm/rerank/transformation.py @@ -80,7 +80,7 @@ class BaseRerankConfig(ABC): model: str, drop_params: bool, query: str, - documents: list[str | dict[str, Any]], + documents: list[str | dict[str, object]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, diff --git a/litellm/llms/base_llm/text_to_speech/transformation.py b/litellm/llms/base_llm/text_to_speech/transformation.py index fb85cdb7687..be49098b62b 100644 --- a/litellm/llms/base_llm/text_to_speech/transformation.py +++ b/litellm/llms/base_llm/text_to_speech/transformation.py @@ -1,6 +1,7 @@ import types from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, TypedDict +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Final, TypedDict import httpx @@ -40,9 +41,10 @@ class BaseTextToSpeechConfig(ABC): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index 28f6348e4d9..138700b3212 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -152,7 +152,7 @@ class BaseVectorStoreConfig: api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, ) -> tuple[str, dict]: pass @@ -164,7 +164,7 @@ class BaseVectorStoreConfig: api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, ) -> tuple[str, dict]: """ Optional async version of transform_search_vector_store_request. diff --git a/litellm/llms/baseten/chat.py b/litellm/llms/baseten/chat.py index 381d487c36d..c165831b90a 100644 --- a/litellm/llms/baseten/chat.py +++ b/litellm/llms/baseten/chat.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final import litellm @@ -42,7 +43,7 @@ class BasetenConfig(OpenAIGPTConfig): frequency_penalty: int | None = None, stream_options: dict | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = dict(locals()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/bedrock/batches/handler.py b/litellm/llms/bedrock/batches/handler.py index 48e79d39b8e..f37abacc883 100644 --- a/litellm/llms/bedrock/batches/handler.py +++ b/litellm/llms/bedrock/batches/handler.py @@ -174,12 +174,13 @@ class BedrockBatchesHandler: ) def job_status() -> "LiteLLMBatch": + auth_kwargs: Final[Mapping[str, object]] = dict(auth_params.model_dump()) return BedrockBatchesHandler.handle_model_invocation_job_status( batch_id=batch_id, aws_region_name=region, logging_obj=logging_obj, aws_bedrock_control_plane_endpoint=control_plane_endpoint, - **auth_params.model_dump(), + **auth_kwargs, ) try: diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index bb5bb246a93..18212b7390f 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Optional, Union from urllib.parse import quote import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm._uuid import uuid @@ -51,6 +52,8 @@ else: HTTPHandler = Any AsyncHTTPHandler = Any +_DECODED_JSON: Final = TypeAdapter(object) + class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): def __init__(self, **kwargs): @@ -314,7 +317,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): return None try: - data: Final[object] = json.loads(json_str) + data: Final = _DECODED_JSON.validate_python(json.loads(json_str)) # Skip non-dict data (some lines contain JSON strings) return data if isinstance(data, dict) else None except json.JSONDecodeError: @@ -482,7 +485,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): if "application/json" in content_type: # Direct JSON response verbose_logger.debug("Parsing JSON response") - response_json: Final[object] = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) verbose_logger.debug("Response JSON: %s", response_json) return self._parse_json_response(response_json) else: @@ -566,7 +569,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): continue try: - data_obj = json.loads(json_str) + data_obj = _DECODED_JSON.validate_python(json.loads(json_str)) if not isinstance(data_obj, dict): continue @@ -706,7 +709,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): ) try: body: Final = response.read() - response_json: Final[object] = json.loads(body) + response_json: Final = _DECODED_JSON.validate_python(json.loads(body)) except (json.JSONDecodeError, Exception) as e: raise BedrockError( status_code=response.status_code, @@ -788,7 +791,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): continue try: - data_obj = json.loads(json_str) + data_obj = _DECODED_JSON.validate_python(json.loads(json_str)) if not isinstance(data_obj, dict): continue @@ -928,7 +931,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): ) try: body: Final = await response.aread() - response_json: Final[object] = json.loads(body) + response_json: Final = _DECODED_JSON.validate_python(json.loads(body)) except (json.JSONDecodeError, Exception) as e: raise BedrockError( status_code=response.status_code, diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index cd3727401ef..4d7ec663d09 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -12,7 +12,8 @@ from itertools import chain from typing import TYPE_CHECKING, Final, Literal, cast, overload import httpx -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config +from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -21,6 +22,7 @@ from litellm.constants import ( RESPONSE_FORMAT_TOOL_NAME, ) from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, filter_exceptions_from_params, filter_internal_params, map_finish_reason, @@ -160,6 +162,14 @@ _TOOLS_AS_SENT: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _JSON_VALUE: Final = TypeAdapter(object) _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) _TOOLS_AS_RECEIVED: Final = TypeAdapter(list[object]) + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _ToolCallInContent(TypedDict): + name: ReadOnly[str] + + +_TOOL_CALL_IN_CONTENT: Final = TypeAdapter(_ToolCallInContent) CLAUDE_ON_BEDROCK_INVOKE_REMEDY: Final = " This model runs tool search on Bedrock through `bedrock/invoke/`." _DEPLOYMENT_PARAMS: Final = TypeAdapter(Mapping[str, object]) @@ -2453,9 +2463,17 @@ class AmazonConverseConfig(BaseConfig): if message.content is not None: try: tool_call_names: Final = self.get_tool_call_names(tools) - json_content: Final = json.loads(message.content) - if json_content.get("type") == "function" and json_content.get("name") in tool_call_names: - tool_calls: Final = [ChatCompletionMessageToolCall(function=Function(**json_content))] + json_content: Final = as_str_mapping(_JSON_VALUE.validate_python(json.loads(message.content))) + if ( + json_content is not None + and json_content.get("type") == "function" + and json_content.get("name") in tool_call_names + ): + tool_calls: Final = [ + ChatCompletionMessageToolCall( + function=Function(**_TOOL_CALL_IN_CONTENT.validate_python(json_content)) + ) + ] message.tool_calls = tool_calls message.content = None diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_ai21_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_ai21_transformation.py index 3005bc82299..777237b4f18 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_ai21_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_ai21_transformation.py @@ -56,9 +56,10 @@ class AmazonAI21Config(AmazonInvokeConfig, BaseConfig): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_cohere_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_cohere_transformation.py index 00c2eaf0f8f..2118d183b22 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_cohere_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_cohere_transformation.py @@ -37,9 +37,10 @@ class AmazonCohereConfig(AmazonInvokeConfig, CohereChatConfig): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_llama_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_llama_transformation.py index d2eb6eef9ef..cb867a09f8f 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_llama_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_llama_transformation.py @@ -37,9 +37,10 @@ class AmazonLlamaConfig(AmazonInvokeConfig, BaseConfig): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_mistral_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_mistral_transformation.py index 44511eeaa3e..4420bb3a418 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_mistral_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_mistral_transformation.py @@ -47,9 +47,10 @@ class AmazonMistralConfig(AmazonInvokeConfig, BaseConfig): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_titan_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_titan_transformation.py index a18e97c63a6..736b3ea3cd2 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_titan_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_titan_transformation.py @@ -43,9 +43,10 @@ class AmazonTitanConfig(AmazonInvokeConfig, BaseConfig): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude2_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude2_transformation.py index b29ddf01d1e..2655c13be44 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude2_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude2_transformation.py @@ -44,9 +44,10 @@ class AmazonAnthropicConfig(AmazonInvokeConfig): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/bedrock/chat/mantle/transformation.py b/litellm/llms/bedrock/chat/mantle/transformation.py index e48196946cd..d61541aab8a 100644 --- a/litellm/llms/bedrock/chat/mantle/transformation.py +++ b/litellm/llms/bedrock/chat/mantle/transformation.py @@ -21,6 +21,7 @@ from litellm.types.utils import ModelResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.llms.anthropic.chat.handler import ModelResponseIterator LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -134,7 +135,7 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig): streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> "ModelResponseIterator": return self.get_model_response_iterator_for_request( streaming_response=streaming_response, sync_stream=sync_stream, json_mode=json_mode, optional_params={} ) @@ -145,7 +146,7 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig): sync_stream: bool, json_mode: bool | None, optional_params: Mapping[str, object], - ) -> object: + ) -> "ModelResponseIterator": from litellm.llms.anthropic.chat.handler import ModelResponseIterator return ModelResponseIterator( diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 4954e646f58..2b946d9664c 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -2021,8 +2021,13 @@ def get_bedrock_response_stream_shape(): _BEDROCK_STREAM_OUTPUT_SHAPES: Final = ("ConverseStreamOutput", "ResponseStream") +class _ModeledErrorMetadata(TypedDict, total=False): + httpStatusCode: ReadOnly[int] + + def _modeled_error_status(member: Shape) -> int | None: - status: Final = (member.metadata or {}).get("error", {}).get("httpStatusCode") + error: Final[_ModeledErrorMetadata] = (member.metadata or {}).get("error", {}) + status: Final = error.get("httpStatusCode") return None if status is None else int(status) diff --git a/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py b/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py index 581b89369f7..fba2673b5aa 100644 --- a/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py +++ b/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py @@ -35,9 +35,10 @@ class AmazonTitanG1Config: @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py b/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py index 3b594662a9b..7d319ca6b06 100644 --- a/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py +++ b/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py @@ -39,9 +39,10 @@ class AmazonTitanV2Config: @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/bedrock/files/handler.py b/litellm/llms/bedrock/files/handler.py index 3d23b69f846..37f95ff7b9b 100644 --- a/litellm/llms/bedrock/files/handler.py +++ b/litellm/llms/bedrock/files/handler.py @@ -1,6 +1,6 @@ import asyncio from collections.abc import Coroutine, Mapping -from typing import Any, Final +from typing import Final import httpx @@ -139,7 +139,7 @@ class BedrockFilesHandler(BaseAWSLLM): optional_params: dict, timeout: float | httpx.Timeout, max_retries: int | None, - ) -> HttpxBinaryResponseContent | Coroutine[Any, Any, HttpxBinaryResponseContent]: + ) -> HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent]: """ Download file content from S3 bucket for Bedrock files. Supports both sync and async operations. diff --git a/litellm/llms/bedrock/image_edit/handler.py b/litellm/llms/bedrock/image_edit/handler.py index 466993de9d1..bdfe6f5a8d2 100644 --- a/litellm/llms/bedrock/image_edit/handler.py +++ b/litellm/llms/bedrock/image_edit/handler.py @@ -10,6 +10,7 @@ import json from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger @@ -39,6 +40,8 @@ if TYPE_CHECKING: else: AWSPreparedRequest = Any +_DECODED_JSON: Final = TypeAdapter(object) + class BedrockImageEditPreparedRequest(LiteLLMBaseModel): """ @@ -317,7 +320,7 @@ class BedrockImageEdit(BaseAWSLLM): additional_args={"complete_input_dict": data}, ) verbose_logger.debug("raw model_response: %s", response.text) - response_dict: Final = response.json() + response_dict: Final = _DECODED_JSON.validate_python(response.json()) if response_dict is None: raise ValueError("Error in response object format, got None") diff --git a/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py b/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py index acc847e0e3c..e5c5edba750 100644 --- a/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py @@ -1,4 +1,5 @@ import types +from collections.abc import Mapping from typing import Final from openai.types.image import Image @@ -20,9 +21,10 @@ class AmazonStability3Config: @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py b/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py index 145e6a0781f..5731541da56 100644 --- a/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py @@ -3,6 +3,7 @@ Transformation logic for Amazon Titan Image Generation. """ import types +from collections.abc import Mapping from typing import Final from openai.types.image import Image @@ -35,16 +36,17 @@ class AmazonTitanImageGenerationConfig: width: int | None = None, height: int | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 12421aa94fd..805652630ed 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -1,4 +1,4 @@ -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, cast @@ -145,7 +145,7 @@ class AmazonAnthropicClaudeMessagesConfig( self, headers: dict, model: str, - messages: list[object], + messages: Sequence[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/bedrock/messages/mantle_transformation.py b/litellm/llms/bedrock/messages/mantle_transformation.py index c71721d800e..d4a2d2eabe9 100644 --- a/litellm/llms/bedrock/messages/mantle_transformation.py +++ b/litellm/llms/bedrock/messages/mantle_transformation.py @@ -7,7 +7,7 @@ stripping, and the anthropic-version / anthropic-beta placement (headers, never the body) that are specific to the bedrock-mantle endpoint. """ -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -90,7 +90,7 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): self, headers: dict, model: str, - messages: list[Any], + messages: Sequence[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/bedrock/model_listing.py b/litellm/llms/bedrock/model_listing.py index 99ff48f9400..409bf2c7b97 100644 --- a/litellm/llms/bedrock/model_listing.py +++ b/litellm/llms/bedrock/model_listing.py @@ -4,7 +4,7 @@ from typing import Final, TypeAlias from urllib.parse import quote, urlencode import httpx -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token @@ -15,6 +15,7 @@ from litellm.types.llms.bedrock import AwsAuthParams LIST_MODELS_TIMEOUT: Final = 10.0 INFERENCE_PROFILES_PAGE_SIZE: Final = 1000 INFERENCE_PROFILES_PAGE_CAP: Final = 20 +_DECODED_JSON: Final = TypeAdapter(object) QueryPairs: TypeAlias = tuple[tuple[str, str], ...] @@ -103,7 +104,7 @@ class BedrockModelLister(BaseAWSLLM): f"Status code: {response.status_code}, Response: {response.text}" ), ) - return response.json() + return _DECODED_JSON.validate_python(response.json()) def _auth_headers(self, url: str) -> Mapping[str, str]: if self._bearer_token is not None: diff --git a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py index a90b4829634..1b01b0ec6d0 100644 --- a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py +++ b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py @@ -544,7 +544,7 @@ class _StreamFrame(TypedDict): def _unpack_uint32(buffer: bytes) -> int: - return struct.unpack("!I", buffer)[0] + return int.from_bytes(buffer, "big") def _collect_stream_delta_text_holders(delta: object) -> list[_DeltaHolder]: @@ -1255,7 +1255,9 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation): headers_bytes = frame_raw[12 : 12 + orig_hdrs_len] try: - payload_dict = json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4]) + payload_dict = _BODY_BY_REFERENCE.validate_python( + json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4]) + ) for local_idx, (_, container, key) in enumerate( _collect_stream_delta_text_holders(payload_dict.get("delta")) ): diff --git a/litellm/llms/black_forest_labs/image_edit/transformation.py b/litellm/llms/black_forest_labs/image_edit/transformation.py index 7cfb0daf334..38bcbb0d905 100644 --- a/litellm/llms/black_forest_labs/image_edit/transformation.py +++ b/litellm/llms/black_forest_labs/image_edit/transformation.py @@ -51,6 +51,7 @@ class _BflPolledResult(TypedDict): _BFL_POLLED_RESULT_ADAPTER: Final = TypeAdapter(_BflPolledResult) +_DECODED_JSON: Final = TypeAdapter(object) _BFL_REQUEST_PARAMS: Final = ( @@ -316,7 +317,7 @@ class BlackForestLabsImageEditConfig(BaseImageEditConfig): The response contains: {"status": "Ready", "result": {"sample": "https://..."}} """ try: - raw_response_data: Final[object] = raw_response.json() + raw_response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise BlackForestLabsError( status_code=raw_response.status_code, diff --git a/litellm/llms/black_forest_labs/image_generation/transformation.py b/litellm/llms/black_forest_labs/image_generation/transformation.py index 928af314506..52f2c9a7c50 100644 --- a/litellm/llms/black_forest_labs/image_generation/transformation.py +++ b/litellm/llms/black_forest_labs/image_generation/transformation.py @@ -40,6 +40,7 @@ else: LiteLLMLoggingObj = Any _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class BlackForestLabsImageGenerationConfig(BaseImageGenerationConfig): @@ -272,7 +273,7 @@ class BlackForestLabsImageGenerationConfig(BaseImageGenerationConfig): The response contains: {"status": "Ready", "result": {"sample": "https://..."}} """ try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise BlackForestLabsError( status_code=raw_response.status_code, diff --git a/litellm/llms/cerebras/chat.py b/litellm/llms/cerebras/chat.py index c3aa26ade35..9fc99500fc5 100644 --- a/litellm/llms/cerebras/chat.py +++ b/litellm/llms/cerebras/chat.py @@ -4,6 +4,7 @@ Cerebras Chat Completions API this is OpenAI compatible - no translation needed / occurs """ +from collections.abc import Mapping from typing import Final from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig @@ -41,7 +42,7 @@ class CerebrasConfig(OpenAIGPTConfig): user: str | None = None, reasoning_effort: str | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = dict(locals()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/chatgpt/chat/streaming_utils.py b/litellm/llms/chatgpt/chat/streaming_utils.py index ee3120bacea..49be583f6d1 100644 --- a/litellm/llms/chatgpt/chat/streaming_utils.py +++ b/litellm/llms/chatgpt/chat/streaming_utils.py @@ -7,6 +7,8 @@ Normalizes non-spec-compliant tool_call chunks from the ChatGPT backend API. from collections.abc import Awaitable from typing import Final, Protocol +from pydantic import TypeAdapter + from litellm.types.utils import ( ChatCompletionDeltaCustomToolCall, ChatCompletionDeltaToolCall, @@ -23,6 +25,9 @@ class ChatGPTChunkStream(Protocol): def __anext__(self) -> Awaitable[ModelResponseStream]: ... +_DELEGATED_ATTRIBUTE: Final = TypeAdapter(object) + + def _first_choice_delta(chunk: ModelResponseStream) -> Delta | None: return chunk.choices[0].delta @@ -47,7 +52,7 @@ class ChatGPTToolCallNormalizer: self._last_id: str | None = None # tracks which tool call the next delta belongs to def __getattr__(self, name: str) -> object: - return getattr(self._stream, name) + return _DELEGATED_ATTRIBUTE.validate_python(getattr(self._stream, name)) def __iter__(self) -> "ChatGPTToolCallNormalizer": return self diff --git a/litellm/llms/clarifai/chat/transformation.py b/litellm/llms/clarifai/chat/transformation.py index 8c2fd9e0aff..d07ae5e4215 100644 --- a/litellm/llms/clarifai/chat/transformation.py +++ b/litellm/llms/clarifai/chat/transformation.py @@ -23,6 +23,7 @@ else: LiteLLMLoggingObj = Any _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class ClarifaiConfig(OpenAIGPTConfig): @@ -114,7 +115,7 @@ class ClarifaiConfig(OpenAIGPTConfig): ) ## Reponse try: - completion_response: Final = raw_response.json() + completion_response: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise OpenAIError( status_code=raw_response.status_code, diff --git a/litellm/llms/claude_code/harness/transformation.py b/litellm/llms/claude_code/harness/transformation.py index f75d86c7e63..8252d2dd461 100644 --- a/litellm/llms/claude_code/harness/transformation.py +++ b/litellm/llms/claude_code/harness/transformation.py @@ -30,6 +30,7 @@ from litellm.harness.types import ( ToolCall, ToolResult, ) +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.llms.base_llm.harness.transformation import ( BaseCLIHarnessConfig, HarnessSessionSetup, @@ -231,12 +232,12 @@ def _system_events(event: Mapping[str, object], state: ClaudeCodeStreamState) -> return event_list() -def _record_result(event: Mapping[str, Any], state: ClaudeCodeStreamState) -> Sequence[Event]: +def _record_result(event: Mapping[str, object], state: ClaudeCodeStreamState) -> Sequence[Event]: state.result_seen = True state.is_error = bool(event.get("is_error", False)) result = event.get("result") state.result_text = result if isinstance(result, str) else None - state.errors = [str(e) for e in event.get("errors") or ()] # mutable-ok: mirrors the JSON errors array + state.errors = [str(e) for e in iter_items(event.get("errors") or ())] # mutable-ok: mirrors the JSON errors array state.structured_output = event.get("structured_output") if event.get("session_id"): state.session_id = str(event["session_id"]) diff --git a/litellm/llms/codestral/completion/handler.py b/litellm/llms/codestral/completion/handler.py index 08799c3a0a5..6bebfe2d444 100644 --- a/litellm/llms/codestral/completion/handler.py +++ b/litellm/llms/codestral/completion/handler.py @@ -2,7 +2,7 @@ ## handler file for TextCompletionCodestral Integration - https://codestral.com/ import json -from collections.abc import Callable +from collections.abc import Callable, Mapping from functools import partial from typing import Final, Protocol @@ -268,7 +268,7 @@ class CodestralTextCompletion: prompt = prompt_factory(model=model, messages=messages) ## Load Config - config: Final = litellm.CodestralTextCompletionConfig.get_config() + config: Final[Mapping[str, object]] = litellm.CodestralTextCompletionConfig.get_config() for k, v in config.items(): if ( k not in optional_params diff --git a/litellm/llms/codex/harness/transformation.py b/litellm/llms/codex/harness/transformation.py index 4512e39189e..d7388bd23fc 100644 --- a/litellm/llms/codex/harness/transformation.py +++ b/litellm/llms/codex/harness/transformation.py @@ -15,7 +15,7 @@ import re from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass, field from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final from pydantic import ConfigDict, TypeAdapter @@ -31,6 +31,7 @@ from litellm.harness.types import ( ToolCall, ToolResult, ) +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.llms.base_llm.harness.transformation import ( BaseCLIHarnessConfig, HarnessSessionSetup, @@ -77,13 +78,13 @@ class CodexStreamState: started: set[str] = field(default_factory=set) # mutable-ok: parser records announced tool items -def _tool_input(item: Mapping[str, Any]) -> tuple[str, str, Mapping[str, object], bool]: +def _tool_input(item: Mapping[str, object]) -> tuple[str, str, Mapping[str, object], bool]: """(normalized name, native name, input, builtin) for a tool-like item.""" item_type = item.get("type") if item_type == "command_execution": return "bash", "command_execution", MappingProxyType({"command": item.get("command", "")}), True if item_type == "file_change": - changes: Final = list(item.get("changes") or ()) # mutable-ok: JSON array, as codex reports it + changes: Final = list(iter_items(item.get("changes") or ())) # mutable-ok: JSON array, as codex reports it return "edit", "apply_patch", MappingProxyType({"changes": changes}), True if item_type == "web_search": return "web_search", "web_search", MappingProxyType({"query": item.get("query", "")}), True diff --git a/litellm/llms/cohere/chat/transformation.py b/litellm/llms/cohere/chat/transformation.py index fdf44d5333e..e7fadd7f375 100644 --- a/litellm/llms/cohere/chat/transformation.py +++ b/litellm/llms/cohere/chat/transformation.py @@ -1,6 +1,6 @@ import json import time -from collections.abc import AsyncIterator, Iterator, Sequence +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import httpx @@ -221,7 +221,8 @@ class CohereChatConfig(BaseConfig): headers: dict, ) -> dict: ## Load Config - for k, v in litellm.CohereChatConfig.get_config().items(): + config: Final[Mapping[str, object]] = litellm.CohereChatConfig.get_config() + for k, v in config.items(): if ( k not in optional_params ): # completion(top_k=3) > cohere_config(top_k=3) <- allows for dynamic variables to be passed in diff --git a/litellm/llms/cohere/rerank/transformation.py b/litellm/llms/cohere/rerank/transformation.py index 8b641c24e9b..7d9c8de7a07 100644 --- a/litellm/llms/cohere/rerank/transformation.py +++ b/litellm/llms/cohere/rerank/transformation.py @@ -15,6 +15,7 @@ from litellm.types.rerank import OptionalRerankParams, RerankRequest, RerankResp from ..common_utils import CohereError _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class CohereRerankConfig(BaseRerankConfig): @@ -144,7 +145,7 @@ class CohereRerankConfig(BaseRerankConfig): No transformation required, litellm follows cohere API response format """ try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception: raise CohereError(message=raw_response.text, status_code=raw_response.status_code) diff --git a/litellm/llms/cohere/rerank_v2/transformation.py b/litellm/llms/cohere/rerank_v2/transformation.py index 009a220534c..83596f90d1f 100644 --- a/litellm/llms/cohere/rerank_v2/transformation.py +++ b/litellm/llms/cohere/rerank_v2/transformation.py @@ -1,4 +1,4 @@ -from typing import Any, Final +from typing import Final from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.cohere.rerank.transformation import CohereRerankConfig @@ -39,7 +39,7 @@ class CohereRerankV2Config(CohereRerankConfig): model: str, drop_params: bool, query: str, - documents: list[str | dict[str, Any]], + documents: list[str | dict[str, object]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, diff --git a/litellm/llms/cometapi/chat/transformation.py b/litellm/llms/cometapi/chat/transformation.py index 2bf119a0a4a..ae04889bc97 100644 --- a/litellm/llms/cometapi/chat/transformation.py +++ b/litellm/llms/cometapi/chat/transformation.py @@ -6,7 +6,7 @@ Documentation: [CometAPI Documentation Link] """ from collections.abc import AsyncIterator, Iterator -from typing import Any, Final +from typing import Final import httpx @@ -137,7 +137,7 @@ class CometAPIConfig(OpenAIGPTConfig): streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> "CometAPIChatCompletionStreamingHandler": """ Get model response iterator for streaming responses """ diff --git a/litellm/llms/compactifai/chat/transformation.py b/litellm/llms/compactifai/chat/transformation.py index a02db2338d8..87fc61985e6 100644 --- a/litellm/llms/compactifai/chat/transformation.py +++ b/litellm/llms/compactifai/chat/transformation.py @@ -3,10 +3,10 @@ CompactifAI chat completion transformation """ from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, Protocol, cast import httpx -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ReadOnly, Required, TypedDict from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.common_utils import OpenAIError @@ -25,11 +25,32 @@ else: LiteLLMLoggingObj = Any +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + +class _CompactifAIToolCall(TypedDict): + function: ReadOnly[Mapping[str, str]] + + +class _CompactifAIMessage(TypedDict, total=False): + content: str | None # writable-ok: json mode moves a lone tool call's arguments into the content in place + tool_calls: Sequence[_CompactifAIToolCall] | None # writable-ok: json mode clears the tool call it moved + + +class _CompactifAIChoice(TypedDict, total=False): + message: ReadOnly[_CompactifAIMessage] + + class CompactifAIResponseFields(TypedDict, total=False): """The chat completion fields of a CompactifAI response body.""" id: ReadOnly[str] - choices: ReadOnly[Sequence[Mapping[str, object]]] + choices: ReadOnly[Required[Sequence[_CompactifAIChoice]]] created: ReadOnly[int] model: ReadOnly[str] system_fingerprint: ReadOnly[str | None] @@ -82,22 +103,19 @@ class CompactifAIChatConfig(OpenAIGPTConfig): ) ## RESPONSE OBJECT - response_json: Final = raw_response.json() + response_json: Final = cast( # cast-ok: the body passes through unvalidated, as before it was typed + CompactifAIResponseFields, _json_body(raw_response) + ) # Handle JSON mode if needed if json_mode: for choice in response_json["choices"]: message = choice.get("message") - if message and message.get("tool_calls"): - # Convert tool calls to content for JSON mode - tool_calls = message.get("tool_calls", []) - if len(tool_calls) == 1: - message["content"] = tool_calls[0]["function"].get("arguments", "") - message["tool_calls"] = None + if message and (tool_calls := message.get("tool_calls")) and len(tool_calls) == 1: + message["content"] = tool_calls[0]["function"].get("arguments", "") + message["tool_calls"] = None - response_fields: Final[CompactifAIResponseFields] = response_json - - returned_response: Final = ModelResponse(**response_fields) + returned_response: Final = ModelResponse(**response_json) # Set model name with provider prefix returned_response.model = f"compactifai/{model}" diff --git a/litellm/llms/custom_httpx/async_client_cleanup.py b/litellm/llms/custom_httpx/async_client_cleanup.py index 6ef61b6f58f..3c0cc90f30f 100644 --- a/litellm/llms/custom_httpx/async_client_cleanup.py +++ b/litellm/llms/custom_httpx/async_client_cleanup.py @@ -20,7 +20,7 @@ async def close_litellm_async_clients(): cache_dict: Final = getattr(litellm.in_memory_llm_clients_cache, "cache_dict", {}) - for key, handler in cache_dict.items(): + for handler in cache_dict.values(): # Handle BaseLLMAIOHTTPHandler instances (aiohttp_openai provider) if isinstance(handler, BaseLLMAIOHTTPHandler) and hasattr(handler, "close"): try: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index a692f3f964e..55b2c21718d 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -49,7 +49,7 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( SUBTITLE_RESPONSE_FORMATS, synthesize_subtitle_document, ) -from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header @@ -412,6 +412,27 @@ def _has_pre_call_deployment_hook(logging_obj: LiteLLMLoggingObj) -> bool: return False +def _run_responses_deployment_hook( + *, + model: str, + request_input: str | ResponseInputParam, + custom_llm_provider: str, + optional_params: Mapping[str, object], + litellm_params: GenericLiteLLMParams, +) -> Mapping[str, object]: + return run_async_function( + async_pre_call_deployment_hook, + { + **dict(litellm_params), + **optional_params, + "model": model, + "input": request_input, + "custom_llm_provider": custom_llm_provider, + }, + CallTypes.responses.value, + ) + + def _mask_presigned_request_headers(transformed_request: bytes | str | dict) -> bytes | str | dict: """A pre-signed request carries its auth inside its own ``headers`` key, which logging treats as request body (only the top-level headers channel gets masked), @@ -477,6 +498,7 @@ _DECODED_BODY_STALE_HEADERS: Final[frozenset[str]] = frozenset({"content-encodin _OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object]) _JSON_OBJECT: Final = TypeAdapter(dict[str, object]) _JSON_OBJECTS: Final = TypeAdapter(list[dict[str, object]]) +_DECODED_JSON: Final = TypeAdapter(object) def _decoded_body_headers(response: httpx.Response) -> httpx.Headers: @@ -2765,16 +2787,12 @@ class BaseLLMHTTPHandler: litellm_params, ) - modified_kwargs: Final = run_async_function( - async_pre_call_deployment_hook, - { - **dict(litellm_params), - **response_api_optional_request_params, - "model": model, - "input": input, - "custom_llm_provider": custom_llm_provider, - }, - CallTypes.responses.value, + modified_kwargs: Final = _run_responses_deployment_hook( + model=model, + request_input=input, + custom_llm_provider=custom_llm_provider, + optional_params=response_api_optional_request_params, + litellm_params=litellm_params, ) if modified_kwargs is None: return ( @@ -3753,7 +3771,7 @@ class BaseLLMHTTPHandler: response: httpx.Response, upload_url_location: str, upload_url_key: str = "upload_url", - ) -> tuple[str | None, dict | None]: + ) -> tuple[object, Mapping[str, object] | None]: """ Extract upload URL from initial file creation response. @@ -3773,8 +3791,10 @@ class BaseLLMHTTPHandler: else: # Response body style (e.g., Manus, S3 presigned URLs) try: - response_data: Final = response.json() - upload_url = response_data.get(upload_url_key) + response_data: Final = as_str_mapping(_DECODED_JSON.validate_python(response.json())) + if response_data is None: + return None, None + upload_url: Final = response_data.get(upload_url_key) return upload_url, response_data if upload_url else None except Exception: return None, None @@ -3940,7 +3960,7 @@ class BaseLLMHTTPHandler: elif isinstance(transformed_request, dict) and "file" in transformed_request: # Handle multipart form-data uploads (e.g., Anthropic Files API) # The dict contains tuples suitable for httpx's `files` parameter - file_request: Final = cast(dict[str, Any], transformed_request) + file_request: Final = cast(dict[str, object], transformed_request) upload_response = sync_httpx_client.post( url=api_base, headers=headers, diff --git a/litellm/llms/custom_httpx/mock_transport.py b/litellm/llms/custom_httpx/mock_transport.py index 0322cceec0f..6acadb8e98a 100644 --- a/litellm/llms/custom_httpx/mock_transport.py +++ b/litellm/llms/custom_httpx/mock_transport.py @@ -12,6 +12,9 @@ import uuid from typing import Final import httpx +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.core_helpers import decoded_json_object # --------------------------------------------------------------------------- # Pre-built response templates @@ -22,7 +25,7 @@ def _mock_id() -> str: return f"chatcmpl-mock-{uuid.uuid4().hex[:8]}" -def _chat_completion_json(model: str) -> dict: +def _chat_completion_json(model: object) -> dict: """Return a minimal valid ChatCompletion object.""" return { "id": _mock_id(), @@ -55,6 +58,8 @@ _JSON_HEADERS: Final = { "content-type": "application/json", } +_DECODED_JSON: Final = TypeAdapter(object) + class MockOpenAITransport(httpx.AsyncBaseTransport, httpx.BaseTransport): """ @@ -64,13 +69,13 @@ class MockOpenAITransport(httpx.AsyncBaseTransport, httpx.BaseTransport): """ @staticmethod - def _parse_request(request: httpx.Request) -> tuple[str, bool]: + def _parse_request(request: httpx.Request) -> tuple[object, bool]: """Extract model from the request body.""" try: - body: Final = json.loads(request.content) + body: Final = _DECODED_JSON.validate_python(json.loads(request.content)) except (json.JSONDecodeError, ValueError): return ("mock-model", False) - model: Final = body.get("model", "mock-model") + model: Final = decoded_json_object(body).get("model", "mock-model") return (model, False) async def handle_async_request(self, request: httpx.Request) -> httpx.Response: diff --git a/litellm/llms/dashscope/cost_calculator.py b/litellm/llms/dashscope/cost_calculator.py index 3f388637a83..55d6fb1a0e2 100644 --- a/litellm/llms/dashscope/cost_calculator.py +++ b/litellm/llms/dashscope/cost_calculator.py @@ -55,10 +55,9 @@ def _extract_token_breakdown(usage: Usage) -> TokenBreakdown: ) -def _flat_rate(model_info: ModelInfo, cost_key: str, fallback_cost_key: str) -> float: - value: Final = model_info.get(cost_key) +def _flat_rate(value: float | None, fallback_value: float | None) -> float: if value is None: - return float(model_info.get(fallback_cost_key) or 0.0) + return float(fallback_value or 0.0) return float(value) @@ -66,8 +65,12 @@ def _flat_rates(model_info: ModelInfo) -> TokenRates: reasoning_rate: Final = model_info.get("output_cost_per_reasoning_token") return TokenRates( input_rate=float(model_info.get("input_cost_per_token") or 0.0), - cache_read_rate=_flat_rate(model_info, "cache_read_input_token_cost", "input_cost_per_token"), - cache_creation_rate=_flat_rate(model_info, "cache_creation_input_token_cost", "input_cost_per_token"), + cache_read_rate=_flat_rate( + model_info.get("cache_read_input_token_cost"), model_info.get("input_cost_per_token") + ), + cache_creation_rate=_flat_rate( + model_info.get("cache_creation_input_token_cost"), model_info.get("input_cost_per_token") + ), output_rate=float(model_info.get("output_cost_per_token") or 0.0), reasoning_rate=None if reasoning_rate is None else float(reasoning_rate), ) diff --git a/litellm/llms/dashscope/image_generation/transformation.py b/litellm/llms/dashscope/image_generation/transformation.py index 6a172e1e074..d1faa6c6cd0 100644 --- a/litellm/llms/dashscope/image_generation/transformation.py +++ b/litellm/llms/dashscope/image_generation/transformation.py @@ -49,6 +49,7 @@ else: _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" @@ -187,7 +188,7 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): ) try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Failed to parse DashScope image generation response: {e}", diff --git a/litellm/llms/databricks/embed/transformation.py b/litellm/llms/databricks/embed/transformation.py index ddaf2bbb621..0feb2aea5a0 100644 --- a/litellm/llms/databricks/embed/transformation.py +++ b/litellm/llms/databricks/embed/transformation.py @@ -3,6 +3,7 @@ Translates from OpenAI's `/v1/embeddings` to Databricks' `/embeddings` """ import types +from collections.abc import Mapping from typing import Final @@ -16,16 +17,17 @@ class DatabricksEmbeddingConfig: ) def __init__(self, instruction: str | None = None) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/databricks/messages/transformation.py b/litellm/llms/databricks/messages/transformation.py index cb8d2b652a2..f5ccbd92fb4 100644 --- a/litellm/llms/databricks/messages/transformation.py +++ b/litellm/llms/databricks/messages/transformation.py @@ -3,6 +3,7 @@ from collections.abc import AsyncIterator, Mapping from typing import Final import httpx +from pydantic import TypeAdapter import litellm from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items @@ -25,6 +26,7 @@ DROP_CONTAINER_WARNING: Final = ( "and the chat completions bridge it replaces never forwarded it." ) BEDROCK_INVOCATION_METRICS_FIELD: Final = "amazon-bedrock-invocationMetrics" +_DECODED_JSON: Final = TypeAdapter(object) def _is_unsupported_server_tool(tool: object) -> bool: @@ -57,7 +59,7 @@ def _without_unsupported_server_tools(params: Mapping[str, object]) -> dict[str, def _without_invocation_metrics(sse_line: bytes) -> bytes: if not sse_line.startswith(b"data:") or BEDROCK_INVOCATION_METRICS_FIELD.encode() not in sse_line: return sse_line - event: Final = as_str_mapping(json.loads(sse_line.removeprefix(b"data:"))) + event: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(sse_line.removeprefix(b"data:")))) if event is None: return sse_line kept_fields: Final = {key: value for key, value in event.items() if key != BEDROCK_INVOCATION_METRICS_FIELD} diff --git a/litellm/llms/deepagents/harness/sandbox_backend.py b/litellm/llms/deepagents/harness/sandbox_backend.py index d39690ccdd1..8ad90faa2b4 100644 --- a/litellm/llms/deepagents/harness/sandbox_backend.py +++ b/litellm/llms/deepagents/harness/sandbox_backend.py @@ -22,7 +22,7 @@ import shlex import uuid from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence from types import MappingProxyType -from typing import Any, Final, TypeVar +from typing import Final, TypeVar from deepagents.backends.protocol import ( DeleteResult, @@ -171,7 +171,7 @@ class SandboxBackend(SandboxBackendProtocol): # pyright: ignore[reportUntypedBa # -- sync bridge (deepagents only calls these outside the event loop) --- - def _sync(self, coro: Coroutine[Any, Any, T]) -> T: + def _sync(self, coro: Coroutine[object, object, T]) -> T: try: running = asyncio.get_running_loop() except RuntimeError: diff --git a/litellm/llms/deepagents/harness/transformation.py b/litellm/llms/deepagents/harness/transformation.py index cb80e3e12f7..54a6a5573ac 100644 --- a/litellm/llms/deepagents/harness/transformation.py +++ b/litellm/llms/deepagents/harness/transformation.py @@ -11,11 +11,13 @@ from __future__ import annotations import itertools import json -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol +from pydantic import TypeAdapter + from litellm.harness.options import DeepAgentsOptions from litellm.harness.types import ( Capabilities, @@ -41,6 +43,7 @@ DEEPAGENTS_DEFAULT_RECURSION_LIMIT: Final = 1000 _MODEL_NODE: Final = "model" # Only these graph nodes produce new messages; middleware hooks may re-emit history. _EVENT_NODES: Final = frozenset({"model", "tools"}) +_TOOL_CALLS: Final = TypeAdapter(Iterable[Mapping[str, object]]) NORMALIZED_TO_NATIVE: Final[Mapping[str, str]] = MappingProxyType( { @@ -208,7 +211,7 @@ def update_events( def _message_events(message: object, skip_tools: frozenset[str]) -> tuple[Event, ...]: kind = getattr(message, "type", None) if kind == "ai": - calls = getattr(message, "tool_calls", None) or () + calls: Final = _TOOL_CALLS.validate_python(getattr(message, "tool_calls", None) or ()) return tuple(tool_call_event(call) for call in calls if call.get("name") not in skip_tools) if kind == "tool" and getattr(message, "name", None) not in skip_tools: status: Final[object] = getattr(message, "status", None) diff --git a/litellm/llms/deepgram/audio_transcription/transformation.py b/litellm/llms/deepgram/audio_transcription/transformation.py index a1c3021c9f5..1c8a056116f 100644 --- a/litellm/llms/deepgram/audio_transcription/transformation.py +++ b/litellm/llms/deepgram/audio_transcription/transformation.py @@ -2,12 +2,13 @@ Translates from OpenAI's `/v1/audio/transcriptions` to Deepgram's `/v1/listen` """ -from collections.abc import Iterable, Mapping -from typing import Final +from collections.abc import Iterable, Mapping, Sequence +from typing import Final, Protocol, cast from urllib.parse import urlencode from httpx import Headers, Response -from pydantic import ConfigDict, TypeAdapter +from pydantic import ConfigDict, InstanceOf, TypeAdapter +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.litellm_core_utils.audio_utils.utils import process_audio_file from litellm.litellm_core_utils.url_utils import join_api_base @@ -27,6 +28,31 @@ from ..common_utils import DeepgramException _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_RESPONSE_BODY: Final = TypeAdapter(InstanceOf[dict[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_WORD_LIST: Final = TypeAdapter(InstanceOf[list[object]], config=ConfigDict(hide_input_in_errors=True)) + + +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + +class _DeepgramAlternative(TypedDict): + transcript: ReadOnly[str] + words: ReadOnly[Sequence[Mapping[str, object]]] + paragraphs: ReadOnly[NotRequired[Mapping[str, str]]] + + +class _DeepgramChannel(TypedDict): + alternatives: ReadOnly[Sequence[_DeepgramAlternative]] + + +class _DeepgramResponse(TypedDict): + results: ReadOnly[Mapping[str, Sequence[_DeepgramChannel]]] + metadata: ReadOnly[Mapping[str, float]] class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): @@ -82,7 +108,10 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): Transforms the raw response from Deepgram to the TranscriptionResponse format """ try: - response_json: Final = raw_response.json() + body: Final = _json_body(raw_response) + response_json: Final = cast( # cast-ok: the body passes through unvalidated, as before it was typed + _DeepgramResponse, body + ) # Get the first alternative from the first channel first_channel: Final = response_json["results"]["channels"][0] @@ -102,7 +131,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): text = first_alternative["paragraphs"]["transcript"] else: # Diarization without paragraphs: reconstruct from words - text = self._reconstruct_diarized_transcript(first_alternative["words"]) + text = self._reconstruct_diarized_transcript(_WORD_LIST.validate_python(first_alternative["words"])) # Create TranscriptionResponse object response: Final = TranscriptionResponse(text=text) @@ -124,7 +153,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): ] # Store full response in hidden params - response.hidden_params = response_json + response.hidden_params = _RESPONSE_BODY.validate_python(body) return response diff --git a/litellm/llms/deepinfra/rerank/transformation.py b/litellm/llms/deepinfra/rerank/transformation.py index 5196c44d504..62a780396a1 100644 --- a/litellm/llms/deepinfra/rerank/transformation.py +++ b/litellm/llms/deepinfra/rerank/transformation.py @@ -6,6 +6,7 @@ from collections.abc import Mapping, Sequence from typing import Final, Protocol import httpx +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm._uuid import uuid @@ -24,6 +25,8 @@ from litellm.types.rerank import ( RerankTokens, ) +_DECODED_JSON: Final = TypeAdapter(object) + class _DeepinfraInferenceStatus(TypedDict, total=False): """The ``inference_status`` block of a DeepInfra rerank response.""" @@ -251,7 +254,7 @@ class DeepinfraRerankConfig(BaseRerankConfig): try: error_data = error_message if isinstance(error_message, str): - error_data = json.loads(error_message) + error_data = _DECODED_JSON.validate_python(json.loads(error_message)) if isinstance(error_data, dict): # Check for {"detail": {"error": "..."}} detail: Final = error_data.get("detail") diff --git a/litellm/llms/deepseek/messages/transformation.py b/litellm/llms/deepseek/messages/transformation.py index 6ddee99aea3..3b2dd5af260 100644 --- a/litellm/llms/deepseek/messages/transformation.py +++ b/litellm/llms/deepseek/messages/transformation.py @@ -2,8 +2,10 @@ DeepSeek Anthropic-compatible messages transformation config. """ -from collections.abc import Sequence -from typing import Any, Final +from collections.abc import Mapping, Sequence +from typing import Final + +from pydantic import TypeAdapter import litellm from litellm.litellm_core_utils.url_utils import join_api_base @@ -13,6 +15,24 @@ from litellm.llms.anthropic.pass_through.messages.transformation import ( from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams +_TOOL_LIST: Final = TypeAdapter(list[object]) +_TOOL_FIELDS: Final = TypeAdapter(dict[object, object]) + + +def _custom_tool_without_type(tool: object) -> Mapping[object, object] | None: + if not isinstance(tool, dict): + return None + fields: Final = _TOOL_FIELDS.validate_python(tool) + if fields.get("type") == "custom": + fields.pop("type", None) + return fields + return None + + +def _sanitized_tool(tool: object) -> object: + stripped: Final = _custom_tool_without_type(tool) + return tool if stripped is None else stripped + class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig): """ @@ -96,19 +116,11 @@ class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig): return join_api_base(base_url, "v1/messages") @staticmethod - def _sanitize_tools_for_deepseek(tools: Any) -> object: + def _sanitize_tools_for_deepseek(tools: object) -> object: if not isinstance(tools, list): return tools - sanitized_tools: Final = [] - for tool in tools: - if isinstance(tool, dict) and tool.get("type") == "custom": - sanitized_tool = dict(tool) - sanitized_tool.pop("type", None) - sanitized_tools.append(sanitized_tool) - else: - sanitized_tools.append(tool) - return sanitized_tools + return [_sanitized_tool(tool) for tool in _TOOL_LIST.validate_python(tools)] def transform_anthropic_messages_request( self, diff --git a/litellm/llms/deprecated_providers/palm.py b/litellm/llms/deprecated_providers/palm.py index 785cffa48ea..b4a32630c42 100644 --- a/litellm/llms/deprecated_providers/palm.py +++ b/litellm/llms/deprecated_providers/palm.py @@ -1,4 +1,5 @@ import types +from collections.abc import Mapping from typing import Final @@ -41,16 +42,17 @@ class PalmConfig: top_p: float | None = None, max_output_tokens: int | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/edenai/embedding/transformation.py b/litellm/llms/edenai/embedding/transformation.py index c79a6839434..13fa7142916 100644 --- a/litellm/llms/edenai/embedding/transformation.py +++ b/litellm/llms/edenai/embedding/transformation.py @@ -5,7 +5,7 @@ per-request cost at the top level of the body. Docs: https://www.edenai.co/docs/v3/llms/embeddings """ -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, Protocol, cast import httpx @@ -24,6 +24,14 @@ if TYPE_CHECKING: _SUPPORTED_PARAMS: Final = ("dimensions", "encoding_format", "user") +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + class EdenAIEmbeddingConfig(BaseEmbeddingConfig): def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited contract return list(_SUPPORTED_PARAMS) @@ -80,10 +88,14 @@ class EdenAIEmbeddingConfig(BaseEmbeddingConfig): optional_params: dict[str, object], # mutable-ok: inherited contract litellm_params: dict[str, object], # mutable-ok: inherited contract ) -> EmbeddingResponse: - body: Final = raw_response.json() + body: Final = _json_body(raw_response) logging_obj.post_call(original_response=body) response: Final[EmbeddingResponse] = convert_to_model_response_object( - response_object=body, model_response_object=model_response, response_type="embedding" + response_object=cast( # cast-ok: the body passes through unvalidated, as before it was typed + dict[str, object], body + ), + model_response_object=model_response, + response_type="embedding", ) set_response_cost_in_hidden_params(response, reported_cost(body)) return response diff --git a/litellm/llms/edenai/image_generation/transformation.py b/litellm/llms/edenai/image_generation/transformation.py index e2a0b8684f6..a0c4e1b0458 100644 --- a/litellm/llms/edenai/image_generation/transformation.py +++ b/litellm/llms/edenai/image_generation/transformation.py @@ -5,7 +5,7 @@ for every image model in the catalog with the real per-request cost at the top l Docs: https://www.edenai.co/docs/v3/llms/image-generation """ -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, Protocol, cast import httpx @@ -36,6 +36,14 @@ _SUPPORTED_PARAMS: Final[tuple[OpenAIImageGenerationOptionalParams, ...]] = ( ) +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + class EdenAIImageGenerationConfig(BaseImageGenerationConfig): def get_supported_openai_params( self, model: str @@ -97,10 +105,14 @@ class EdenAIImageGenerationConfig(BaseImageGenerationConfig): api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: - body: Final = raw_response.json() + body: Final = _json_body(raw_response) logging_obj.post_call(original_response=body) response: Final[ImageResponse] = convert_to_model_response_object( - response_object=body, model_response_object=model_response, response_type="image_generation" + response_object=cast( # cast-ok: the body passes through unvalidated, as before it was typed + dict[str, object], body + ), + model_response_object=model_response, + response_type="image_generation", ) set_response_cost_in_hidden_params(response, reported_cost(body)) return response diff --git a/litellm/llms/elevenlabs/audio_transcription/transformation.py b/litellm/llms/elevenlabs/audio_transcription/transformation.py index 5a673b5ac41..3dc42f35d3c 100644 --- a/litellm/llms/elevenlabs/audio_transcription/transformation.py +++ b/litellm/llms/elevenlabs/audio_transcription/transformation.py @@ -3,7 +3,7 @@ Translates from OpenAI's `/v1/audio/transcriptions` to ElevenLabs's `/v1/speech- """ from collections.abc import Iterable, Mapping -from typing import Final +from typing import Final, Protocol from httpx import Headers, Response from pydantic import ConfigDict, TypeAdapter @@ -27,6 +27,15 @@ from ..common_utils import ElevenLabsException _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_HIDDEN_PARAMS: Final = TypeAdapter(dict[str, object]) + + +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): @@ -120,7 +129,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): Transforms the raw response from ElevenLabs to the TranscriptionResponse format """ try: - response_json: Final = raw_response.json() + response_json: Final = _json_body(raw_response) response_object: Final = _JSON_OBJECT.validate_python(response_json) # Extract the main transcript text @@ -135,20 +144,18 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): # Map ElevenLabs words to OpenAI format if "words" in response_object: - response["words"] = [] - for word_data in _JSON_OBJECTS.validate_python(response_object["words"]): - # Only include actual words, skip spacing and audio events - if word_data.get("type") == "word": - response["words"].append( - { - "word": word_data.get("text", ""), - "start": word_data.get("start", 0), - "end": word_data.get("end", 0), - } - ) + response["words"] = [ + { + "word": word_data.get("text", ""), + "start": word_data.get("start", 0), + "end": word_data.get("end", 0), + } + for word_data in _JSON_OBJECTS.validate_python(response_object["words"]) + if word_data.get("type") == "word" + ] # Store full response in hidden params - response.hidden_params = response_json + response.hidden_params = _HIDDEN_PARAMS.validate_python(response_json) return response diff --git a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py index 0fdd89aec91..fc215e3d94e 100644 --- a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py +++ b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py @@ -19,6 +19,7 @@ else: LiteLLMLoggingObj = Any _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class FalAIFluxProV11UltraConfig(FalAIBaseConfig): @@ -222,7 +223,7 @@ class FalAIFluxProV11UltraConfig(FalAIBaseConfig): } """ try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", diff --git a/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py b/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py index 038a06c5749..26b257f5c89 100644 --- a/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py +++ b/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py @@ -18,6 +18,7 @@ else: LiteLLMLoggingObj = Any _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class FalAIRecraftV3Config(FalAIBaseConfig): @@ -195,7 +196,7 @@ class FalAIRecraftV3Config(FalAIBaseConfig): } """ try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", diff --git a/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py index 30d51cc8084..fdd552efd73 100644 --- a/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py +++ b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py @@ -19,6 +19,7 @@ else: LiteLLMLoggingObj = Any _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class FalAIStableDiffusionConfig(FalAIBaseConfig): @@ -236,7 +237,7 @@ class FalAIStableDiffusionConfig(FalAIBaseConfig): } """ try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", diff --git a/litellm/llms/featherless_ai/chat/transformation.py b/litellm/llms/featherless_ai/chat/transformation.py index a68261c58e9..4bf0486f723 100644 --- a/litellm/llms/featherless_ai/chat/transformation.py +++ b/litellm/llms/featherless_ai/chat/transformation.py @@ -1,3 +1,5 @@ +from collections.abc import Mapping +from types import MappingProxyType from typing import Final import litellm @@ -42,7 +44,7 @@ class FeatherlessAIConfig(OpenAIGPTConfig): tool_choice: str | None = None, tools: list | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/fireworks_ai/rerank/transformation.py b/litellm/llms/fireworks_ai/rerank/transformation.py index 964f85c821e..84c93974cd7 100644 --- a/litellm/llms/fireworks_ai/rerank/transformation.py +++ b/litellm/llms/fireworks_ai/rerank/transformation.py @@ -44,6 +44,7 @@ class _FireworksAIResultFields(LiteLLMBaseModel): _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) _STR: Final = TypeAdapter(str) +_DECODED_JSON: Final = TypeAdapter(object) class FireworksAIRerankConfig(FireworksAIMixin, BaseRerankConfig): @@ -200,7 +201,7 @@ class FireworksAIRerankConfig(FireworksAIMixin, BaseRerankConfig): Transform Fireworks AI rerank response to LiteLLM RerankResponse format """ try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Failed to parse response: {e}", diff --git a/litellm/llms/gdc/chat/transformation.py b/litellm/llms/gdc/chat/transformation.py index cd9ffe9e03e..680db52b283 100644 --- a/litellm/llms/gdc/chat/transformation.py +++ b/litellm/llms/gdc/chat/transformation.py @@ -10,13 +10,17 @@ from collections.abc import Callable from typing import Final, Protocol from urllib.parse import urlsplit +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict, Unpack import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig from litellm.types.llms.openai import AllMessageValues +_DECODED_JSON: Final = TypeAdapter(object) + class _OpenAIGPTConfigOptions(TypedDict, total=False): """The sampling defaults ``OpenAIGPTConfig.__init__`` accepts and stashes on the class.""" @@ -203,10 +207,10 @@ class GDCGeminiConfig(OpenAILikeChatConfig): import google.auth try: - json_obj: Final = json.loads(api_key) + json_obj: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(api_key))) except json.JSONDecodeError: return None, False - if not isinstance(json_obj, dict) or json_obj.get("type") != self._GDCH_CREDENTIAL_TYPE: + if json_obj is None or json_obj.get("type") != self._GDCH_CREDENTIAL_TYPE: raise ValueError( "GDC only accepts a GDCH service account credential as a JSON api_key " '(expected "type": "gdch_service_account"). Other Google credential types are ' diff --git a/litellm/llms/gemini/agents/transformation.py b/litellm/llms/gemini/agents/transformation.py index c19bd9ce87c..f7352da5fb6 100644 --- a/litellm/llms/gemini/agents/transformation.py +++ b/litellm/llms/gemini/agents/transformation.py @@ -10,11 +10,12 @@ Proxies the Gemini v1beta Agents API: """ from collections.abc import Mapping -from typing import Any, Final +from typing import Any, Final, Protocol import httpx from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import decoded_json_object from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.agents.transformation import BaseAgentsAPIConfig from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo @@ -29,6 +30,15 @@ from litellm.types.agents import ( # create-agent body verbatim. _GEMINI_AGENT_BODY_KEYS: Final = ("base_agent", "instructions", "base_environment") + +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + # LiteLLM-internal keys that must never be forwarded to Gemini. _LITELLM_INTERNAL_KEYS: Final = frozenset( { @@ -156,18 +166,19 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): """ self._raise_for_status(raw_response) try: - data: dict[str, Any] = raw_response.json() + decoded: object = _json_body(raw_response) except Exception: verbose_logger.warning( "GeminiAgentsConfig: non-JSON create response (status=%d).", raw_response.status_code, ) - data = {"id": name} + decoded = {"id": name} + data: Final = dict(decoded_json_object(decoded)) # Gemini uses "id" as the identifier; normalise to both fields. data.setdefault("id", name) data.setdefault("name", data["id"]) verbose_logger.debug("GeminiAgentsConfig create response: %s", data) - return AgentCreateResponse(**data) + return AgentCreateResponse.model_validate(data) # ------------------------------------------------------------------ # # LIST # @@ -192,13 +203,16 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): ) -> AgentListResponse: self._raise_for_status(raw_response) try: - data = raw_response.json() + decoded: object = _json_body(raw_response) except Exception: - data = {} + decoded = {} + data: Final = decoded_json_object(decoded) verbose_logger.debug("GeminiAgentsConfig list response: %s", data) - return AgentListResponse( - agents=data.get("agents", []), - next_page_token=data.get("nextPageToken"), + return AgentListResponse.model_validate( + { + "agents": data.get("agents", []), + "next_page_token": data.get("nextPageToken"), + } ) # ------------------------------------------------------------------ # @@ -222,13 +236,14 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): """Same shape as create response — Gemini returns "id" as identifier.""" self._raise_for_status(raw_response) try: - data = raw_response.json() + decoded: object = _json_body(raw_response) except Exception: - data = {"id": name} + decoded = {"id": name} + data: Final = dict(decoded_json_object(decoded)) data.setdefault("id", name) data.setdefault("name", data["id"]) verbose_logger.debug("GeminiAgentsConfig get response: %s", data) - return AgentCreateResponse(**data) + return AgentCreateResponse.model_validate(data) # ------------------------------------------------------------------ # # DELETE # @@ -285,11 +300,14 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): """ self._raise_for_status(raw_response) try: - data = raw_response.json() + decoded: object = _json_body(raw_response) except Exception: - data = {} + decoded = {} + data: Final = decoded_json_object(decoded) verbose_logger.debug("GeminiAgentsConfig list_versions response for '%s': %s", name, data) - return AgentVersionsResponse( - agent_versions=data.get("agentVersions", []), - next_page_token=data.get("nextPageToken"), + return AgentVersionsResponse.model_validate( + { + "agent_versions": data.get("agentVersions", []), + "next_page_token": data.get("nextPageToken"), + } ) diff --git a/litellm/llms/gemini/audio_transcription/transformation.py b/litellm/llms/gemini/audio_transcription/transformation.py index ea116a7e9de..3457b22faa4 100644 --- a/litellm/llms/gemini/audio_transcription/transformation.py +++ b/litellm/llms/gemini/audio_transcription/transformation.py @@ -3,6 +3,7 @@ from collections.abc import Mapping, Sequence from typing import Final from httpx import Headers, Response +from pydantic import TypeAdapter from litellm.litellm_core_utils.audio_utils.subtitle_utils import SUBTITLE_RESPONSE_FORMATS from litellm.litellm_core_utils.audio_utils.utils import ( @@ -36,6 +37,7 @@ from litellm.types.utils import ( INTERACTIONS_API_REVISION: Final = "2026-05-20" WORD_INFO_ANNOTATION_TYPE: Final = "word_info" +_DECODED_JSON: Final = TypeAdapter(object) class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig): @@ -132,7 +134,7 @@ class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig): raw_response: Response, ) -> TranscriptionResponse: try: - response_json: Final = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except ValueError: raise GeminiError( status_code=raw_response.status_code, diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index 13df3921d13..9634c580858 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -2,9 +2,9 @@ import base64 import datetime import json import math -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from types import MappingProxyType -from typing import Any, Final +from typing import Any, Final, cast import httpx from pydantic import JsonValue, TypeAdapter @@ -389,7 +389,7 @@ class GeminiModelInfo(BaseLLMModelInfo): def get_base_model(model: str) -> str | None: return model.replace("gemini/", "") - def process_model_name(self, models: list[dict[str, str]]) -> list[str]: + def process_model_name(self, models: Iterable[Mapping[str, str]]) -> list[str]: litellm_model_names: Final = [] for model in models: stripped_model_name = model["name"].replace("models/", "") @@ -416,9 +416,11 @@ class GeminiModelInfo(BaseLLMModelInfo): f"Failed to fetch models from Gemini. Status code: {response.status_code}, Response: {response.json()}" ) - models: Final[list[dict[str, str]]] = response.json()["models"] + listing: Final = cast( # cast-ok: the body passes through unvalidated, as before it was typed + Mapping[str, Iterable[Mapping[str, str]]], _JSON_VALUE.validate_python(response.json()) + ) - litellm_model_names: Final = self.process_model_name(models) + litellm_model_names: Final = self.process_model_name(listing["models"]) return litellm_model_names def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: diff --git a/litellm/llms/gemini/google_genai/guardrail_translation/handler.py b/litellm/llms/gemini/google_genai/guardrail_translation/handler.py index 44cdac11325..34007bc8fd4 100644 --- a/litellm/llms/gemini/google_genai/guardrail_translation/handler.py +++ b/litellm/llms/gemini/google_genai/guardrail_translation/handler.py @@ -15,7 +15,10 @@ from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, Optional +from pydantic import TypeAdapter + from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, StreamTransformSink, @@ -28,6 +31,7 @@ if TYPE_CHECKING: from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth _EMPTY_REQUEST_DATA: Final[Mapping[str, object]] = MappingProxyType({}) +_DECODED_JSON: Final = TypeAdapter(object) def _field(container: object, name: str) -> object | None: @@ -108,12 +112,10 @@ def _write_back_texts(text_parts: Sequence[object], guardrailed_texts: Sequence[ def _parse_json_dict_or_none(payload: str) -> Mapping[str, object] | None: try: - parsed: Final = json.loads(payload) + parsed: Final = _DECODED_JSON.validate_python(json.loads(payload)) except json.JSONDecodeError: return None - if isinstance(parsed, dict): - return parsed - return None + return as_str_mapping(parsed) def _sse_payload_texts(sse_text: str) -> tuple[str, ...]: diff --git a/litellm/llms/gemini/google_genai/transformation.py b/litellm/llms/gemini/google_genai/transformation.py index bc3180a2129..0e29df36e80 100644 --- a/litellm/llms/gemini/google_genai/transformation.py +++ b/litellm/llms/gemini/google_genai/transformation.py @@ -3,7 +3,7 @@ Transformation for Calling Google models in their native format. """ from copy import deepcopy -from typing import TYPE_CHECKING, Any, Final, Literal, cast +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast import httpx @@ -36,6 +36,14 @@ else: from ..common_utils import get_api_key_from_env +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): """ Configuration for calling Google models in their native format. @@ -382,7 +390,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): from litellm.types.google_genai.main import GenerateContentResponse try: - response = raw_response.json() + decoded: Final = _json_body(raw_response) except Exception as e: raise self.get_error_class( error_message=f"Error transforming generate content response: {e}", @@ -391,7 +399,9 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): ) logging_obj.model_call_details["httpx_response"] = raw_response - response = self.convert_citation_sources_to_citations(response) + response: Final = self.convert_citation_sources_to_citations( + cast(dict[str, object], decoded) # cast-ok: the body passes through unvalidated, as before it was typed + ) return GenerateContentResponse(**response) diff --git a/litellm/llms/gemini/image_edit/transformation.py b/litellm/llms/gemini/image_edit/transformation.py index 450c38a2206..d48a66f1f93 100644 --- a/litellm/llms/gemini/image_edit/transformation.py +++ b/litellm/llms/gemini/image_edit/transformation.py @@ -1,12 +1,13 @@ import base64 from collections.abc import Mapping from io import BufferedReader, BytesIO -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, Protocol, cast import httpx from httpx._types import RequestFiles from litellm.images.utils import ImageEditRequestUtils +from litellm.litellm_core_utils.core_helpers import decoded_json_object from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.llms.gemini.common_utils import ( @@ -18,6 +19,7 @@ from litellm.llms.gemini.image_usage_transformation import ( ) from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.llms.vertex_ai import GenerateContentResponseBody from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( FileTypes, @@ -34,6 +36,14 @@ else: LiteLLMLoggingObj = Any +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + class GeminiImageEditConfig(BaseImageEditConfig): DEFAULT_BASE_URL: str = "https://generativelanguage.googleapis.com/v1beta" SUPPORTED_PARAMS: list[str] = ["n", "size", "imageConfig"] @@ -125,7 +135,7 @@ class GeminiImageEditConfig(BaseImageEditConfig): ) -> ImageResponse: model_response: Final = ImageResponse() try: - response_json: Final = raw_response.json() + response_json: Final = _json_body(raw_response) except Exception as exc: raise self.get_error_class( error_message=f"Error transforming image edit response: {exc}", @@ -133,7 +143,10 @@ class GeminiImageEditConfig(BaseImageEditConfig): headers=raw_response.headers, ) - candidates: Final = response_json.get("candidates", []) + payload: Final = cast( # cast-ok: the body passes through unvalidated, as before it was typed + GenerateContentResponseBody, response_json + ) + candidates: Final = payload.get("candidates", []) data_list: Final[list[ImageObject]] = [] for candidate in candidates: @@ -141,17 +154,18 @@ class GeminiImageEditConfig(BaseImageEditConfig): parts = content.get("parts", []) for part in parts: inline_data = part.get("inlineData") - if inline_data and inline_data.get("data"): + image_data = inline_data.get("data") if inline_data else None + if image_data: data_list.append( ImageObject( - b64_json=inline_data["data"], + b64_json=image_data, url=None, ) ) model_response.data = cast(list[OpenAIImage], data_list) - if "usageMetadata" in response_json: - model_response.usage = transform_gemini_image_usage(response_json["usageMetadata"]) + if "usageMetadata" in payload: + model_response.usage = transform_gemini_image_usage(dict(decoded_json_object(payload["usageMetadata"]))) return model_response def _prepare_inline_image_parts(self, image: FileTypes | list[FileTypes]) -> list[dict[str, object]]: diff --git a/litellm/llms/gemini/image_generation/transformation.py b/litellm/llms/gemini/image_generation/transformation.py index 1aa3d074b1a..1f986b11af2 100644 --- a/litellm/llms/gemini/image_generation/transformation.py +++ b/litellm/llms/gemini/image_generation/transformation.py @@ -37,6 +37,7 @@ else: _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) _JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) _JSON_CONTAINER: Final[TypeAdapter[Mapping[str, object] | str | Sequence[object]]] = TypeAdapter( Mapping[str, object] | str | Sequence[object], config=ConfigDict(hide_input_in_errors=True) ) @@ -188,7 +189,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): Transform Google AI Imagen response to litellm ImageResponse format """ try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", @@ -226,9 +227,9 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): ) # Extract usage metadata for Gemini models - if "usageMetadata" in response_data: + if "usageMetadata" in response_object: model_response.usage = transform_gemini_image_usage( - _JSON_DICT.validate_python(response_data["usageMetadata"]) + _JSON_DICT.validate_python(response_object["usageMetadata"]) ) web_search_requests: Final = get_gemini_image_web_search_requests(response_object) if web_search_requests and model_response.usage is not None: diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 9af7edeac86..6f778ab2761 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -5,13 +5,14 @@ This file contains the transformation logic for the Gemini realtime API. import json from collections import OrderedDict from collections.abc import Mapping, Sequence -from typing import Any, Final, cast +from typing import Final, cast from typing_extensions import ReadOnly, Required, TypedDict import litellm from litellm import verbose_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig @@ -167,16 +168,17 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): return True @staticmethod - def _usage_detail_alias(details: Mapping[str, int | None] | None, defaults: dict[str, int]) -> dict[str, int]: - if not isinstance(details, dict): + def _usage_detail_alias(details: object, defaults: dict[str, int]) -> dict[str, object]: + detail_mapping: Final = as_str_mapping(details) + if detail_mapping is None or not isinstance(details, dict): return dict(defaults) return { **defaults, - **{key: value for key, value in details.items() if value is not None}, + **{key: value for key, value in detail_mapping.items() if value is not None}, } @staticmethod - def _add_pipecat_usage_detail_aliases(usage_dict: dict[str, Any]) -> dict[str, object]: + def _add_pipecat_usage_detail_aliases(usage_dict: dict[str, object]) -> dict[str, object]: usage_dict.setdefault( "input_token_details", GeminiRealtimeConfig._usage_detail_alias( diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py index 0656095927c..3f6b900fd01 100644 --- a/litellm/llms/gemini/videos/transformation.py +++ b/litellm/llms/gemini/videos/transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from httpx._types import RequestFiles +from pydantic import TypeAdapter import litellm from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS @@ -37,6 +38,8 @@ else: LiteLLMLoggingObj = Any BaseLLMException = Any +_DECODED_JSON: Final = TypeAdapter(object) + def _convert_image_to_gemini_format(image_file) -> dict[str, str]: """ @@ -60,7 +63,7 @@ def _convert_image_to_gemini_format(image_file) -> dict[str, str]: def _json_payload(raw_response: httpx.Response) -> object: """Read an HTTP response body as an opaque JSON payload.""" - return raw_response.json() + return _DECODED_JSON.validate_python(raw_response.json()) def _usage_video_resolution_from_parameters( diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index 936e6339865..65921b628fb 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -273,7 +273,7 @@ class GigaChatConfig(BaseConfig): # Default to None (don't set function_call) return None - def _upload_image(self, image_url: str) -> str | None: + def _upload_image(self, image_url: str) -> object: """ Upload image to GigaChat and return file_id. @@ -293,7 +293,7 @@ class GigaChatConfig(BaseConfig): verbose_logger.error("Failed to upload image: %s", e) return None - def _transform_list_content(self, content: Sequence) -> tuple[str, Sequence[str]]: + def _transform_list_content(self, content: Sequence) -> tuple[str, Sequence[object]]: """ Extract text and image attachments from a multimodal message content list. @@ -304,7 +304,7 @@ class GigaChatConfig(BaseConfig): Tuple of (combined text, list of attachment file ids) """ texts: Final[list[str]] = [] # mutable-ok: accumulator - attachments: Final[list[str]] = [] # mutable-ok: accumulator + attachments: Final[list[object]] = [] # mutable-ok: accumulator for part in content: if isinstance(part, dict): if part.get("type") == "text": diff --git a/litellm/llms/gigachat/file_handler.py b/litellm/llms/gigachat/file_handler.py index aff0465d1ec..d53ae3af334 100644 --- a/litellm/llms/gigachat/file_handler.py +++ b/litellm/llms/gigachat/file_handler.py @@ -12,6 +12,8 @@ import uuid from collections.abc import Mapping from typing import Final +from pydantic import TypeAdapter + from litellm._logging import verbose_logger from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.custom_httpx.http_handler import ( @@ -19,12 +21,14 @@ from litellm.llms.custom_httpx.http_handler import ( get_httpx_client, ) from litellm.llms.gigachat.utils import get_api_base +from litellm.llms.openai.common_utils import decoded_json_object from litellm.types.utils import LlmProviders from .authenticator import get_access_token, get_access_token_async # Simple in-memory cache for file IDs -_file_cache: Final[dict[str, str]] = {} +_file_cache: Final[dict[str, object]] = {} +_DECODED_JSON: Final = TypeAdapter(object) def _get_url_hash(url: str) -> str: @@ -91,7 +95,7 @@ def upload_file_sync( credentials: str | None = None, api_base: str | None = None, litellm_params: Mapping[str, object] | None = None, -) -> str | None: +) -> object: """ Upload file to GigaChat and return file_id (sync). @@ -137,9 +141,9 @@ def upload_file_sync( timeout=60, ) response.raise_for_status() - result: Final = response.json() + result: Final = _DECODED_JSON.validate_python(response.json()) - file_id: Final = result.get("id") + file_id: Final = decoded_json_object(result).get("id") if file_id: _file_cache[url_hash] = file_id verbose_logger.debug("File uploaded successfully, file_id: %s", file_id) @@ -156,7 +160,7 @@ async def upload_file_async( credentials: str | None = None, api_base: str | None = None, litellm_params: Mapping[str, object] | None = None, -) -> str | None: +) -> object: """ Upload file to GigaChat and return file_id (async). @@ -205,9 +209,9 @@ async def upload_file_async( timeout=60, ) response.raise_for_status() - result: Final = response.json() + result: Final = _DECODED_JSON.validate_python(response.json()) - file_id: Final = result.get("id") + file_id: Final = decoded_json_object(result).get("id") if file_id: _file_cache[url_hash] = file_id verbose_logger.debug("File uploaded successfully, file_id: %s", file_id) diff --git a/litellm/llms/gigachat/passthrough/transformation.py b/litellm/llms/gigachat/passthrough/transformation.py index 872eee6dbf9..27f75eece50 100644 --- a/litellm/llms/gigachat/passthrough/transformation.py +++ b/litellm/llms/gigachat/passthrough/transformation.py @@ -5,11 +5,13 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final import httpx +from pydantic import TypeAdapter from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig from litellm.llms.gigachat.authenticator import get_access_token from litellm.llms.gigachat.chat.streaming import GigaChatModelResponseIterator from litellm.llms.gigachat.utils import GIGACHAT_BASE_URL +from litellm.llms.openai.common_utils import decoded_json_object from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import EmbeddingResponse @@ -20,6 +22,8 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.utils import CostResponseTypes +_DECODED_JSON: Final = TypeAdapter(object) + class GigaChatPassthroughConfig(BasePassthroughConfig): def is_streaming_request(self, endpoint: str, request_data: Mapping[str, object]) -> bool: @@ -161,7 +165,7 @@ class GigaChatPassthroughConfig(BasePassthroughConfig): continue chunk = chunk.removeprefix("data: ") try: - message = json.loads(chunk) + message = _DECODED_JSON.validate_python(json.loads(chunk)) except json.JSONDecodeError: continue @@ -169,7 +173,7 @@ class GigaChatPassthroughConfig(BasePassthroughConfig): streaming_response=None, sync_stream=False, ) - translated_chunk = gigachat_iterator.chunk_parser(chunk=message) + translated_chunk = gigachat_iterator.chunk_parser(chunk=decoded_json_object(message)) if isinstance(translated_chunk, dict) and generic_chunk_has_all_required_fields( # pyright: ignore[reportUnnecessaryIsInstance] # runtime guard for patched chunk_parser dict(translated_chunk) diff --git a/litellm/llms/github_copilot/chat/transformation.py b/litellm/llms/github_copilot/chat/transformation.py index 9553327db97..83bc8df23a6 100644 --- a/litellm/llms/github_copilot/chat/transformation.py +++ b/litellm/llms/github_copilot/chat/transformation.py @@ -4,9 +4,11 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # spread dict loses the role literal without a cast import httpx +from pydantic import TypeAdapter from litellm.exceptions import AuthenticationError from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.openai.common_utils import decoded_json_object from litellm.llms.openai.openai import OpenAIConfig from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk from litellm.types.utils import ModelResponse @@ -23,6 +25,8 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.tokenizer import Encoding +_DECODED_JSON: Final = TypeAdapter(object) + class GithubCopilotConfig(OpenAIConfig): def __init__( @@ -309,7 +313,7 @@ class GithubCopilotConfig(OpenAIConfig): json_mode: bool | None = None, ) -> "ModelResponse": try: - response_json = raw_response.json() + decoded_response: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception: return super().transform_response( model=model, @@ -325,12 +329,13 @@ class GithubCopilotConfig(OpenAIConfig): json_mode=json_mode, ) + response_json: Final = decoded_json_object(decoded_response) if not response_json.get("choices"): - response_json = self._synthesize_choices_for_anthropic_native(response_json) + synthesized: Final = self._synthesize_choices_for_anthropic_native(dict(response_json)) raw_response = httpx.Response( status_code=raw_response.status_code, headers=raw_response.headers, - content=json.dumps(response_json).encode(), + content=json.dumps(synthesized).encode(), ) return super().transform_response( diff --git a/litellm/llms/github_copilot/messages/transformation.py b/litellm/llms/github_copilot/messages/transformation.py index 56c3693a02c..613b7e42100 100644 --- a/litellm/llms/github_copilot/messages/transformation.py +++ b/litellm/llms/github_copilot/messages/transformation.py @@ -1,4 +1,5 @@ -from typing import Any, Final, cast # noqa: TID251 # narrows untyped request dicts at the session boundary +from collections.abc import Sequence +from typing import Final, cast # noqa: TID251 # narrows untyped request dicts at the session boundary from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.url_utils import ensure_api_route @@ -54,7 +55,7 @@ class GithubCopilotAnthropicMessagesConfig(AnthropicMessagesConfig): self, headers: dict, model: str, - messages: list[Any], + messages: Sequence[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py index bf0b8c79068..b055d388dd5 100644 --- a/litellm/llms/github_copilot/responses/transformation.py +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -9,6 +9,7 @@ https://github.com/caozhiyuan/copilot-api """ import os +from collections.abc import Mapping from typing import ( TYPE_CHECKING, Any, @@ -16,6 +17,8 @@ from typing import ( cast, # noqa: TID251 # narrows untyped request dicts at the session boundary ) +from pydantic import TypeAdapter + import litellm from litellm._logging import verbose_logger from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH @@ -47,6 +50,9 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_OBJECT_LIST: Final = TypeAdapter(list[object]) +_OBJECT_MAPPING: Final = TypeAdapter(Mapping[object, object]) + def github_copilot_supports_responses_api(model: str) -> bool: """ @@ -387,7 +393,9 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): """ return self._contains_vision_content(input_param) - def _contains_vision_content(self, value: Any, depth: int = 0, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> bool: + def _contains_vision_content( + self, value: object, depth: int = 0, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH + ) -> bool: """ Recursively check if a value contains vision content. @@ -404,22 +412,28 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): # Check arrays if isinstance(value, list): - return any(self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) for item in value) + items: Final = _OBJECT_LIST.validate_python(value) + return any(self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) for item in items) # Only check dict/object types if not isinstance(value, dict): return False + fields: Final = _OBJECT_MAPPING.validate_python(value) + # Check if this item is an input_image - item_type: Final = value.get("type") + item_type: Final = fields.get("type") if isinstance(item_type, str) and item_type.lower() == "input_image": return True # Check content field recursively - if "content" in value and isinstance(value["content"], list): - return any( - self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) for item in value["content"] - ) + if "content" in fields: + content: Final = fields["content"] + if isinstance(content, list): + content_items: Final = _OBJECT_LIST.validate_python(content) + return any( + self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) for item in content_items + ) return False diff --git a/litellm/llms/gradient_ai/chat/transformation.py b/litellm/llms/gradient_ai/chat/transformation.py index 0ba5ca45c2a..2a2368ad8de 100644 --- a/litellm/llms/gradient_ai/chat/transformation.py +++ b/litellm/llms/gradient_ai/chat/transformation.py @@ -1,3 +1,5 @@ +from collections.abc import Mapping +from types import MappingProxyType from typing import Final, Literal from litellm.litellm_core_utils.url_utils import join_api_base @@ -43,7 +45,7 @@ class GradientAIConfig(OpenAILikeChatConfig): include_guardrails_info: bool | None = None, provide_citations: bool | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 8c269cb5dfd..349a4b4ab87 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -40,6 +40,7 @@ class GroqExecutedToolIdentity(LiteLLMBaseModel): _EXECUTED_TOOLS_ADAPTER: Final = TypeAdapter(tuple[GroqExecutedToolIdentity, ...]) +_ATTRIBUTE_VALUE: Final = TypeAdapter(object) class GroqChatConfig(OpenAILikeChatConfig): @@ -324,7 +325,7 @@ class GroqChatConfig(OpenAILikeChatConfig): json_mode=json_mode, ) - original_service_tier: Final[object] = getattr(model_response, "service_tier") + original_service_tier: Final = _ATTRIBUTE_VALUE.validate_python(getattr(model_response, "service_tier")) mapped_service_tier: Final[Literal["auto", "default", "flex"]] = self._map_groq_service_tier( original_service_tier=original_service_tier ) diff --git a/litellm/llms/groq/cost_calculator.py b/litellm/llms/groq/cost_calculator.py index bcebb347aea..7f75a607410 100644 --- a/litellm/llms/groq/cost_calculator.py +++ b/litellm/llms/groq/cost_calculator.py @@ -19,7 +19,10 @@ if TYPE_CHECKING: def cost_per_web_search_request(usage: Usage, model_info: "ModelInfo") -> float: search_costs: Final = model_info.get("search_context_cost_per_query") cost_per_search: Final = search_costs.get("search_context_size_medium", 0.0) if search_costs else 0.0 - server_tool_use: Final = getattr(usage, "server_tool_use", None) + try: + server_tool_use: Final = usage.server_tool_use + except AttributeError: + return 0.0 if server_tool_use is None: return 0.0 searches: Final = server_tool_use.web_search_requests or 0 diff --git a/litellm/llms/groq/stt/transformation.py b/litellm/llms/groq/stt/transformation.py index 93d36dd7b0f..33e01ab7c0e 100644 --- a/litellm/llms/groq/stt/transformation.py +++ b/litellm/llms/groq/stt/transformation.py @@ -3,6 +3,7 @@ Translate from OpenAI's `/v1/audio/transcriptions` to Groq's `/v1/audio/transcri """ import types +from collections.abc import Mapping from typing import Final import litellm @@ -39,16 +40,17 @@ class GroqSTTConfig: tools: list | None = None, tool_choice: str | dict | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/hosted_vllm/rerank/transformation.py b/litellm/llms/hosted_vllm/rerank/transformation.py index 4a5729945b4..0b77a04c69a 100644 --- a/litellm/llms/hosted_vllm/rerank/transformation.py +++ b/litellm/llms/hosted_vllm/rerank/transformation.py @@ -7,7 +7,7 @@ from types import MappingProxyType from typing import Final import httpx -from pydantic import ValidationError +from pydantic import TypeAdapter, ValidationError from litellm._uuid import uuid from litellm.exceptions import UnsupportedParamsError @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig +from litellm.llms.openai.common_utils import decoded_json_object from litellm.secret_managers.main import get_secret_str from litellm.types.rerank import ( HostedVLLMRerankTruncationParams, @@ -28,6 +29,8 @@ from litellm.types.rerank import ( RerankTokens, ) +_DECODED_JSON: Final = TypeAdapter(object) + class HostedVLLMRerankError(BaseLLMException): def __init__( @@ -193,11 +196,11 @@ class HostedVLLMRerankConfig(BaseRerankConfig): Process response from Hosted VLLM rerank API """ try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception: raise ValueError(f"Error parsing response: {raw_response.text}, status_code={raw_response.status_code}") - return self._transform_response(raw_response_json) + return self._transform_response(dict(decoded_json_object(raw_response_json))) def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: return HostedVLLMRerankError(message=error_message, status_code=status_code, headers=headers) diff --git a/litellm/llms/hosted_vllm/videos/transformation.py b/litellm/llms/hosted_vllm/videos/transformation.py index 30ae6835263..2045d2c8988 100644 --- a/litellm/llms/hosted_vllm/videos/transformation.py +++ b/litellm/llms/hosted_vllm/videos/transformation.py @@ -8,6 +8,7 @@ from typing import Final from urllib.parse import urlparse from httpx._types import FileTypes, RequestFiles +from pydantic import TypeAdapter from litellm.images.utils import ImageEditRequestUtils from litellm.litellm_core_utils.url_utils import SSRFError, join_api_base, validate_url @@ -16,6 +17,7 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams from litellm.types.videos.main import VideoCreateOptionalRequestParams +_DECODED_JSON: Final = TypeAdapter(object) _EXCLUDED_FORM_KEYS: Final = frozenset( { "model", @@ -82,7 +84,7 @@ def _maybe_json(value: object) -> object: stripped: Final = value.strip() if not stripped or stripped[0] not in "{[": return value - return json.loads(stripped) + return _DECODED_JSON.validate_python(json.loads(stripped)) def _reject_unsafe_media_url(url: str) -> None: diff --git a/litellm/llms/huggingface/embedding/handler.py b/litellm/llms/huggingface/embedding/handler.py index 60917c68221..a1d1a3f6a19 100644 --- a/litellm/llms/huggingface/embedding/handler.py +++ b/litellm/llms/huggingface/embedding/handler.py @@ -3,6 +3,7 @@ import os from typing import Final, Literal, Protocol, get_args import httpx +from pydantic import TypeAdapter import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -11,6 +12,7 @@ from litellm.llms.custom_httpx.http_handler import ( HTTPHandler, get_async_httpx_client, ) +from litellm.llms.openai.common_utils import decoded_json_object from litellm.types.utils import EmbeddingResponse from ...base import BaseLLM @@ -21,6 +23,8 @@ config: Final = HuggingFaceEmbeddingConfig() HF_HUB_URL: Final = "https://huggingface.co" +_DECODED_JSON: Final = TypeAdapter(object) + hf_tasks_embeddings: Final = ( Literal[ # pipeline tags + hf tei endpoints - https://huggingface.github.io/text-embeddings-inference/#/ "sentence-similarity", "feature-extraction", "rerank", "embed", "similarity" @@ -34,7 +38,7 @@ class _SupportsTokenEncode(Protocol): def encode(self, text: str) -> list[int]: ... -def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: str) -> str | None: +def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: str) -> object: if task_type is not None: if task_type in get_args(hf_tasks_embeddings): return task_type @@ -44,14 +48,14 @@ def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: model_info: Final = http_client.get(url=f"{api_base}/api/models/{model}") - model_info_dict: Final = model_info.json() + model_info_dict: Final = decoded_json_object(_DECODED_JSON.validate_python(model_info.json())) - pipeline_tag: Final[str | None] = model_info_dict.get("pipeline_tag", None) + pipeline_tag: Final = model_info_dict.get("pipeline_tag", None) return pipeline_tag -async def async_get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: str) -> str | None: +async def async_get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: str) -> object: if task_type is not None: if task_type in get_args(hf_tasks_embeddings): return task_type @@ -63,9 +67,9 @@ async def async_get_hf_task_embedding_for_model(model: str, task_type: str | Non model_info: Final = await http_client.get(url=f"{api_base}/api/models/{model}") - model_info_dict: Final = model_info.json() + model_info_dict: Final = decoded_json_object(_DECODED_JSON.validate_python(model_info.json())) - pipeline_tag: Final[str | None] = model_info_dict.get("pipeline_tag", None) + pipeline_tag: Final = model_info_dict.get("pipeline_tag", None) return pipeline_tag @@ -77,7 +81,7 @@ class HuggingFaceEmbedding(BaseLLM): def __init__(self) -> None: super().__init__() - def _transform_input_on_pipeline_tag(self, input: list, pipeline_tag: str | None) -> dict: + def _transform_input_on_pipeline_tag(self, input: list, pipeline_tag: object) -> dict: if pipeline_tag is None: return {"inputs": input} if pipeline_tag == "sentence-similarity" or pipeline_tag == "similarity": diff --git a/litellm/llms/huggingface/embedding/transformation.py b/litellm/llms/huggingface/embedding/transformation.py index 0883a9a8077..1b7d6483739 100644 --- a/litellm/llms/huggingface/embedding/transformation.py +++ b/litellm/llms/huggingface/embedding/transformation.py @@ -447,14 +447,13 @@ class HuggingFaceEmbeddingConfig(BaseConfig): except Exception: # this should remain non blocking we should not block a response returning if calculating usage fails pass - output_text: Final = model_response["choices"][0]["message"].get("content", "") + first_message: Final = model_response.choices[0].message + output_text: Final = first_message.content if output_text is not None and len(output_text) > 0: completion_tokens = 0 try: if encoding is not None: - completion_tokens = len( - encoding.encode(model_response["choices"][0]["message"].get("content", "")) - ) ##[TODO] use the llama2 tokenizer here + completion_tokens = len(encoding.encode(output_text)) ##[TODO] use the llama2 tokenizer here except Exception: # this should remain non blocking we should not block a response returning if calculating usage fails pass diff --git a/litellm/llms/huggingface/rerank/transformation.py b/litellm/llms/huggingface/rerank/transformation.py index 1217a2540d4..1f130f84283 100644 --- a/litellm/llms/huggingface/rerank/transformation.py +++ b/litellm/llms/huggingface/rerank/transformation.py @@ -86,7 +86,7 @@ class HuggingFaceRerankConfig(BaseRerankConfig): model: str, drop_params: bool, query: str, - documents: list[str | dict[str, Any]], + documents: list[str | dict[str, object]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, diff --git a/litellm/llms/langflow/chat/transformation.py b/litellm/llms/langflow/chat/transformation.py index 0938cc0d07b..3807be562cf 100644 --- a/litellm/llms/langflow/chat/transformation.py +++ b/litellm/llms/langflow/chat/transformation.py @@ -1,9 +1,11 @@ """LangFlow run API: POST {api_base}/api/v1/run/{flow_id}""" +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final from urllib.parse import quote import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -11,6 +13,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.openai.common_utils import decoded_json_object from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage @@ -27,6 +30,8 @@ else: AsyncHTTPHandler = Any CustomStreamWrapper = Any +_DECODED_JSON: Final = TypeAdapter(object) + class LangFlowError(BaseLLMException): """Exception class for LangFlow API errors.""" @@ -179,7 +184,7 @@ class LangFlowConfig(BaseConfig): verbose_logger.debug("LangFlow request payload: %s", payload) return payload - def _extract_content_from_response(self, response_json: dict) -> str | None: + def _extract_content_from_response(self, response_json: Mapping[str, object]) -> str | None: """ Extract the assistant text from a LangFlow run response. @@ -236,7 +241,7 @@ class LangFlowConfig(BaseConfig): json_mode: bool | None = None, ) -> ModelResponse: try: - response_json: Final = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise LangFlowError( message=f"LangFlow returned a non-JSON response: {e}", @@ -245,7 +250,7 @@ class LangFlowConfig(BaseConfig): verbose_logger.debug("LangFlow response: %s", response_json) - content: Final = self._extract_content_from_response(response_json) + content: Final = self._extract_content_from_response(decoded_json_object(response_json)) if content is None: raise LangFlowError( message=( diff --git a/litellm/llms/langgraph/chat/sse_iterator.py b/litellm/llms/langgraph/chat/sse_iterator.py index 499cbab3b7b..575820bceee 100644 --- a/litellm/llms/langgraph/chat/sse_iterator.py +++ b/litellm/llms/langgraph/chat/sse_iterator.py @@ -9,10 +9,13 @@ import uuid from typing import Final import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices +_DECODED_JSON: Final = TypeAdapter(object) + class LangGraphSSEStreamIterator: """ @@ -60,7 +63,7 @@ class LangGraphSSEStreamIterator: return None try: - data: Final = json.loads(json_str) + data: Final = _DECODED_JSON.validate_python(json.loads(json_str)) return self._process_data(data) except json.JSONDecodeError: verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100]) diff --git a/litellm/llms/langgraph/chat/transformation.py b/litellm/llms/langgraph/chat/transformation.py index ce624eecc0e..603d5a07292 100644 --- a/litellm/llms/langgraph/chat/transformation.py +++ b/litellm/llms/langgraph/chat/transformation.py @@ -13,6 +13,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final, Optional, Union import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -21,6 +22,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.langgraph.chat.sse_iterator import LangGraphSSEStreamIterator +from litellm.llms.openai.common_utils import decoded_json_object from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage @@ -37,6 +39,8 @@ else: AsyncHTTPHandler = Any CustomStreamWrapper = Any +_DECODED_JSON: Final = TypeAdapter(object) + class LangGraphError(BaseLLMException): """Exception class for LangGraph API errors.""" @@ -233,7 +237,7 @@ class LangGraphConfig(BaseConfig): verbose_logger.debug("LangGraph request payload: %s", payload) return payload - def _extract_content_from_response(self, response_json: dict) -> str: + def _extract_content_from_response(self, response_json: Mapping[str, object]) -> str: """ Extract content from LangGraph non-streaming response. @@ -434,10 +438,10 @@ class LangGraphConfig(BaseConfig): Transform the LangGraph response to LiteLLM ModelResponse format. """ try: - response_json: Final = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) verbose_logger.debug("LangGraph response: %s", response_json) - content: Final = self._extract_content_from_response(response_json) + content: Final = self._extract_content_from_response(decoded_json_object(response_json)) # Create the message message: Final = Message(content=content, role="assistant") diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index 01fdc0b8bf8..42bd63b0160 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -3,7 +3,7 @@ Translate from OpenAI's `/v1/chat/completions` to Lemonade's `/v1/chat/completio """ from collections.abc import Iterable, Mapping -from typing import TYPE_CHECKING, Any, Final, Protocol +from typing import TYPE_CHECKING, Final, Protocol from urllib.parse import quote import httpx @@ -27,6 +27,7 @@ if TYPE_CHECKING: _JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) _STR: Final = TypeAdapter(str) +_OPAQUE_VALUE: Final = TypeAdapter(object) class _JsonResponse(Protocol): @@ -123,7 +124,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): return ["lemonade/" + _STR.validate_python(model["id"]) for model in model_list] @staticmethod - def _get_positive_int(value: Any) -> int | None: + def _get_positive_int(value: object) -> int | None: if isinstance(value, bool): return None if isinstance(value, int) and value > 0: @@ -163,7 +164,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): provider_specific_entry.get("context_window"), provider_specific_entry.get("max_context_window"), ): - parsed = self._get_positive_int(value) + parsed = self._get_positive_int(_OPAQUE_VALUE.validate_python(value)) if parsed is not None: return parsed return None diff --git a/litellm/llms/litellm_proxy/skills/prompt_injection.py b/litellm/llms/litellm_proxy/skills/prompt_injection.py index 3377338a03e..e8e1c68a112 100644 --- a/litellm/llms/litellm_proxy/skills/prompt_injection.py +++ b/litellm/llms/litellm_proxy/skills/prompt_injection.py @@ -9,9 +9,10 @@ import posixpath import zipfile from collections.abc import Mapping from io import BytesIO -from typing import Any, Final +from typing import Final from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.models.skills import LiteLLM_SkillsTable @@ -251,27 +252,30 @@ class SkillPromptInjectionHandler: if len(description) > max_desc_length: description = description[: max_desc_length - 3] + "..." - tool: Final[dict[str, Any]] = { + params: Final = ( + as_str_mapping(skill.metadata.get("parameters")) + if skill.metadata and isinstance(skill.metadata, dict) + else None + ) + parameters: Final[Mapping[str, object]] = ( + params + if params and isinstance(params, dict) + else { + "type": "object", + "properties": {}, + "required": [], + } + ) + + return { "type": "function", "function": { "name": func_name, "description": description, - "parameters": { - "type": "object", - "properties": {}, - "required": [], - }, + "parameters": parameters, }, } - # If skill has metadata with parameter definitions, use them - if skill.metadata and isinstance(skill.metadata, dict): - params: Final = skill.metadata.get("parameters") - if params and isinstance(params, dict): - tool["function"]["parameters"] = params - - return tool - def convert_skill_to_anthropic_tool(self, skill: LiteLLM_SkillsTable) -> Mapping[str, object]: """ Convert a LiteLLM skill to an Anthropic-style tool (messages API format). diff --git a/litellm/llms/litellm_proxy/skills/sandbox_executor.py b/litellm/llms/litellm_proxy/skills/sandbox_executor.py index 3e38dd81905..8e5c9effa17 100644 --- a/litellm/llms/litellm_proxy/skills/sandbox_executor.py +++ b/litellm/llms/litellm_proxy/skills/sandbox_executor.py @@ -7,7 +7,8 @@ Supports Docker, Podman, and Kubernetes backends. import base64 import os -from typing import Any, Final, Protocol, TypedDict +from collections.abc import Sequence +from typing import Final, Protocol, TypedDict from typing_extensions import ReadOnly @@ -41,6 +42,13 @@ class _GeneratedFile(TypedDict): mime_type: ReadOnly[str] +class _SandboxExecution(TypedDict): + success: ReadOnly[bool] + output: ReadOnly[str] + error: ReadOnly[str] + files: ReadOnly[Sequence[_GeneratedFile]] + + class SkillsSandboxExecutor: """ Executes skill code in llm-sandbox Docker container. @@ -76,7 +84,7 @@ class SkillsSandboxExecutor: code: str, skill_files: dict[str, bytes], requirements: str | None = None, - ) -> dict[str, Any]: + ) -> _SandboxExecution: """ Execute code with skill files in sandbox. diff --git a/litellm/llms/lm_studio/embed/transformation.py b/litellm/llms/lm_studio/embed/transformation.py index 66b2dbb534d..d6264ceb6e0 100644 --- a/litellm/llms/lm_studio/embed/transformation.py +++ b/litellm/llms/lm_studio/embed/transformation.py @@ -7,6 +7,7 @@ Docs - https://lmstudio.ai/docs/basics/server """ import types +from collections.abc import Mapping from typing import Final @@ -18,16 +19,17 @@ class LmStudioEmbeddingConfig: def __init__( self, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/maritalk.py b/litellm/llms/maritalk.py index 094dee4cee1..36bde5847fb 100644 --- a/litellm/llms/maritalk.py +++ b/litellm/llms/maritalk.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final from httpx._models import Headers @@ -32,7 +33,7 @@ class MaritalkConfig(OpenAIGPTConfig): tools: list[dict] | None = None, tool_choice: str | dict | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = dict(locals()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/milvus/vector_stores/transformation.py b/litellm/llms/milvus/vector_stores/transformation.py index 67de3a06af8..14470922e57 100644 --- a/litellm/llms/milvus/vector_stores/transformation.py +++ b/litellm/llms/milvus/vector_stores/transformation.py @@ -4,12 +4,14 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import TypeAdapter from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.vector_store.transformation import ( BaseQueryEmbeddingVectorStoreConfig, VectorStoreEmbeddingExecutor, ) +from litellm.llms.openai.common_utils import decoded_json_object from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams from litellm.types.vector_stores import ( @@ -30,6 +32,8 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_DECODED_JSON: Final = TypeAdapter(object) + MILVUS_OPTIONAL_PARAMS: Final = { "annsField", "limit", @@ -218,15 +222,17 @@ class MilvusVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig): results: Final = response_json.get("data", []) # Try to get text_field from optional_params first, then litellm_params - optional_params: Final = litellm_logging_obj.model_call_details.get("optional_params", {}) + optional_params: Final = decoded_json_object( + _DECODED_JSON.validate_python(litellm_logging_obj.model_call_details.get("optional_params", {})) + ) text_field = optional_params.get("milvus_text_field", "") # Fallback to litellm_params if not in optional_params if not text_field: - text_field = litellm_logging_obj.model_call_details.get("litellm_params", {}).get( - "milvus_text_field", "" - ) + text_field = decoded_json_object( + _DECODED_JSON.validate_python(litellm_logging_obj.model_call_details.get("litellm_params", {})) + ).get("milvus_text_field", "") # Transform results to standard format search_results: Final[list[VectorStoreSearchResult]] = [] diff --git a/litellm/llms/minimax/messages/transformation.py b/litellm/llms/minimax/messages/transformation.py index f9f3502237c..f7a263a467c 100644 --- a/litellm/llms/minimax/messages/transformation.py +++ b/litellm/llms/minimax/messages/transformation.py @@ -2,8 +2,6 @@ MiniMax Anthropic transformation config - extends AnthropicConfig for MiniMax's Anthropic-compatible API """ -from typing import Any # noqa: TID251 # override below must mirror the legacy base signature - import litellm from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.anthropic.pass_through.messages.transformation import ( @@ -54,7 +52,7 @@ class MinimaxMessagesConfig(AnthropicMessagesConfig): self, headers: dict, # mutable-ok: mirrors the legacy base override signature model: str, - messages: list[Any], # mutable-ok: mirrors the legacy base override signature + messages: list[object], # mutable-ok: mirrors the legacy base override signature optional_params: dict, # mutable-ok: mirrors the legacy base override signature litellm_params: dict, # mutable-ok: mirrors the legacy base override signature api_key: str | None = None, diff --git a/litellm/llms/modelscope/image_generation/transformation.py b/litellm/llms/modelscope/image_generation/transformation.py index 36c8705f256..c308de92d1d 100644 --- a/litellm/llms/modelscope/image_generation/transformation.py +++ b/litellm/llms/modelscope/image_generation/transformation.py @@ -33,6 +33,7 @@ else: LiteLLMLoggingObj = object _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) @@ -173,7 +174,7 @@ class ModelScopeImageGenerationConfig(BaseImageGenerationConfig): {"created": timestamp, "data": [{"url": "..."}]} """ try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error parsing ModelScope response: {e}", diff --git a/litellm/llms/nadir/chat/transformation.py b/litellm/llms/nadir/chat/transformation.py index d22b1a51bab..c8bde7d9fad 100644 --- a/litellm/llms/nadir/chat/transformation.py +++ b/litellm/llms/nadir/chat/transformation.py @@ -2,8 +2,9 @@ import math from typing import Final import httpx +from pydantic import TypeAdapter -from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_response_cost_in_hidden_params from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse @@ -19,13 +20,17 @@ _SUPPORTED_OPENAI_PARAMS: Final = ( "temperature", "top_p", ) +_DECODED_JSON: Final = TypeAdapter(object) def _reported_cost_usd(raw_response: httpx.Response) -> float | None: try: - cost: Final = raw_response.json()["nadir_metadata"]["cost"]["total_cost_usd"] + body: Final = as_str_mapping(_DECODED_JSON.validate_python(raw_response.json())) except (ValueError, KeyError, TypeError): return None + metadata: Final = as_str_mapping(body.get("nadir_metadata")) if body is not None else None + cost_block: Final = as_str_mapping(metadata.get("cost")) if metadata is not None else None + cost: Final = cost_block.get("total_cost_usd") if cost_block is not None else None if isinstance(cost, bool) or not isinstance(cost, (int, float)): return None if not math.isfinite(cost) or cost < 0: diff --git a/litellm/llms/nvidia_nim/embed.py b/litellm/llms/nvidia_nim/embed.py index 5a0105a3776..d2ef446299e 100644 --- a/litellm/llms/nvidia_nim/embed.py +++ b/litellm/llms/nvidia_nim/embed.py @@ -9,6 +9,7 @@ API calling is done using the OpenAI SDK with an api_base """ import types +from collections.abc import Mapping from typing import Final @@ -32,16 +33,17 @@ class NvidiaNimEmbeddingConfig: input_type: str | None = None, truncate: str | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/nvidia_riva/audio_transcription/handler.py b/litellm/llms/nvidia_riva/audio_transcription/handler.py index 748203b22d9..cafeece0a4e 100644 --- a/litellm/llms/nvidia_riva/audio_transcription/handler.py +++ b/litellm/llms/nvidia_riva/audio_transcription/handler.py @@ -28,7 +28,7 @@ without the optional STT extras installed. import asyncio import inspect -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Callable, Iterable, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol @@ -48,6 +48,8 @@ from litellm.llms.nvidia_riva.audio_transcription.transformation import ( RIVA_TARGET_NUM_CHANNELS, RIVA_TARGET_SAMPLE_RATE_HZ, NvidiaRivaAudioTranscriptionConfig, + RivaFinalResult, + RivaWordTiming, ) from litellm.llms.nvidia_riva.common_utils import ( NvidiaRivaException, @@ -535,14 +537,14 @@ class NvidiaRivaAudioTranscription: yield chunk @staticmethod - def _collect_final_results(stream: Iterable[object], require_final: bool = True) -> list[dict[str, object]]: + def _collect_final_results(stream: Iterable[object], require_final: bool = True) -> Sequence[RivaFinalResult]: """ Walk the gRPC responses, ignore empty / non-final chunks, and return a list of normalized final-result dicts. Matching the user's note: the ``id`` blocks with no ``results`` are streaming heartbeats and must be skipped. """ - final_results: Final[list[dict[str, object]]] = [] + final_results: Final[list[RivaFinalResult]] = [] for response in stream: results = getattr(response, "results", None) or [] for result in results: @@ -554,16 +556,19 @@ class NvidiaRivaAudioTranscription: top = alternatives[0] transcript = getattr(top, "transcript", "") or "" words_proto = getattr(top, "words", None) or [] - words = [] - for word in words_proto: - words.append( - { - "word": getattr(word, "word", ""), - "start_time_ms": int(getattr(word, "start_time", 0) or 0), - "end_time_ms": int(getattr(word, "end_time", 0) or 0), - } - ) - final_results.append({"transcript": transcript, "words": words}) + final_results.append( + { + "transcript": transcript, + "words": [ + RivaWordTiming( + word=getattr(word, "word", ""), + start_time_ms=int(getattr(word, "start_time", 0) or 0), + end_time_ms=int(getattr(word, "end_time", 0) or 0), + ) + for word in words_proto + ], + } + ) return final_results diff --git a/litellm/llms/nvidia_riva/audio_transcription/transformation.py b/litellm/llms/nvidia_riva/audio_transcription/transformation.py index 262dfb7bd78..5862a5d10b1 100644 --- a/litellm/llms/nvidia_riva/audio_transcription/transformation.py +++ b/litellm/llms/nvidia_riva/audio_transcription/transformation.py @@ -11,9 +11,11 @@ dict at call time. Reference: https://docs.nvidia.com/deeplearning/riva/user-guide/docs/asr/asr-overview.html """ -from typing import Any, Final +from collections.abc import Sequence +from typing import Final, TypedDict from httpx import Headers, Response +from typing_extensions import ReadOnly from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.openai import ( @@ -34,6 +36,17 @@ RIVA_TARGET_NUM_CHANNELS: Final = 1 RIVA_TARGET_ENCODING: Final = "LINEAR_PCM" +class RivaWordTiming(TypedDict): + word: ReadOnly[str] + start_time_ms: ReadOnly[int] + end_time_ms: ReadOnly[int] + + +class RivaFinalResult(TypedDict): + transcript: ReadOnly[str] + words: ReadOnly[Sequence[RivaWordTiming]] + + class NvidiaRivaAudioTranscriptionConfig(BaseAudioTranscriptionConfig): """ Config for NVIDIA Riva ASR (gRPC). @@ -219,7 +232,7 @@ class NvidiaRivaAudioTranscriptionConfig(BaseAudioTranscriptionConfig): @staticmethod def build_transcription_response( - final_results: list[dict[str, Any]], + final_results: Sequence[RivaFinalResult], response_format: str, duration_seconds: float | None, timestamp_granularities: list[str] | None, diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 8e1502b2610..e8d37f857e6 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -189,6 +189,7 @@ def _normalize_tool_choice(selected_params: dict) -> None: _TOOL_CHOICE_MODES: Final = frozenset({"auto", "none", "required", "any"}) _TOOLS_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) _JSON_OBJECT_ADAPTER: Final = TypeAdapter(dict[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) def _as_json_object(value: object) -> Mapping[str, object] | None: @@ -701,7 +702,7 @@ class OCIChatConfig(BaseConfig): api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: - response_json: Final = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) if not isinstance(response_json, dict): raise OCIError( diff --git a/litellm/llms/oci/embed/transformation.py b/litellm/llms/oci/embed/transformation.py index ffcb156bb16..6dee64bf19a 100644 --- a/litellm/llms/oci/embed/transformation.py +++ b/litellm/llms/oci/embed/transformation.py @@ -56,6 +56,7 @@ else: LiteLLMLoggingObj = Any _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) # OCI sends up to 96 texts per embedText request (Cohere limit). OCI_EMBED_BATCH_LIMIT: Final = 96 @@ -272,7 +273,7 @@ class OCIEmbedConfig(BaseEmbeddingConfig): ) try: - json_response: Final = raw_response.json() + json_response: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise OCIError( status_code=raw_response.status_code, diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index 7894904c870..f313c62f026 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -13,6 +13,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.utils import ModelResponse, ModelResponseStream _PROVIDER_FIELDS: Final = TypeAdapter(dict[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) class OllamaResponseIdNormalizer: @@ -152,7 +153,7 @@ class OllamaModelInfo(BaseLLMModelInfo): try: resp: Final = httpx.get(join_api_base(base, "api/tags"), headers=headers) resp.raise_for_status() - data: Final = resp.json() + data: Final = _DECODED_JSON.validate_python(resp.json()) # Expecting a dict with a 'models' list models_list = [] if isinstance(data, dict) and "models" in data and isinstance(data["models"], list): diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 30de22b17ff..239b4e32645 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -547,7 +547,7 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator): held: Final = self.held_content self.held_content = "" try: - parsed: Final[object] = json.loads(_strip_json_code_fence(held)) + parsed: Final = _JSON_VALUE.validate_python(json.loads(_strip_json_code_fence(held))) except json.JSONDecodeError: return GenericStreamingChunk(text=held, is_finished=True, finish_reason="stop", usage=usage) if isinstance(parsed, dict) and "name" in parsed and "arguments" in parsed: diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index e472661efb9..12ead582975 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -23,13 +23,14 @@ from openai.types.chat.chat_completion import Choice from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice from openai.types.chat.chat_completion_chunk import ChoiceDelta from openai.types.completion_usage import CompletionUsage -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter if TYPE_CHECKING: from aiohttp import ClientSession import litellm from litellm.litellm_core_utils.completion_timeout import CompletionTimeout +from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object from litellm.litellm_core_utils.token_counter import token_counter from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import ( @@ -134,6 +135,9 @@ class OpenAIError(BaseLLMException): ) +_DECODED_JSON: Final = TypeAdapter(object) + + ####### Error Handling Utils for OpenAI API ####################### ################################################################### def drop_params_from_unprocessable_entity_error( @@ -152,19 +156,20 @@ def drop_params_from_unprocessable_entity_error( """ invalid_params: Final[list[str]] = [] if isinstance(e, httpx.HTTPStatusError): - error_json: Final = e.response.json() + error_json: Final = decoded_json_object(_DECODED_JSON.validate_python(e.response.json())) error_message: Final = error_json.get("error", {}) error_body = error_message else: error_body = e.body - if error_body is not None and isinstance(error_body, dict) and error_body.get("message"): - message = error_body.get("message", {}) + error_mapping: Final = as_str_mapping(error_body) + if isinstance(error_mapping, dict) and error_mapping.get("message"): + message = error_mapping.get("message", {}) if isinstance(message, str): try: - message = json.loads(message) + message = _DECODED_JSON.validate_python(json.loads(message)) except json.JSONDecodeError: message = {"detail": message} - detail: Final = message.get("detail") + detail: Final = decoded_json_object(message).get("detail") if isinstance(detail, list) and len(detail) > 0 and isinstance(detail[0], dict): for error_dict in detail: diff --git a/litellm/llms/openai/evals/transformation.py b/litellm/llms/openai/evals/transformation.py index 581e5e8ad87..73d211e1613 100644 --- a/litellm/llms/openai/evals/transformation.py +++ b/litellm/llms/openai/evals/transformation.py @@ -2,10 +2,10 @@ OpenAI Evals API configuration and transformations """ -from collections.abc import Mapping from typing import Final import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base @@ -32,9 +32,11 @@ from litellm.types.llms.openai_evals import ( from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +_DECODED_JSON: Final = TypeAdapter(object) -def _parsed_response_json(raw_response: httpx.Response) -> Mapping[str, object]: - return raw_response.json() + +def _parsed_response_json(raw_response: httpx.Response) -> object: + return _DECODED_JSON.validate_python(raw_response.json()) class OpenAIEvalsConfig(BaseEvalsAPIConfig): diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 437f19a4dc6..deee502c9f2 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -20,7 +20,7 @@ from litellm.llms.anthropic.pass_through.responses_adapters.transformation impor LiteLLMAnthropicToResponsesAPIAdapter, ) from litellm.llms.base_llm.base_utils import BaseTokenCounter -from litellm.llms.openai.common_utils import OpenAIError, get_openai_credentials +from litellm.llms.openai.common_utils import OpenAIError, decoded_json_object, get_openai_credentials from litellm.llms.openai.responses.count_tokens.handler import ( OpenAICountTokensHandler, ) @@ -244,7 +244,7 @@ class OpenAITokenCounter(BaseTokenCounter): if result is not None: return TokenCountResponse.model_validate( { - "total_tokens": result.get("input_tokens", 0), + "total_tokens": decoded_json_object(result).get("input_tokens", 0), "request_model": request_model, "model_used": model_to_use, "tokenizer_type": "openai_api", diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index c1ee42444c5..0f885146e36 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -6,12 +6,13 @@ This module handles the transformation of requests to OpenAI's /v1/responses/inp import copy from collections.abc import Mapping, Sequence -from typing import Any, Final, Literal, TypeAlias +from typing import Final, Literal, Protocol, TypeAlias from openai.types.responses.input_token_count_params import InputTokenCountParams from pydantic import JsonValue, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict +from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes from litellm.litellm_core_utils.tool_fields import DropPolicy from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_text_error @@ -61,6 +62,26 @@ class _AnthropicCountMessage(TypedDict): _ANTHROPIC_COUNT_MESSAGES: Final = TypeAdapter(tuple[_AnthropicCountMessage, ...]) + + +class _FieldReader(Protocol): + def get(self, key: str, default: object = None, /) -> object: ... + + +class _FunctionFields(_FieldReader, Protocol): + def __contains__(self, key: object, /) -> bool: ... + + def __getitem__(self, key: str, /) -> object: ... + + +def _field_reader(value: object) -> _FieldReader: + return require_attributes(value, _FieldReader, "get") + + +def _function_fields(value: object) -> _FunctionFields: + return require_attributes(value, _FunctionFields, "get", "__contains__", "__getitem__") + + _ANTHROPIC_OUTPUT_SCHEMA: Final = TypeAdapter(AnthropicOutputSchema) @@ -196,7 +217,7 @@ class OpenAICountTokensConfig: @staticmethod def _transform_tools_for_responses_api( - tools: list[dict[str, Any]], + tools: list[dict[str, object]], ) -> list[dict[str, object]]: """ Transform OpenAI chat and Anthropic tools to Responses API tools format. @@ -208,7 +229,7 @@ class OpenAICountTokensConfig: transformed: Final[list[dict[str, object]]] = [] for tool in tools: if tool.get("type") == "function" and "function" in tool: - func = tool["function"] + func = _function_fields(tool["function"]) item: dict[str, object] = { "type": "function", "name": func.get("name", ""), @@ -235,7 +256,7 @@ class OpenAICountTokensConfig: @staticmethod def messages_to_responses_input( - messages: list[dict[str, Any]], + messages: Sequence[Mapping[str, object]], ) -> tuple[list[dict[str, object]], str | None]: """ Convert standard chat messages format to OpenAI Responses API input format. @@ -276,14 +297,17 @@ class OpenAICountTokensConfig: if content: input_items.append({"role": "assistant", "content": content}) if tool_calls: - for tc in tool_calls: - func = tc.get("function", {}) + for tc in iter_items(tool_calls): + tool_call = _field_reader(tc) + function = tool_call.get("function", {}) + call_id = tool_call.get("id", "") + function_fields = _field_reader(function) input_items.append( { "type": "function_call", - "call_id": tc.get("id", ""), - "name": func.get("name", ""), - "arguments": func.get("arguments", ""), + "call_id": call_id, + "name": function_fields.get("name", ""), + "arguments": function_fields.get("arguments", ""), } ) elif not content: diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 3ea126c5a2a..d37a12a91d6 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -36,7 +36,7 @@ from collections.abc import Mapping, MutableSequence, Sequence from dataclasses import dataclass from itertools import accumulate, chain, repeat from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, NamedTuple, Union, cast +from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, Union, cast from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -47,6 +47,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i OpenAiResponsesToChatCompletionStreamIterator, tool_call_dict_from_output_item, ) +from litellm.litellm_core_utils.core_helpers import has_attributes from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, StreamingScanKey, @@ -237,6 +238,10 @@ class ResponsesStreamChunk(TypedDict, total=False): content_index: ReadOnly[int] +class _StreamEventFields(Protocol): + def get(self, key: str, /) -> object: ... + + _TERMINAL_ENVELOPE_EVENT_TYPES: Final = frozenset( { ResponsesAPIStreamEvents.RESPONSE_COMPLETED.value, @@ -1264,7 +1269,7 @@ class OpenAIResponsesHandler(BaseTranslation): def _sync_stream_events_with_rewrites( self, - stream_events: Sequence[Any], + stream_events: Sequence[object], rewrites_by_position: Mapping[tuple[int, int], str], ) -> None: """Sync pre-completion stream events with the rewritten completed @@ -1278,7 +1283,7 @@ class OpenAIResponsesHandler(BaseTranslation): {position: chain((rewritten,), repeat("")) for position, rewritten in rewrites_by_position.items()} ) for event in stream_events: - if not (isinstance(event, dict) or hasattr(event, "get")): + if not has_attributes(event, _StreamEventFields, "get"): continue event_type = event.get("type") output_index = event.get("output_index") diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index 12c8b1da166..0ab55d653e0 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -1,4 +1,4 @@ -from typing import Any, Final +from typing import Final import litellm from litellm.litellm_core_utils.url_utils import join_api_base @@ -41,7 +41,7 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): self, headers: dict[str, str], model: str, - messages: list[Any], + messages: list[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, @@ -151,7 +151,7 @@ class JSONProviderAnthropicMessagesConfig(OpenAILikeAnthropicMessagesConfig): self, headers: dict[str, str], model: str, - messages: list[Any], + messages: list[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/opencode/harness/transformation.py b/litellm/llms/opencode/harness/transformation.py index 46e8b750ca5..e801b66c448 100644 --- a/litellm/llms/opencode/harness/transformation.py +++ b/litellm/llms/opencode/harness/transformation.py @@ -26,10 +26,10 @@ import itertools import json import secrets import string -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError @@ -45,6 +45,7 @@ from litellm.harness.types import ( ToolCall, ToolResult, ) +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.llms.base_llm.harness.transformation import ( BaseCLIHarnessConfig, HarnessSessionCopy, @@ -269,10 +270,17 @@ def session_copy(exported: str, path: str, workdir: str) -> HarnessSessionCopy | ) -def _as_dict(value: object) -> Mapping[str, Any]: +def _as_dict(value: object) -> Mapping[str, object]: return value if isinstance(value, dict) else MappingProxyType({}) +def _spread_items(value: object) -> Iterable[object]: + try: + return iter_items(value) + except TypeError: + raise TypeError(f"Value after * must be an iterable, not {type(value).__name__}") from None + + def _tool_events(part: Mapping[str, object]) -> Sequence[Event]: native = str(part.get("tool") or "") call_id = str(part.get("callID") or part.get("id") or "") @@ -346,7 +354,7 @@ def build_opencode_config( token_path: str, permissions: PermissionMode, disable_tools: Sequence[str] = (), - user_config: Mapping[str, Any] | None = None, + user_config: Mapping[str, object] | None = None, instructions_path: str | None = None, skills_path: str | None = None, ) -> Mapping[str, object]: @@ -355,10 +363,16 @@ def build_opencode_config( validate_user_config(user) qualified = f"{OPENCODE_PROVIDER_ID}/{model}" extra_instructions: Final = (instructions_path,) if instructions_path else () - instructions: Final = [*(user.get("instructions") or ()), *extra_instructions] # mutable-ok: opencode config JSON + instructions: Final = [ + *_spread_items(user.get("instructions") or ()), + *extra_instructions, + ] # mutable-ok: opencode config JSON user_skills = _as_dict(user.get("skills")) extra_skills: Final = (skills_path,) if skills_path else () - skill_paths: Final = [*(user_skills.get("paths") or ()), *extra_skills] # mutable-ok: opencode config JSON + skill_paths: Final = [ + *_spread_items(user_skills.get("paths") or ()), + *extra_skills, + ] # mutable-ok: opencode config JSON options: Final = {"baseURL": base_url, "apiKey": "{file:" + token_path + "}"} # mutable-ok: opencode config JSON models: Final[dict[str, Mapping[str, object]]] = {model: {}} # mutable-ok: opencode config JSON provider: Final = { # mutable-ok: opencode config JSON diff --git a/litellm/llms/openrouter/image_edit/transformation.py b/litellm/llms/openrouter/image_edit/transformation.py index 9644ee3f280..c2d8398aead 100644 --- a/litellm/llms/openrouter/image_edit/transformation.py +++ b/litellm/llms/openrouter/image_edit/transformation.py @@ -47,7 +47,7 @@ from typing import TYPE_CHECKING, Any, Final, cast import httpx from httpx._types import RequestFiles -from pydantic import ConfigDict, RootModel, with_config +from pydantic import ConfigDict, RootModel, TypeAdapter, with_config from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm @@ -74,6 +74,8 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_DECODED_JSON: Final = TypeAdapter(object) + @with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) class _ImageUrl(TypedDict): @@ -248,7 +250,7 @@ class OpenRouterImageEditConfig(BaseImageEditConfig): logging_obj: LiteLLMLoggingObj, ) -> ImageResponse: try: - response_json: Final[object] = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise OpenRouterException( message=f"Error parsing OpenRouter response: {e}", diff --git a/litellm/llms/openrouter/image_generation/transformation.py b/litellm/llms/openrouter/image_generation/transformation.py index ce6e62aaa88..a9e07555d6a 100644 --- a/litellm/llms/openrouter/image_generation/transformation.py +++ b/litellm/llms/openrouter/image_generation/transformation.py @@ -59,6 +59,7 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_DECODED_JSON: Final = TypeAdapter(object) _JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) @@ -359,7 +360,7 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig): ImageResponse: Populated image response """ try: - response_json: Final = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise OpenRouterException( message=f"Error parsing OpenRouter response: {e}", diff --git a/litellm/llms/opensandbox/sandbox/transformation.py b/litellm/llms/opensandbox/sandbox/transformation.py index 574fbfa9e5b..d30a9ff534e 100644 --- a/litellm/llms/opensandbox/sandbox/transformation.py +++ b/litellm/llms/opensandbox/sandbox/transformation.py @@ -20,6 +20,7 @@ from litellm.constants import ( OPEN_SANDBOX_POLL_INTERVAL, OPEN_SANDBOX_READY_TIMEOUT, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.sandbox.transformation import ( @@ -44,6 +45,13 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i _JSON_VALUE: Final = TypeAdapter(object) +def _decoded_json_object(value: object) -> Mapping[str, object]: + mapping: Final = as_str_mapping(value) + if mapping is not None: + return mapping + raise AttributeError(f"{type(value).__name__!r} object has no attribute 'get'", name="get", obj=value) + + class OpenSandboxSandboxConfig(BaseSandboxConfig): def _http(self, client: AsyncHTTPHandler | None) -> AsyncHTTPHandler: if client is not None: @@ -321,7 +329,7 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig): headers=headers, params={"use_server_proxy": use_server_proxy}, ) - data: Final = response.json() + data: Final = _decoded_json_object(_JSON_VALUE.validate_python(response.json())) endpoint: Final = data.get("endpoint") if not endpoint: raise ValueError(f"OpenSandbox did not return an execd endpoint for {sandbox_id}") @@ -492,7 +500,7 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig): if not data: return None try: - parsed: Final = json.loads(data) + parsed: Final = _JSON_VALUE.validate_python(json.loads(data)) except json.JSONDecodeError: return None if not isinstance(parsed, dict): diff --git a/litellm/llms/pass_through/guardrail_translation/handler.py b/litellm/llms/pass_through/guardrail_translation/handler.py index 885fc98ffea..3ce8d7ddad5 100644 --- a/litellm/llms/pass_through/guardrail_translation/handler.py +++ b/litellm/llms/pass_through/guardrail_translation/handler.py @@ -6,11 +6,14 @@ It uses the field targeting configuration from litellm_logging_obj to extract specific fields for guardrail processing. """ -from collections.abc import AsyncIterable, AsyncIterator, Callable, Mapping, Sequence +from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Mapping, Sequence from typing import TYPE_CHECKING, Final, Optional, Protocol, runtime_checkable +from pydantic import TypeAdapter + from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import without_server_streaming_classification +from litellm.litellm_core_utils.core_helpers import has_attributes from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, StreamingScanKey, @@ -215,6 +218,19 @@ class _EventStreamEndpointChecker(Protocol): def event_stream_endpoint_is_de_anonymizable(self, endpoint: str, model: str, body: object) -> bool: ... +class _EventStreamDeAnonymizer(Protocol): + async def de_anonymize_event_stream( + self, + body_bytes: bytes, + proxy_logging_obj: "ProxyLogging", + user_api_key_dict: "UserAPIKeyAuth", + data: Mapping[str, object], + ) -> bytes: ... + + +_TRANSLATION_RESULT: Final = TypeAdapter(object) + + _PROVIDER_HANDLERS: dict[str, type[BaseTranslation]] = {} @@ -260,10 +276,12 @@ class LlmPassthroughRouteHandler(BaseTranslation): provider, ) return data - return await handler_cls().process_input_messages( - data=data, - guardrail_to_apply=guardrail_to_apply, - litellm_logging_obj=litellm_logging_obj, + return _TRANSLATION_RESULT.validate_python( + await handler_cls().process_input_messages( + data=data, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + ) ) async def process_output_response( @@ -282,12 +300,14 @@ class LlmPassthroughRouteHandler(BaseTranslation): provider, ) return response - return await handler_cls().process_output_response( - response=response, - guardrail_to_apply=guardrail_to_apply, - litellm_logging_obj=litellm_logging_obj, - user_api_key_dict=user_api_key_dict, - request_data=request_data, + return _TRANSLATION_RESULT.validate_python( + await handler_cls().process_output_response( + response=response, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=request_data, + ) ) @staticmethod @@ -399,9 +419,11 @@ class LlmPassthroughRouteHandler(BaseTranslation): return handler_cls.event_stream_media_type() @staticmethod - def _resolve_event_stream_de_anonymizer(provider: str | None): + def _resolve_event_stream_de_anonymizer(provider: str | None) -> Callable[..., Awaitable[bytes]] | None: handler_cls: Final = _get_provider_handlers().get(provider or "") - return getattr(handler_cls, "de_anonymize_event_stream", None) + if not has_attributes(handler_cls, _EventStreamDeAnonymizer, "de_anonymize_event_stream"): + return None + return handler_cls.de_anonymize_event_stream @staticmethod def supports_event_stream_relay(provider: object, endpoint: object, model: object, body: object) -> bool: diff --git a/litellm/llms/perplexity/responses/transformation.py b/litellm/llms/perplexity/responses/transformation.py index a2b7eceea40..77817a5e5bb 100644 --- a/litellm/llms/perplexity/responses/transformation.py +++ b/litellm/llms/perplexity/responses/transformation.py @@ -12,6 +12,7 @@ Ref: https://docs.perplexity.ai/api-reference/responses-post from typing import Any, Final import httpx +from pydantic import TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import join_api_base @@ -22,6 +23,8 @@ from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +_DECODED_JSON: Final = TypeAdapter(object) + class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): def get_supported_openai_params(self, model: str) -> list: @@ -104,7 +107,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): ) -> ResponsesAPIResponse: """Check for Perplexity's status:'failed' on HTTP 200 before delegating to base.""" try: - raw_response_json = raw_response.json() + raw_response_json = _DECODED_JSON.validate_python(raw_response.json()) except Exception: raw_response_json = None diff --git a/litellm/llms/petals/completion/transformation.py b/litellm/llms/petals/completion/transformation.py index ee20c2b12d5..a875992448a 100644 --- a/litellm/llms/petals/completion/transformation.py +++ b/litellm/llms/petals/completion/transformation.py @@ -1,3 +1,5 @@ +from collections.abc import Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Final from httpx import Headers, Response @@ -57,7 +59,7 @@ class PetalsConfig(BaseConfig): top_p: float | None = None, repetition_penalty: float | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/pg_vector/vector_stores/transformation.py b/litellm/llms/pg_vector/vector_stores/transformation.py index eeecd02653d..89851f60d76 100644 --- a/litellm/llms/pg_vector/vector_stores/transformation.py +++ b/litellm/llms/pg_vector/vector_stores/transformation.py @@ -76,7 +76,7 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig): api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, ) -> tuple[str, dict]: encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id") url: Final = join_api_base(api_base, f"{encoded_vector_store_id}/search") diff --git a/litellm/llms/recraft/image_generation/transformation.py b/litellm/llms/recraft/image_generation/transformation.py index 7b1a7714ec6..49a2938e1c0 100644 --- a/litellm/llms/recraft/image_generation/transformation.py +++ b/litellm/llms/recraft/image_generation/transformation.py @@ -26,6 +26,7 @@ else: _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class RecraftImageGenerationConfig(BaseImageGenerationConfig): @@ -134,7 +135,7 @@ class RecraftImageGenerationConfig(BaseImageGenerationConfig): https://www.recraft.ai/docs#generate-image """ try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", diff --git a/litellm/llms/runwayml/image_generation/transformation.py b/litellm/llms/runwayml/image_generation/transformation.py index e721ac678a1..23c8c86e7ca 100644 --- a/litellm/llms/runwayml/image_generation/transformation.py +++ b/litellm/llms/runwayml/image_generation/transformation.py @@ -1,9 +1,10 @@ import asyncio import time from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, cast import httpx +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_logger @@ -11,6 +12,7 @@ from litellm.constants import ( RUNWAYML_DEFAULT_API_VERSION, RUNWAYML_POLLING_TIMEOUT, ) +from litellm.litellm_core_utils.core_helpers import decoded_json_object from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, @@ -36,9 +38,25 @@ class _RunwayMLTask(TypedDict, total=False): id: ReadOnly[str] status: ReadOnly[str] - output: ReadOnly[Sequence[str | Mapping[str, str]]] - failure: ReadOnly[str] - failureCode: ReadOnly[str] + output: ReadOnly[Sequence[str | Mapping[str, str | None]] | None] + failure: ReadOnly[str | None] + failureCode: ReadOnly[str | None] + + +_JSON_VALUE: Final = TypeAdapter(object) + + +def _decoded_json(response: httpx.Response) -> object: + return _JSON_VALUE.validate_python(response.json()) + + +def _runway_task(decoded: object) -> _RunwayMLTask: + task: Final = decoded_json_object(decoded) + return cast(_RunwayMLTask, task) # cast-ok: the payload passes through unvalidated, as before it was typed + + +def _parse_runway_task(response: httpx.Response) -> _RunwayMLTask: + return _runway_task(_decoded_json(response)) class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): @@ -234,7 +252,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): response = client.get(url=task_url, headers=headers) response.raise_for_status() - response_data: _RunwayMLTask = response.json() + response_data: _RunwayMLTask = _parse_runway_task(response) # Check task status status = self._check_task_status(response_data=response_data) @@ -282,7 +300,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): response = await client.get(url=task_url, headers=headers) response.raise_for_status() - response_data: _RunwayMLTask = response.json() + response_data: _RunwayMLTask = _parse_runway_task(response) # Check task status status = self._check_task_status(response_data=response_data) @@ -328,13 +346,14 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): } """ try: - response_data: _RunwayMLTask = raw_response.json() + decoded: Final = _decoded_json(raw_response) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", status_code=raw_response.status_code, headers=raw_response.headers, ) + response_data: _RunwayMLTask = _runway_task(decoded) verbose_logger.debug("RunwayML starting polling...") @@ -358,7 +377,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): ) # Update response_data with polled result - response_data = raw_response.json() + response_data = _parse_runway_task(raw_response) verbose_logger.debug("RunwayML polling complete, transforming to OpenAI format") @@ -388,13 +407,14 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): We need to poll the task until it completes (status SUCCEEDED) using async polling. """ try: - response_data: _RunwayMLTask = raw_response.json() + decoded: Final = _decoded_json(raw_response) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", status_code=raw_response.status_code, headers=raw_response.headers, ) + response_data: _RunwayMLTask = _runway_task(decoded) verbose_logger.debug("RunwayML starting polling (async)...") @@ -418,7 +438,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): ) # Update response_data with polled result - response_data = raw_response.json() + response_data = _parse_runway_task(raw_response) verbose_logger.debug("RunwayML polling complete (async), transforming to OpenAI format") diff --git a/litellm/llms/runwayml/text_to_speech/transformation.py b/litellm/llms/runwayml/text_to_speech/transformation.py index 1f052648f13..ce1ec1714f1 100644 --- a/litellm/llms/runwayml/text_to_speech/transformation.py +++ b/litellm/llms/runwayml/text_to_speech/transformation.py @@ -41,6 +41,11 @@ class _RunwayTtsTaskResponse(TypedDict, total=False): failureCode: ReadOnly[str] +def _parse_runway_tts_task(response: httpx.Response) -> _RunwayTtsTaskResponse: + task: Final[_RunwayTtsTaskResponse] = response.json() + return task + + class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): """ Configuration for RunwayML Text-to-Speech @@ -321,7 +326,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): response = client.get(url=task_url, headers=headers) response.raise_for_status() - response_data: _RunwayTtsTaskResponse = response.json() + response_data: _RunwayTtsTaskResponse = _parse_runway_tts_task(response) # Check task status status = self._check_task_status(response_data=response_data) @@ -368,7 +373,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): response = await client.get(url=task_url, headers=headers) response.raise_for_status() - response_data: _RunwayTtsTaskResponse = response.json() + response_data: _RunwayTtsTaskResponse = _parse_runway_tts_task(response) # Check task status status = self._check_task_status(response_data=response_data) @@ -459,7 +464,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): from litellm.types.llms.openai import HttpxBinaryResponseContent try: - response_data: Final[_RunwayTtsTaskResponse] = raw_response.json() + response_data: Final[_RunwayTtsTaskResponse] = _parse_runway_tts_task(raw_response) except Exception as e: raise self.get_error_class( error_message=f"Error parsing RunwayML TTS response: {e}", @@ -489,7 +494,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): ) # Get the completed task data - task_data: Final[_RunwayTtsTaskResponse] = polled_response.json() + task_data: Final[_RunwayTtsTaskResponse] = _parse_runway_tts_task(polled_response) verbose_logger.debug("RunwayML TTS polling complete, downloading audio") @@ -528,7 +533,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): from litellm.types.llms.openai import HttpxBinaryResponseContent try: - response_data: Final[_RunwayTtsTaskResponse] = raw_response.json() + response_data: Final[_RunwayTtsTaskResponse] = _parse_runway_tts_task(raw_response) except Exception as e: raise self.get_error_class( error_message=f"Error parsing RunwayML TTS response: {e}", @@ -558,7 +563,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): ) # Get the completed task data - task_data: Final[_RunwayTtsTaskResponse] = polled_response.json() + task_data: Final[_RunwayTtsTaskResponse] = _parse_runway_tts_task(polled_response) verbose_logger.debug("RunwayML TTS polling complete (async), downloading audio") diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index 106567f6fba..ad8a24c1535 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -119,7 +119,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): @staticmethod def _parse_task_response(raw_response: httpx.Response) -> _RunwayTaskResponse: - return raw_response.json() + return _parse_runway_task_response(raw_response) def get_supported_openai_params(self, model: str) -> list: """ diff --git a/litellm/llms/sagemaker/common_utils.py b/litellm/llms/sagemaker/common_utils.py index 14f0e3ead62..eaa805e7769 100644 --- a/litellm/llms/sagemaker/common_utils.py +++ b/litellm/llms/sagemaker/common_utils.py @@ -1,3 +1,4 @@ +import builtins import codecs import functools import json @@ -34,6 +35,14 @@ class _SageMakerMessagesChunkData(TypedDict, total=False): choices: ReadOnly[Required[list[Mapping[str, object]]]] +class _SageMakerPayloadPart(TypedDict, total=False): + bytes: ReadOnly[builtins.bytes] + + +class _SageMakerStreamEvent(TypedDict, total=False): + chunk: ReadOnly[_SageMakerPayloadPart] + + class _SageMakerChunkDataRoot(RootModel[_SageMakerChunkData]): model_config = ConfigDict(frozen=True, hide_input_in_errors=True) @@ -345,7 +354,7 @@ class AWSEventStreamDecoder: ), ) response_dict: Final = event.to_response_dict() - parsed_response: Final = self.parser.parse(response_dict, response_stream_shape) + parsed_response: Final[_SageMakerStreamEvent] = self.parser.parse(response_dict, response_stream_shape) body: Final = response_dict.get("body") if response_dict["status_code"] != 200: diff --git a/litellm/llms/sagemaker/embedding/cohere_transformation.py b/litellm/llms/sagemaker/embedding/cohere_transformation.py index 35352f65dda..8c0f4437edf 100644 --- a/litellm/llms/sagemaker/embedding/cohere_transformation.py +++ b/litellm/llms/sagemaker/embedding/cohere_transformation.py @@ -10,7 +10,8 @@ be of type Object`. Reference: https://docs.cohere.com/v2/reference/embed """ -from typing import TYPE_CHECKING, Any, cast +from collections.abc import Sequence +from typing import TYPE_CHECKING, cast if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -25,6 +26,7 @@ from litellm.llms.bedrock.embed.cohere_transformation import ( BedrockCohereEmbeddingConfig, ) from litellm.llms.cohere.embed.v1_transformation import CohereEmbeddingConfig +from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import EmbeddingResponse from ..common_utils import SagemakerError @@ -123,7 +125,7 @@ class SagemakerCohereEmbeddingConfig(BaseEmbeddingConfig): self, headers: dict, model: str, - messages: list[Any], + messages: Sequence[AllMessageValues], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/sagemaker/embedding/openai_transformation.py b/litellm/llms/sagemaker/embedding/openai_transformation.py index a3f0ebfbf59..1222c697832 100644 --- a/litellm/llms/sagemaker/embedding/openai_transformation.py +++ b/litellm/llms/sagemaker/embedding/openai_transformation.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: from litellm.types.llms.openai import AllEmbeddingInputValues from httpx._models import Headers, Response -from pydantic import BaseModel, ConfigDict, ValidationError +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig @@ -25,6 +25,7 @@ from litellm.types.utils import EmbeddingResponse, Usage from ..common_utils import STR_KEYED_MAPPING, SagemakerError SUPPORTED_OPENAI_PARAMS: Final = ("dimensions", "encoding_format", "user") +_DECODED_JSON: Final = TypeAdapter(object) class OpenAIEmbeddingUsage(BaseModel): @@ -94,7 +95,7 @@ class SagemakerOpenAIEmbeddingConfig(BaseEmbeddingConfig): litellm_params: Mapping[str, object] | None = None, ) -> "EmbeddingResponse": try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except ValueError as e: raise SagemakerError( message=f"Failed to parse response: {e}", diff --git a/litellm/llms/sagemaker/embedding/transformation.py b/litellm/llms/sagemaker/embedding/transformation.py index 1395f9b98ec..2cd02683167 100644 --- a/litellm/llms/sagemaker/embedding/transformation.py +++ b/litellm/llms/sagemaker/embedding/transformation.py @@ -4,7 +4,8 @@ Translate from OpenAI's `/v1/embeddings` to Sagemaker's `/invoke` In the Huggingface TGI format. """ -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Sequence +from typing import TYPE_CHECKING, Final if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -15,6 +16,7 @@ from httpx._models import Headers, Response from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.voyage.embedding.transformation import VoyageEmbeddingConfig +from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import EmbeddingResponse, Usage from ..common_utils import SagemakerError @@ -158,7 +160,7 @@ class SagemakerEmbeddingConfig(BaseEmbeddingConfig): self, headers: dict, model: str, - messages: list[Any], + messages: Sequence[AllMessageValues], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index 8a7baf18e8f..622c28f3c90 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any -from ..credentials import get_token_creator +from ..credentials import AICoreConfiguration, AICoreDeploymentList, get_token_creator from .handler import ( AsyncSAPStreamIterator, GenAIHubOrchestrationError, @@ -186,11 +186,13 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): # Keep a short, tight client lifecycle here to avoid fd leaks client: Final = litellm.module_level_client # with httpx.Client(timeout=30) as client: - deployments: Final = client.get(join_api_base(self.base_url, "lm/deployments"), headers=self.headers).json() + deployments: Final[AICoreDeploymentList] = client.get( + join_api_base(self.base_url, "lm/deployments"), headers=self.headers + ).json() valid: Final[list[tuple[str, str]]] = [] for dep in deployments.get("resources", []): if dep.get("scenarioId") == "orchestration": - cfg = client.get( + cfg: AICoreConfiguration = client.get( join_api_base(self.base_url, f"lm/configurations/{dep['configurationId']}"), headers=self.headers, ).json() diff --git a/litellm/llms/sap/credentials.py b/litellm/llms/sap/credentials.py index 2622d0fa9b6..74d065956cc 100644 --- a/litellm/llms/sap/credentials.py +++ b/litellm/llms/sap/credentials.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from datetime import datetime, timedelta, timezone from pathlib import Path from threading import Lock -from typing import Any, Final, Protocol +from typing import Final, Protocol import httpx from pydantic import TypeAdapter @@ -34,6 +34,21 @@ DEFAULT_HOME_PATH: Final = os.path.join(os.path.expanduser("~"), ".aicore") _DECODED_JSON: Final = TypeAdapter(object) +class AICoreDeployment(TypedDict): + configurationId: ReadOnly[str] + scenarioId: ReadOnly[str] + deploymentUrl: ReadOnly[str] + createdAt: ReadOnly[str] + + +class AICoreDeploymentList(TypedDict, total=False): + resources: ReadOnly[Sequence[AICoreDeployment]] + + +class AICoreConfiguration(TypedDict): + executableId: ReadOnly[str] + + def _get_home() -> str: return os.getenv(HOME_PATH_ENV_VAR, DEFAULT_HOME_PATH) @@ -59,7 +74,7 @@ def _get_nested(d: object, path: Sequence[str]) -> object: return cur -def _load_json_env(var_name: str) -> dict[str, object] | None: +def _load_json_env(var_name: str) -> Mapping[str, Sequence[Mapping[str, object]]] | None: raw: Final = os.environ.get(var_name) if not raw: return None @@ -76,7 +91,7 @@ def _str_or_none(value: object) -> str | None: return None -def _load_vcap() -> dict[str, Any]: +def _load_vcap() -> Mapping[str, Sequence[Mapping[str, object]]]: return _load_json_env(VCAP_SERVICES_ENV_VAR) or {} @@ -196,7 +211,7 @@ def resolve_resource_group(sources: list[Source]) -> str | None: def _parse_service_key_once( service_key: str | dict | None, -) -> dict[str, object] | None: +) -> object: """ Pre-parse service_key if it's a string to avoid repeated JSON parsing. @@ -208,7 +223,7 @@ def _parse_service_key_once( return service_key if isinstance(service_key, str): try: - return json.loads(service_key) + return _DECODED_JSON.validate_python(json.loads(service_key)) except json.JSONDecodeError: verbose_logger.warning("SAP service key is a string but not valid JSON. Skipping this source.") return None @@ -218,7 +233,7 @@ def _parse_service_key_once( return None -def _resolve_credential_from_service_key(service_key: str | dict | None, cv: CredentialsValue) -> str | None: +def _resolve_credential_from_service_key(service_key: object, cv: CredentialsValue) -> str | None: if service_key is None: return None val: Final = _str_or_none(_get_nested(service_key, (("credentials",) + cv.vcap_key) if cv.vcap_key else (cv.name,))) @@ -253,14 +268,16 @@ def fetch_credentials( """ config: Final = init_conf(profile) - service_key = _parse_service_key_once(service_key or litellm.sap_service_key or os.environ.get(SERVICE_KEY_ENV_VAR)) + parsed_service_key: Final = _parse_service_key_once( + service_key or litellm.sap_service_key or os.environ.get(SERVICE_KEY_ENV_VAR) + ) vcap_service: Final = _get_vcap_service(VCAP_AICORE_SERVICE_NAME) sources: Final = [ Source("kwargs", lambda cv: _str_or_none(kwargs.get(cv.name))), Source( "service key", - lambda cv: _resolve_credential_from_service_key(service_key, cv), + lambda cv: _resolve_credential_from_service_key(parsed_service_key, cv), ), Source( "environment variables", diff --git a/litellm/llms/sap/embed/transformation.py b/litellm/llms/sap/embed/transformation.py index 107fef0b355..a59b7e1e00d 100644 --- a/litellm/llms/sap/embed/transformation.py +++ b/litellm/llms/sap/embed/transformation.py @@ -19,7 +19,7 @@ from litellm.types.llms.openai import AllEmbeddingInputValues from litellm.types.utils import EmbeddingResponse from ..chat.handler import GenAIHubOrchestrationError -from ..credentials import get_token_creator +from ..credentials import AICoreConfiguration, AICoreDeploymentList, get_token_creator class Usage(LiteLLMBaseModel): @@ -102,10 +102,12 @@ class GenAIHubEmbeddingConfig(BaseEmbeddingConfig): def deployment_url(self) -> str: with httpx.Client(timeout=30) as client: valid_deployments: Final = [] - deployments: Final = client.get(join_api_base(self.base_url, "lm/deployments"), headers=self.headers).json() + deployments: Final[AICoreDeploymentList] = client.get( + join_api_base(self.base_url, "lm/deployments"), headers=self.headers + ).json() for deployment in deployments.get("resources", []): if deployment["scenarioId"] == "orchestration": - config_details = client.get( + config_details: AICoreConfiguration = client.get( join_api_base(self.base_url, f"lm/configurations/{deployment['configurationId']}"), headers=self.headers, ).json() diff --git a/litellm/llms/stability/image_edit/transformations.py b/litellm/llms/stability/image_edit/transformations.py index 5d5fad6128e..974996c5e48 100644 --- a/litellm/llms/stability/image_edit/transformations.py +++ b/litellm/llms/stability/image_edit/transformations.py @@ -6,10 +6,12 @@ Handles transformation between OpenAI-compatible format and Stability AI API for API Reference: https://platform.stability.ai/docs/api-reference """ +from collections.abc import Sequence from typing import TYPE_CHECKING, Any, Final, cast import httpx from httpx._types import RequestFiles +from typing_extensions import ReadOnly, TypedDict from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.secret_managers.main import get_secret_str @@ -30,6 +32,13 @@ else: LiteLLMLoggingObj = Any +class _StabilityImageEditResponse(TypedDict, total=False): + image: ReadOnly[str] + finish_reason: ReadOnly[str] + seed: ReadOnly[int] + errors: ReadOnly[Sequence[str]] + + class StabilityImageEditConfig(BaseImageEditConfig): """ Configuration for Stability AI image edit. @@ -259,7 +268,7 @@ class StabilityImageEditConfig(BaseImageEditConfig): OpenAI expects: {"data": [{"b64_json": "base64..."}], "created": timestamp} """ try: - response_data: Final = raw_response.json() + response_data: Final[_StabilityImageEditResponse] = raw_response.json() except Exception as e: raise self.get_error_class( error_message=f"Error parsing Stability AI response: {e}", diff --git a/litellm/llms/tencent/messages/transformation.py b/litellm/llms/tencent/messages/transformation.py index 1e52bc37d47..30c3ecaf3a9 100644 --- a/litellm/llms/tencent/messages/transformation.py +++ b/litellm/llms/tencent/messages/transformation.py @@ -5,8 +5,6 @@ Tencent TokenHub exposes an Anthropic-compatible Messages API endpoint alongside its standard OpenAI-compatible chat completions endpoint. """ -from typing import Any - import litellm from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.pass_through.messages.transformation import ( @@ -48,7 +46,7 @@ class TencentAnthropicMessagesConfig(AnthropicMessagesConfig): self, headers: dict, model: str, - messages: list[Any], + messages: list[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index b37bbd78f2b..fe0f96cb805 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -7,6 +7,7 @@ from collections.abc import AsyncIterator, Iterator from typing import TYPE_CHECKING, Final, Literal from httpx import Headers, Response +from typing_extensions import ReadOnly, TypedDict from litellm.constants import DEFAULT_MAX_TOKENS_FOR_TRITON from litellm.litellm_core_utils.prompt_templates.factory import prompt_factory @@ -32,6 +33,10 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer +class _TritonGenerateResponse(TypedDict): + text_output: ReadOnly[str] + + class TritonConfig(BaseConfig): """ Base class for Triton configurations. @@ -220,7 +225,7 @@ class TritonGenerateConfig(TritonConfig): json_mode: bool | None = None, ) -> ModelResponse: try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final[_TritonGenerateResponse] = raw_response.json() except Exception: raise TritonError(message=raw_response.text, status_code=raw_response.status_code) model_response.choices = [Choices(index=0, message=Message(content=raw_response_json["text_output"]))] diff --git a/litellm/llms/triton/embedding/transformation.py b/litellm/llms/triton/embedding/transformation.py index 8c6e0b832fc..3a735f741fd 100644 --- a/litellm/llms/triton/embedding/transformation.py +++ b/litellm/llms/triton/embedding/transformation.py @@ -1,6 +1,8 @@ +from collections.abc import Sequence from typing import Final import httpx +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.llms.base_llm.chat.transformation import AllMessageValues, BaseLLMException from litellm.llms.base_llm.embedding.transformation import ( @@ -14,6 +16,16 @@ from litellm.utils import token_counter from ..common_utils import TritonError +class _TritonEmbeddingOutput(TypedDict): + shape: ReadOnly[Sequence[int]] + data: ReadOnly[Sequence[float]] + + +class _TritonEmbeddingResponse(TypedDict): + outputs: ReadOnly[Sequence[_TritonEmbeddingOutput]] + model_name: NotRequired[ReadOnly[str]] + + class TritonEmbeddingConfig(BaseEmbeddingConfig): """ Transformations for triton /embeddings endpoint (This is a trtllm model) @@ -79,7 +91,7 @@ class TritonEmbeddingConfig(BaseEmbeddingConfig): litellm_params: dict = {}, ) -> EmbeddingResponse: try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final[_TritonEmbeddingResponse] = raw_response.json() except Exception: raise TritonError(message=raw_response.text, status_code=raw_response.status_code) @@ -134,7 +146,7 @@ class TritonEmbeddingConfig(BaseEmbeddingConfig): return TritonError(message=error_message, status_code=status_code, headers=headers) @staticmethod - def split_embedding_by_shape(data: list[float], shape: list[int]) -> list[list[float]]: + def split_embedding_by_shape(data: Sequence[float], shape: Sequence[int]) -> list[Sequence[float]]: if len(shape) != 2: raise ValueError("Shape must be of length 2.") embedding_size: Final = shape[1] diff --git a/litellm/llms/vercel_ai_gateway/chat/transformation.py b/litellm/llms/vercel_ai_gateway/chat/transformation.py index 3fc74b6acc1..cf633358a41 100644 --- a/litellm/llms/vercel_ai_gateway/chat/transformation.py +++ b/litellm/llms/vercel_ai_gateway/chat/transformation.py @@ -6,9 +6,11 @@ Calls done in OpenAI/openai.py as Vercel AI Gateway is openai-compatible. Docs: https://vercel.com/docs/ai-gateway """ +from collections.abc import Sequence from typing import Final import httpx +from typing_extensions import ReadOnly, TypedDict import litellm from litellm.litellm_core_utils.url_utils import join_api_base @@ -20,6 +22,14 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig from ..common_utils import VercelAIGatewayException +class _VercelAIGatewayModel(TypedDict): + id: ReadOnly[str] + + +class _VercelAIGatewayModelList(TypedDict): + data: ReadOnly[Sequence[_VercelAIGatewayModel]] + + class VercelAIGatewayConfig(OpenAIGPTConfig): @property def custom_llm_provider(self) -> str | None: @@ -99,5 +109,5 @@ class VercelAIGatewayConfig(OpenAIGPTConfig): if response.status_code != 200: raise Exception(f"Failed to get models: {response.text}") - models: Final = response.json()["data"] - return [model["id"] for model in models] + models: Final[_VercelAIGatewayModelList] = response.json() + return [model["id"] for model in models["data"]] diff --git a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py index 39d2e9b4c49..26de755d5ca 100644 --- a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py +++ b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py @@ -3,6 +3,7 @@ from collections.abc import Mapping, Sequence from typing import Final from httpx import Headers, Response +from pydantic import TypeAdapter import litellm from litellm.exceptions import UnsupportedParamsError @@ -45,6 +46,7 @@ from litellm.types.utils import ( DEFAULT_GEMINI_TRANSCRIBE_LOCATION: Final = "global" AUDIO_MODALITY: Final = "AUDIO" +_DECODED_JSON: Final = TypeAdapter(object) class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase): @@ -177,7 +179,7 @@ class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexB raw_response: Response, ) -> TranscriptionResponse: try: - response_json: Final = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except ValueError: raise VertexAIError( status_code=raw_response.status_code, diff --git a/litellm/llms/vertex_ai/batches/transformation.py b/litellm/llms/vertex_ai/batches/transformation.py index a8d02045740..99ef750b5fc 100644 --- a/litellm/llms/vertex_ai/batches/transformation.py +++ b/litellm/llms/vertex_ai/batches/transformation.py @@ -202,7 +202,7 @@ class VertexAIBatchTransformation: @classmethod def transform_vertex_ai_batch_list_response_to_openai_list_response( cls, response: dict[str, Any] - ) -> dict[str, Any]: + ) -> Mapping[str, object]: """ Transforms Vertex AI batch list response into OpenAI-compatible list response. """ diff --git a/litellm/llms/vertex_ai/files/handler.py b/litellm/llms/vertex_ai/files/handler.py index f2da04a7db7..54c0fa97550 100644 --- a/litellm/llms/vertex_ai/files/handler.py +++ b/litellm/llms/vertex_ai/files/handler.py @@ -3,7 +3,7 @@ import json import os import time from collections.abc import Coroutine, Mapping -from typing import Any, Final +from typing import Final from urllib.parse import unquote import httpx @@ -197,7 +197,7 @@ class VertexAIFilesHandler(GCSBucketBase): timeout: float | httpx.Timeout, max_retries: int | None, litellm_params: dict | None = None, - ) -> HttpxBinaryResponseContent | Coroutine[Any, Any, HttpxBinaryResponseContent]: + ) -> HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent]: """ Download file content from GCS bucket for VertexAI files. Supports both sync and async operations. diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index e36aba00293..410f29ef88a 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -5,7 +5,7 @@ import json import os import re import time -from collections.abc import AsyncGenerator, Callable, Iterable, Iterator, Mapping +from collections.abc import AsyncGenerator, Callable, Iterable, Iterator, Mapping, Sequence from contextlib import aclosing from dataclasses import dataclass from functools import partial @@ -16,6 +16,7 @@ from urllib.parse import quote, unquote import httpx from httpx import Headers, Response from openai.types.file_deleted import FileDeleted +from pydantic import TypeAdapter from typing_extensions import ReadOnly, Required import litellm @@ -31,7 +32,7 @@ from litellm.litellm_core_utils.cloud_storage_security import ( split_configured_cloud_bucket_name, validate_managed_cloud_file_id, ) -from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_file_data, @@ -96,6 +97,7 @@ _JSONL_NEWLINE: Final = b"\n" _BATCH_OUTPUT_FIRST_ROW_PEEK_LIMIT_BYTES: Final = 32 * 1024 * 1024 _PASSTHROUGH_MANAGED_GCS_PREFIX: Final = f"{VERTEX_AI_MANAGED_GCS_PREFIX}passthrough/" _RAW_UPLOAD_CHUNK_BYTES: Final = 1024 * 1024 +_DECODED_JSON: Final = TypeAdapter(object) class _GcsObjectMetadataJson(TypedDict, total=False): @@ -827,7 +829,7 @@ def _iter_openai_jsonl_entries( def _load_openai_jsonl_row(lineno: int, line: str) -> object: try: - return json.loads(line) + return _DECODED_JSON.validate_python(json.loads(line)) except json.JSONDecodeError as e: raise VertexAIError( status_code=400, @@ -967,7 +969,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): def _get_gcs_object_name_from_batch_jsonl( self, - openai_jsonl_content: list[dict[str, Any]], + openai_jsonl_content: Sequence[Mapping[str, object]], deployment_model: str | None = None, ) -> str: """ @@ -985,8 +987,9 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): `projects/../locations/../endpoints/` batch job model instead of a nonexistent publisher model. """ - first_entry: Final = openai_jsonl_content[0] - body_model: Final = first_entry["body"].get("model") if isinstance(first_entry.get("body"), dict) else None + body: Final = openai_jsonl_content[0].get("body") + body_fields: Final = as_str_mapping(body) + body_model: Final = body_fields.get("model") if body_fields is not None and isinstance(body, dict) else None jsonl_model: Final = body_model if isinstance(body_model, str) else "" raw_model: Final = deployment_model.removeprefix("vertex_ai/") if deployment_model else jsonl_model return _managed_batch_object_name(raw_model, passthrough=False) diff --git a/litellm/llms/vertex_ai/fine_tuning/handler.py b/litellm/llms/vertex_ai/fine_tuning/handler.py index 40564d536a7..7cb52281a6a 100644 --- a/litellm/llms/vertex_ai/fine_tuning/handler.py +++ b/litellm/llms/vertex_ai/fine_tuning/handler.py @@ -27,6 +27,11 @@ from litellm.types.utils import LiteLLMFineTuningJob _JSON_VALUE: Final = TypeAdapter(object) +def _response_tuning_job(response: httpx.Response) -> ResponseTuningJob: + tuning_job: Final[ResponseTuningJob] = response.json() + return ResponseTuningJob(**tuning_job) + + class VertexFineTuningAPI(VertexLLM): """ Vertex methods to support for batches @@ -187,9 +192,7 @@ class VertexFineTuningAPI(VertexLLM): "got response from creating fine tuning job: %s", _JSON_VALUE.validate_python(response.json()) ) - vertex_response: Final = ResponseTuningJob( - **response.json(), - ) + vertex_response: Final = _response_tuning_job(response) verbose_logger.debug("vertex_response %s", vertex_response) open_ai_response: Final = self.convert_vertex_response_to_open_ai_response(vertex_response) @@ -272,9 +275,7 @@ class VertexFineTuningAPI(VertexLLM): verbose_logger.debug( "got response from creating fine tuning job: %s", _JSON_VALUE.validate_python(response.json()) ) - vertex_response: Final = ResponseTuningJob( - **response.json(), - ) + vertex_response: Final = _response_tuning_job(response) verbose_logger.debug("vertex_response %s", vertex_response) open_ai_response: Final = self.convert_vertex_response_to_open_ai_response(vertex_response) diff --git a/litellm/llms/vertex_ai/gemini/long_decode.py b/litellm/llms/vertex_ai/gemini/long_decode.py index 571fa94d4a5..25aae70a853 100644 --- a/litellm/llms/vertex_ai/gemini/long_decode.py +++ b/litellm/llms/vertex_ai/gemini/long_decode.py @@ -332,7 +332,8 @@ def with_initial_prompt_tokens(usage: Usage, initial_prompt_tokens: int | None) def parse_body(response: httpx.Response) -> GenerateContentResponseBody: try: - return GenerateContentResponseBody(**response.json()) + body: Final[GenerateContentResponseBody] = response.json() + return GenerateContentResponseBody(**body) except Exception as e: raise VertexAIError( message=( diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py index 0a800497de1..f15c6393cb2 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py @@ -6,6 +6,7 @@ import json from typing import TYPE_CHECKING, Final, Literal import httpx +from typing_extensions import ReadOnly, TypedDict import litellm from litellm.llms.custom_httpx.http_handler import ( @@ -35,6 +36,11 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +class _GeminiFileResource(TypedDict, total=False): + mimeType: ReadOnly[str] + uri: ReadOnly[str] + + class GoogleBatchEmbeddings(VertexLLM): @staticmethod def _flatten_and_detect_file_refs( @@ -74,7 +80,7 @@ class GoogleBatchEmbeddings(VertexLLM): if response.status_code != 200: raise Exception(f"Error fetching file {element}: {response.status_code} {response.text}") - file_data = response.json() + file_data: _GeminiFileResource = response.json() resolved_files[element] = { "mime_type": file_data.get("mimeType", ""), "uri": file_data.get("uri", element), @@ -111,7 +117,7 @@ class GoogleBatchEmbeddings(VertexLLM): if response.status_code != 200: raise Exception(f"Error fetching file {element}: {response.status_code} {response.text}") - file_data = response.json() + file_data: _GeminiFileResource = response.json() resolved_files[element] = { "mime_type": file_data.get("mimeType", ""), "uri": file_data.get("uri", element), diff --git a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py index 725a7f39917..6d071363823 100644 --- a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py @@ -190,7 +190,7 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM): payload: Final = json.dumps(request_body) empty_files: Final = cast(RequestFiles, []) - return cast(tuple[dict[str, Any], RequestFiles | None], (payload, empty_files)) + return cast(tuple[dict[str, object], RequestFiles | None], (payload, empty_files)) def transform_image_edit_response( self, diff --git a/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py index 085ff715c30..deaa32f7308 100644 --- a/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py +++ b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Final, cast import httpx from httpx._types import RequestFiles from pydantic import ConfigDict, TypeAdapter +from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH @@ -34,6 +35,10 @@ _JSON_CONTAINER: Final[TypeAdapter[Mapping[str, object] | str | Sequence[object] ) +class _VertexImagenPredictResponse(TypedDict, total=False): + predictions: ReadOnly[Sequence[object]] + + class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): """ Vertex AI Imagen Image Edit Configuration @@ -207,7 +212,7 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): ) -> ImageResponse: model_response: Final = ImageResponse() try: - response_json: Final = raw_response.json() + response_json: Final[_VertexImagenPredictResponse] = raw_response.json() except Exception as exc: raise self.get_error_class( error_message=f"Error transforming image edit response: {exc}", diff --git a/litellm/llms/vertex_ai/image_generation/image_generation_handler.py b/litellm/llms/vertex_ai/image_generation/image_generation_handler.py index 7ba7b060991..ec915e10fcd 100644 --- a/litellm/llms/vertex_ai/image_generation/image_generation_handler.py +++ b/litellm/llms/vertex_ai/image_generation/image_generation_handler.py @@ -168,7 +168,7 @@ class VertexImageGeneration(VertexLLM): if response.status_code != 200: raise Exception(f"Error: {response.status_code} {response.text}") - json_response: Final = response.json() + json_response: Final[Mapping[str, object]] = response.json() return self.process_image_generation_response(json_response, model_response, model) async def aimage_generation( @@ -272,7 +272,7 @@ class VertexImageGeneration(VertexLLM): if response.status_code != 200: raise Exception(f"Error: {response.status_code} {response.text}") - json_response: Final = response.json() + json_response: Final[Mapping[str, object]] = response.json() return self.process_image_generation_response(json_response, model_response, model) def is_image_generation_response(self, json_response: dict[str, Any]) -> bool: diff --git a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py index 916e25a027e..f6df772dd56 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py @@ -33,6 +33,7 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_DECODED_JSON: Final = TypeAdapter(object) _JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_CONTAINER: Final[TypeAdapter[Mapping[str, object] | str | Sequence[object]]] = TypeAdapter( @@ -299,7 +300,7 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): Transform Gemini image generation response to litellm ImageResponse format """ try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", diff --git a/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py b/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py index da77fa338db..c282ae22ea6 100644 --- a/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py +++ b/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py @@ -208,7 +208,7 @@ class VertexAIMultimodalEmbeddingConfig(BaseEmbeddingConfig): if raw_response.status_code != 200: raise Exception(f"Error: {raw_response.status_code} {raw_response.text}") - _json_response: Final = raw_response.json() + _json_response: Final[MultimodalPredictions] = raw_response.json() if "predictions" not in _json_response: raise InternalServerError( message=f"embedding response does not contain 'predictions', got {_json_response}", diff --git a/litellm/llms/vertex_ai/rag_engine/transformation.py b/litellm/llms/vertex_ai/rag_engine/transformation.py index a34ef519578..61cb083fa4c 100644 --- a/litellm/llms/vertex_ai/rag_engine/transformation.py +++ b/litellm/llms/vertex_ai/rag_engine/transformation.py @@ -4,7 +4,10 @@ Transformation utilities for Vertex AI RAG Engine. Handles transforming LiteLLM's unified formats to Vertex AI RAG Engine API format. """ -from typing import Any, Final +from collections.abc import Mapping +from typing import Final + +from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_logger from litellm.constants import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE @@ -13,6 +16,15 @@ from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.types.rag import RAGChunkingStrategy +class VertexRAGChunkingConfig(TypedDict): + chunk_size: ReadOnly[int] + chunk_overlap: ReadOnly[int] + + +class VertexRAGTransformationConfig(TypedDict): + chunking_config: ReadOnly[VertexRAGChunkingConfig] + + class VertexAIRAGTransformation(VertexBase): """ Transformation class for Vertex AI RAG Engine API. @@ -55,7 +67,7 @@ class VertexAIRAGTransformation(VertexBase): def transform_chunking_strategy_to_vertex_format( self, chunking_strategy: RAGChunkingStrategy | None, - ) -> dict[str, Any]: + ) -> VertexRAGTransformationConfig: """ Transform LiteLLM's unified chunking_strategy to Vertex AI RAG format. @@ -105,7 +117,7 @@ class VertexAIRAGTransformation(VertexBase): self, gcs_uri: str, chunking_strategy: RAGChunkingStrategy | None = None, - ) -> dict[str, Any]: + ) -> Mapping[str, object]: """ Build the request payload for importing RAG files. diff --git a/litellm/llms/vertex_ai/rerank/transformation.py b/litellm/llms/vertex_ai/rerank/transformation.py index dce4d2f2a87..398471a7c9b 100644 --- a/litellm/llms/vertex_ai/rerank/transformation.py +++ b/litellm/llms/vertex_ai/rerank/transformation.py @@ -6,10 +6,11 @@ Why separate file? Make it easy to see how transformation works import math import uuid -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Final import httpx +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -27,6 +28,17 @@ from litellm.types.rerank import ( ) +class _VertexRankedRecord(TypedDict): + id: ReadOnly[str] + score: NotRequired[ReadOnly[float]] + title: NotRequired[ReadOnly[str]] + content: NotRequired[ReadOnly[str]] + + +class _VertexRankResponse(TypedDict, total=False): + records: ReadOnly[Sequence[_VertexRankedRecord]] + + class VertexAIRerankConfig(BaseRerankConfig, VertexBase): """ Configuration for Vertex AI Discovery Engine Rerank API @@ -172,7 +184,7 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase): Transform Vertex AI Discovery Engine response to Cohere format """ try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final[_VertexRankResponse] = raw_response.json() except Exception as e: raise ValueError(f"Failed to parse response: {e}") diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/ai21/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/ai21/transformation.py index 45cc5320401..3873a4be0b5 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/ai21/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/ai21/transformation.py @@ -1,4 +1,5 @@ import types +from collections.abc import Mapping from typing import Final import litellm @@ -18,16 +19,17 @@ class VertexAIAi21Config(OpenAIGPTConfig): self, max_tokens: int | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index 5d748ad6e74..cfd5b5d3980 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -1,4 +1,5 @@ -from typing import Any, Final +from collections.abc import Sequence +from typing import Final from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.anthropic.pass_through.messages.transformation import ( @@ -29,7 +30,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert self, headers: dict, model: str, - messages: list[Any], + messages: Sequence[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py index ddc83bc53a5..96fccf007a5 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py @@ -1,4 +1,5 @@ import types +from collections.abc import Mapping from typing import Final, Literal from pydantic import TypeAdapter @@ -70,16 +71,17 @@ class VertexAITextEmbeddingConfig(LiteLLMBaseModel): | None = None, title: str | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/volcengine/chat/transformation.py b/litellm/llms/volcengine/chat/transformation.py index 52b3242c186..4ccab38e761 100644 --- a/litellm/llms/volcengine/chat/transformation.py +++ b/litellm/llms/volcengine/chat/transformation.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig @@ -34,7 +35,7 @@ class VolcEngineChatConfig(OpenAILikeChatConfig): top_p: int | None = None, response_format: dict | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = dict(locals()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/volcengine/responses/transformation.py b/litellm/llms/volcengine/responses/transformation.py index 74f6182bd89..747ea21e4fb 100644 --- a/litellm/llms/volcengine/responses/transformation.py +++ b/litellm/llms/volcengine/responses/transformation.py @@ -45,6 +45,11 @@ class _EventModelClass(Protocol): def model_validate(self, obj: Mapping[str, object]) -> ResponsesAPIStreamingResponse: ... +class _FieldDefaultCarrier(Protocol): + @property + def default(self) -> object: ... + + class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): _SUPPORTED_OPTIONAL_PARAMS: list[str] = [ # Doc-listed knobs @@ -430,6 +435,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): annotation: Final[object] = field.annotation return annotation + @staticmethod + def _field_default(field: _FieldDefaultCarrier) -> object: + return field.default + @staticmethod def _fill_missing_fields(chunk: Mapping[str, object], event_model: object | None) -> Mapping[str, object]: """ @@ -451,7 +460,7 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): continue # Explicit default or factory - field_default: object = field.default + field_default: object = VolcEngineResponsesAPIConfig._field_default(field) if field_default is not pyd_fields.PydanticUndefined and field_default is not None: patched[name] = field_default continue diff --git a/litellm/llms/voyage/rerank/transformation.py b/litellm/llms/voyage/rerank/transformation.py index 7fe643870d5..21c6e6a82b9 100644 --- a/litellm/llms/voyage/rerank/transformation.py +++ b/litellm/llms/voyage/rerank/transformation.py @@ -29,6 +29,7 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) _OPTIONAL_INT: Final[TypeAdapter[int | None]] = TypeAdapter(int | None) _STR: Final = TypeAdapter(str) +_DECODED_JSON: Final = TypeAdapter(object) class VoyageRerankConfig(BaseRerankConfig): @@ -112,7 +113,7 @@ class VoyageRerankConfig(BaseRerankConfig): logging_obj.post_call(original_response=raw_response.text) try: - _json_response: Final = raw_response.json() + _json_response: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception: raise VoyageError( message=f"Failed to parse response: {raw_response.text}", diff --git a/litellm/llms/watsonx/audio_transcription/transformation.py b/litellm/llms/watsonx/audio_transcription/transformation.py index e5da7dd412f..35fda05c028 100644 --- a/litellm/llms/watsonx/audio_transcription/transformation.py +++ b/litellm/llms/watsonx/audio_transcription/transformation.py @@ -29,6 +29,7 @@ from ...openai.transcriptions.whisper_transformation import ( from ..common_utils import IBMWatsonXMixin _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTranscriptionConfig): @@ -172,7 +173,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran removed before creating the TranscriptionResponse object. """ try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise ValueError(f"Error transforming response to json: {e}\nResponse: {raw_response.text}") diff --git a/litellm/llms/watsonx/rerank/transformation.py b/litellm/llms/watsonx/rerank/transformation.py index 591ddda3cbd..87f30483196 100644 --- a/litellm/llms/watsonx/rerank/transformation.py +++ b/litellm/llms/watsonx/rerank/transformation.py @@ -30,6 +30,7 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) _OPTIONAL_INT: Final[TypeAdapter[int | None]] = TypeAdapter(int | None) _STR: Final = TypeAdapter(str) +_DECODED_JSON: Final = TypeAdapter(object) class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig): @@ -170,7 +171,7 @@ class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig): Transform IBM watsonx.ai rerank response to LiteLLM RerankResponse format """ try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Failed to parse response: {e}", diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py index 984b62eeb59..2b943fa4107 100644 --- a/litellm/llms/xai/chat/transformation.py +++ b/litellm/llms/xai/chat/transformation.py @@ -32,6 +32,7 @@ from ...openai.chat.gpt_transformation import ( ) _RESPONSE_BODY: Final = TypeAdapter(dict[object, object], config=ConfigDict(hide_input_in_errors=True)) +_FILTERED_TOOL: Final = TypeAdapter(object) def _usage_restated_from_xai_ticks(usage: Usage | None) -> Usage | None: @@ -204,11 +205,13 @@ class XAIChatConfig(OpenAIGPTConfig): if param == "max_completion_tokens": optional_params["max_tokens"] = value elif param == "tools" and value is not None: - tools = [] - for tool in value: - tool = filter_value_from_dict(tool, "strict") - if tool is not None: - tools.append(tool) + tools = [ + filtered_tool + for filtered_tool in ( + _FILTERED_TOOL.validate_python(filter_value_from_dict(tool, "strict")) for tool in value + ) + if filtered_tool is not None + ] if len(tools) > 0: optional_params["tools"] = tools elif param in supported_openai_params: diff --git a/litellm/llms/xai/cost_calculator.py b/litellm/llms/xai/cost_calculator.py index de2cbf5f9e3..aec535c2086 100644 --- a/litellm/llms/xai/cost_calculator.py +++ b/litellm/llms/xai/cost_calculator.py @@ -120,12 +120,11 @@ def _web_search_cost_per_call_from_model_info(model_info: "ModelInfo") -> float: search_costs: Final = model_info.get("search_context_cost_per_query") if not isinstance(search_costs, Mapping): return _DEFAULT_WEB_SEARCH_COST_PER_CALL - for key in ( - "search_context_size_medium", - "search_context_size_low", - "search_context_size_high", + for value in ( + search_costs.get("search_context_size_medium"), + search_costs.get("search_context_size_low"), + search_costs.get("search_context_size_high"), ): - value = search_costs.get(key) if value is None: continue try: diff --git a/litellm/main.py b/litellm/main.py index 327a7a1c4b5..c22cd8eb11c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -802,6 +802,11 @@ def _transcription_response_from_cached_dict(cached_response_dict: Mapping[str, return TranscriptionResponse(**cached_response_dict) +def _text_completion_response_from_cached_dict(cached_response_dict: Mapping[str, object]) -> TextCompletionResponse: + fields: Final = {key: value for key, value in cached_response_dict.items() if key != "stream"} + return TextCompletionResponse(stream=bool(cached_response_dict.get("stream", False)), **fields) + + async def _async_streaming(response, model, custom_llm_provider, args): try: print_verbose(f"received response in _async_streaming: {response}") @@ -3560,7 +3565,7 @@ def _complete_nadir(ctx: CompletionDispatchContext) -> _CompletionDispatchResult encoding=_get_encoding(), api_key=api_key, logging_obj=ctx.logging, - client=ctx.client, + client=_dispatch_client_http(ctx), ) ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response) @@ -4197,6 +4202,7 @@ _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints) _BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint) +_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object) _AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS: Final = ( @@ -4741,7 +4747,7 @@ def _complete_petals(ctx: CompletionDispatchContext) -> _CompletionDispatchResul ) if stream is True: ## [BETA] # Fake streaming for petals - resp_string: Final = model_response["choices"][0]["message"]["content"] + resp_string: Final = _OPAQUE_VALUE_ADAPTER.validate_python(model_response["choices"][0]["message"]["content"]) return CustomStreamWrapper( resp_string, model, @@ -7719,7 +7725,7 @@ async def atext_completion(*args, **kwargs) -> TextCompletionResponse | TextComp init_response: Final = await loop.run_in_executor(None, func_with_context) if isinstance(init_response, dict) or isinstance(init_response, TextCompletionResponse): ## CACHING SCENARIO if isinstance(init_response, dict): - response = TextCompletionResponse(**init_response) + response = _text_completion_response_from_cached_dict(init_response) else: response = init_response elif asyncio.iscoroutine(init_response): @@ -7899,7 +7905,6 @@ def text_completion( tokenizer: Final = get_encoding("p50k_base") ## if it's a 2d list - each element in the list is a text_completion() request if len(prompt) > 0 and isinstance(prompt[0], list): - responses: Final = [None for x in prompt] # init responses def process_prompt(i, individual_prompt): decoded_prompt: Final = tokenizer.decode(individual_prompt) @@ -7916,16 +7921,16 @@ def text_completion( text_completion_response["object"] = "text_completion" text_completion_response["created"] = response.get("created", None) text_completion_response["model"] = response.get("model", None) - return response["choices"][0] + return response.choices[0] with concurrent.futures.ThreadPoolExecutor() as executor: completed_futures: Final = [ executor.submit(process_prompt, i, individual_prompt) for i, individual_prompt in enumerate(prompt) ] - for i, future in enumerate(concurrent.futures.as_completed(completed_futures)): - responses[i] = future.result() - text_completion_response.choices = responses + text_completion_response.choices = [ + future.result() for future in concurrent.futures.as_completed(completed_futures) + ] return text_completion_response # else: diff --git a/litellm/models/organization_membership.py b/litellm/models/organization_membership.py index 6a0d22cb63b..f3abbc2e93d 100644 --- a/litellm/models/organization_membership.py +++ b/litellm/models/organization_membership.py @@ -6,6 +6,7 @@ Canonical definition for ``litellm_organizationmembership``. Re-exported from """ from datetime import datetime +from typing import Final from pydantic import ConfigDict, model_validator @@ -31,9 +32,10 @@ class LiteLLM_OrganizationMembershipTable(LiteLLMPydanticObjectBase): @model_validator(mode="after") def populate_user_email(self) -> "LiteLLM_OrganizationMembershipTable": - if self.user_email is None and self.user is not None: - if isinstance(self.user, dict): - self.user_email = self.user.get("user_email") + user: Final[object] = self.user + if self.user_email is None and user is not None: + if isinstance(user, dict): + self.user_email = user.get("user_email") else: - self.user_email = getattr(self.user, "user_email", None) + self.user_email = getattr(user, "user_email", None) return self diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 7f0d392ab1b..824633bdf5a 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -99,6 +99,7 @@ if TYPE_CHECKING: _EMPTY_TOOLSET_GRANTS: Final[Mapping[str, Sequence[str]]] = MappingProxyType({}) OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) +_CACHED_VALUE: Final = TypeAdapter(object) def _normalize_caller_admission_credential(value: str) -> str: @@ -4258,7 +4259,7 @@ class MCPRequestHandler: cache_key: Final = f"agent_object_permission_id:{agent_id}" try: - cached: Final[object] = await user_api_key_cache.async_get_cache(key=cache_key) + cached: Final = _CACHED_VALUE.validate_python(await user_api_key_cache.async_get_cache(key=cache_key)) if cached == MCPRequestHandler._AGENT_NO_PERMISSION_SENTINEL: return None if isinstance(cached, str) and cached: diff --git a/litellm/proxy/_experimental/mcp_server/catalog.py b/litellm/proxy/_experimental/mcp_server/catalog.py index 71eb2112bd5..2db548f5863 100644 --- a/litellm/proxy/_experimental/mcp_server/catalog.py +++ b/litellm/proxy/_experimental/mcp_server/catalog.py @@ -1173,12 +1173,9 @@ async def list_gateway_catalog( try: page: Final = await fetch_optional_catalog_page(context, request, server, allowed, cursor) + page_meta: Final[Mapping[str, object]] = page.meta or {} return page.model_copy( - update={ - "meta": { - key: value for key, value in (page.meta or {}).items() if key != SERVER_OUTCOMES_META_KEY - } - } + update={"meta": {key: value for key, value in page_meta.items() if key != SERVER_OUTCOMES_META_KEY}} ) except Exception as error: if cursor is not None: diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 7302b74f15a..0666cc7e6c0 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -165,6 +165,7 @@ _CLIENT_FORWARDED_AUTH_TYPES: Final["frozenset[str]"] = frozenset({"true_passthr _MINTED_TOKEN_CREDENTIAL_FIELDS: Final["frozenset[str]"] = frozenset({"access_token", "refresh_token", "expires_in"}) _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) def _bind_submitted_oauth_client( @@ -339,7 +340,7 @@ def _decrypt_env_vars_on_returned_row(row: object) -> None: return if isinstance(env_vars, str): try: - env_vars = json.loads(env_vars) + env_vars = _DECODED_JSON.validate_python(json.loads(env_vars)) except (json.JSONDecodeError, TypeError): return if not isinstance(env_vars, list): @@ -476,8 +477,9 @@ def _prepare_mcp_server_data( # Serialize JSON fields from ``data_dict`` (not ``data``) so the # exclude_unset filter is respected. Reading back from ``data`` would # reintroduce defaults (e.g. ``env={}``) for fields the caller never set. - if data_dict.get("static_headers") is not None: - data_dict["static_headers"] = encrypt_secret_map(data_dict["static_headers"]) + static_headers: Final[Mapping[str, str] | None] = data_dict.get("static_headers") + if static_headers is not None: + data_dict["static_headers"] = encrypt_secret_map(static_headers) # env_vars is read from ``data_dict`` (not ``data``) like every other JSON # column so the exclude_unset filter is respected: a partial update that @@ -489,11 +491,13 @@ def _prepare_mcp_server_data( _encrypt_global_env_var_values(serialized_env_vars) data_dict["env_vars"] = safe_dumps(serialized_env_vars) - if data_dict.get("mcp_info") is not None: - data_dict["mcp_info"] = safe_dumps(data_dict["mcp_info"]) + mcp_info: Final[object] = data_dict.get("mcp_info") + if mcp_info is not None: + data_dict["mcp_info"] = safe_dumps(mcp_info) - if data_dict.get("env") is not None: - data_dict["env"] = encrypt_secret_map(data_dict["env"]) + env: Final[Mapping[str, str] | None] = data_dict.get("env") + if env is not None: + data_dict["env"] = encrypt_secret_map(env) if "tool_name_to_display_name" in data_dict: data_dict["tool_name_to_display_name"] = safe_dumps(data_dict["tool_name_to_display_name"] or {}) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 3b2c2e09df5..f5c54451006 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -701,7 +701,7 @@ async def _store_per_user_token_server_side( expires_in = None refresh_token: Final[str | None] = token_response.get("refresh_token") or None - raw_scope: Final[object] = token_response.get("scope", requested_scope) + raw_scope: Final[object] = _DECODED_JSON.validate_python(token_response.get("scope", requested_scope)) scopes: Final[list[str] | None] = raw_scope.split() if isinstance(raw_scope, str) else None try: diff --git a/litellm/proxy/_experimental/mcp_server/faults/classify.py b/litellm/proxy/_experimental/mcp_server/faults/classify.py index bb41436b495..7826fc34eac 100644 --- a/litellm/proxy/_experimental/mcp_server/faults/classify.py +++ b/litellm/proxy/_experimental/mcp_server/faults/classify.py @@ -10,6 +10,7 @@ from __future__ import annotations from typing import Final import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.proxy._experimental.mcp_server.faults.types import ( @@ -25,6 +26,8 @@ from litellm.proxy._experimental.mcp_server.faults.types import ( UpstreamReportedFault, ) +_DECODED_JSON: Final = TypeAdapter(object) + def _safe_text(response: httpx.Response) -> str: try: @@ -35,7 +38,7 @@ def _safe_text(response: httpx.Response) -> str: def _safe_json(response: httpx.Response) -> object: try: - return response.json() + return _DECODED_JSON.validate_python(response.json()) except Exception: return None diff --git a/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py b/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py index 84673c8c738..a3e33929f6d 100644 --- a/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py +++ b/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py @@ -226,7 +226,7 @@ def rate_limit_retry_delay( ) -> float: if retry_after_seconds is not None: return min(retry_after_seconds, max_delay) - return min(max_delay, base_delay * 2**attempt * (1 + jitter)) + return min(max_delay, base_delay * 2.0**attempt * (1 + jitter)) T: Final = TypeVar("T") diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 7eddf592262..db70c7d77de 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -488,6 +488,10 @@ class _AuthorizationServerMetadataPayload(TypedDict, total=False): token_endpoint_auth_methods_supported: Sequence[str] +def _mapping_value(source: Mapping[str, object], key: str, default: object) -> object: + return source.get(key, default) + + def _blank_to_none(value: str | None) -> str | None: """Collapse an absent, empty, or whitespace-only string to ``None``. @@ -2757,7 +2761,9 @@ class MCPServerManager: server_id=server_id, name=name_for_prefix, protocol_version=TypeAdapter(MCPUpstreamProtocol).validate_python( - server_config.get("protocol_version", mcp_info.get("protocol_version", "auto")) + _mapping_value( + server_config, "protocol_version", _mapping_value(mcp_info, "protocol_version", "auto") + ) ), alias=alias, server_name=server_name, diff --git a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py index 6d4fefcdb6c..cc92b8cce9b 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py +++ b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py @@ -11,6 +11,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Final import httpx +from pydantic import TypeAdapter from litellm._internal_context import with_service_target from litellm._logging import verbose_logger @@ -41,6 +42,8 @@ from litellm.types.llms.custom_http import httpxSpecialProvider if TYPE_CHECKING: from litellm.types.mcp_server.mcp_server_manager import MCPServer +_DECODED_JSON: Final = TypeAdapter(object) + class MCPOAuth2TokenCache(InMemoryCache): """ @@ -160,7 +163,7 @@ class MCPOAuth2TokenCache(InMemoryCache): f"failed with status {exc.response.status_code}" ) from exc - body: Final = response.json() + body: Final = _DECODED_JSON.validate_python(response.json()) if not isinstance(body, dict): raise ValueError( diff --git a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py index c31560a9f63..b3aa040c507 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py @@ -18,6 +18,7 @@ from typing import Final, Literal, Protocol, TypeAlias import jwt from fastapi import HTTPException from jwt.types import Options +from pydantic import TypeAdapter from typing_extensions import assert_never from litellm._logging import verbose_logger @@ -38,6 +39,7 @@ _ALLOWED_ID_TOKEN_ALGORITHMS: Final = ( "PS512", ) _JWKS_CACHE_TTL_SECONDS: Final = 3600 +_DECODED_JSON: Final = TypeAdapter(object) _jwks_cache: Final = InMemoryCache(default_ttl=_JWKS_CACHE_TTL_SECONDS) JwksFetcher: TypeAlias = Callable[ @@ -103,7 +105,7 @@ async def _fetch_issuer_jwks(binding: MCPOAuthIdentityBinding) -> Sequence[Mappi client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check) response: Final = await client.get(jwks_url) response.raise_for_status() - document: Final = response.json() + document: Final = _DECODED_JSON.validate_python(response.json()) keys: Final = document.get("keys") if isinstance(document, dict) else None if not isinstance(keys, list): raise TypeError(f"JWKS document at {jwks_url} has no 'keys' array") @@ -116,7 +118,7 @@ async def _discover_jwks_url(issuer: str) -> str: client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check) response: Final = await client.get(discovery_url) response.raise_for_status() - metadata: Final = response.json() + metadata: Final = _DECODED_JSON.validate_python(response.json()) jwks_uri: Final = metadata.get("jwks_uri") if isinstance(metadata, dict) else None if not isinstance(jwks_uri, str) or not jwks_uri: raise ValueError(f"OIDC discovery at {discovery_url} returned no jwks_uri") diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchange_provider.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchange_provider.py index 2501f751d8e..d84a65fee02 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchange_provider.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchange_provider.py @@ -12,6 +12,7 @@ from __future__ import annotations from typing import Final import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.constants import ( @@ -34,6 +35,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger _GATEWAY_FAULT_OAUTH_ERRORS: Final = frozenset( {"invalid_client", "unauthorized_client", "unsupported_grant_type", "invalid_target", "invalid_scope"} ) +_DECODED_JSON: Final = TypeAdapter(object) def _oauth_error_fields(response: httpx.Response) -> tuple[str | None, str | None]: @@ -46,7 +48,7 @@ def _oauth_error_fields(response: httpx.Response) -> tuple[str | None, str | Non it can carry IdP internals and must never reach the caller. """ try: - body: Final[object] = response.json() + body: Final = _DECODED_JSON.validate_python(response.json()) except Exception: # noqa: BLE001 return None, None if not isinstance(body, dict): diff --git a/litellm/proxy/_experimental/mcp_server/result_conversion.py b/litellm/proxy/_experimental/mcp_server/result_conversion.py index 61435f94c94..fdb9a01fea6 100644 --- a/litellm/proxy/_experimental/mcp_server/result_conversion.py +++ b/litellm/proxy/_experimental/mcp_server/result_conversion.py @@ -13,7 +13,7 @@ from __future__ import annotations import json import math from collections.abc import Sequence -from typing import Final, TypeAlias, TypeVar +from typing import Final, Protocol, TypeAlias, TypeVar from mcp.types import CacheableResult, CallToolResult, ContentBlock, InputRequiredResult, TextContent, Tool from typing_extensions import ReadOnly, TypedDict, assert_never @@ -105,8 +105,17 @@ def complete_call_tool_result(outcome: ToolOutcome, compat: WireCompat) -> CallT return converted +class _StructuredContentCarrier(Protocol): + @property + def structured_content(self) -> object: ... + + +def _structured_content(result: _StructuredContentCarrier) -> object: + return result.structured_content + + def _downgrade_structured_content(result: CallToolResult) -> CallToolResult: - structured: Final = result.structured_content + structured: Final = _structured_content(result) if structured is None or isinstance(structured, dict): return result fallback: Final = TextContent(type="text", text=json.dumps(structured)) diff --git a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py index 2f88d8c361f..a7748b9ca88 100644 --- a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py +++ b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py @@ -6,7 +6,7 @@ Filters MCP tools semantically for /chat/completions and /responses endpoints. import asyncio from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final, TypeVar from litellm._logging import verbose_logger from litellm.exceptions import ContextWindowExceededError @@ -20,6 +20,8 @@ if TYPE_CHECKING: from litellm.router import Router +_Tool: Final = TypeVar("_Tool") + class SemanticToolFilterContextWindowError(Exception): """Raised when the embedding model exceeds its context window, so semantic filtering cannot run.""" @@ -265,9 +267,9 @@ class SemanticMCPToolFilter: async def filter_tools( self, query: str, - available_tools: list[Any], + available_tools: list[_Tool], top_k: int | None = None, - ) -> list[Any]: + ) -> list[_Tool]: """ Filter tools semantically based on query. @@ -390,7 +392,7 @@ class SemanticMCPToolFilter: separator: Final = client_name[-len(canonical) - 1] return separator in ("_", "-") - def _get_tools_by_names(self, tool_names: Sequence[str], available_tools: Sequence[object]) -> list[object]: + def _get_tools_by_names(self, tool_names: Sequence[str], available_tools: Sequence[_Tool]) -> list[_Tool]: """ Get tools from available_tools by their names, preserving the semantic router's ordering. @@ -406,13 +408,13 @@ class SemanticMCPToolFilter: # Exact matches win over suffix matches when both are present, and # each incoming tool is returned at most once even if two canonical # names happen to be tail-compatible with the same incoming name. - available_by_name: Final[dict[str, object]] = {} + available_by_name: Final[dict[str, _Tool]] = {} for tool in available_tools: client_name, _ = self.extract_tool_info(tool) if client_name and client_name not in available_by_name: available_by_name[client_name] = tool - matched: Final[list[object]] = [] + matched: Final[list[_Tool]] = [] used_ids: Final[set[int]] = set() for canonical in tool_names: tool = available_by_name.get(canonical) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index bb7c81f9d60..7207c4150ad 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -14,7 +14,7 @@ import time import types from collections import Counter from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterable, Mapping, Sequence, Set -from typing import TYPE_CHECKING, Final, NoReturn, Protocol +from typing import TYPE_CHECKING, Final, Literal, NoReturn, Protocol import httpx from fastapi import FastAPI, HTTPException @@ -150,6 +150,16 @@ def _scope_headers(scope: Scope) -> Sequence[tuple[bytes, bytes]]: return asgi_scope_fields(scope).get("headers", []) +class _RawHeaderPairsSource(Protocol): + def get( + self, key: Literal["headers"], default: Sequence[tuple[bytes | str, bytes | str]], / + ) -> Sequence[tuple[bytes | str, bytes | str]]: ... + + +def _raw_header_pairs(source: _RawHeaderPairsSource) -> Sequence[tuple[bytes | str, bytes | str]]: + return source.get("headers", []) + + def unsupported_protocol_version(scope: Scope) -> str | None: """Admit configured HTTP revisions while keeping SSE on the legacy protocol.""" from litellm.proxy._experimental.mcp_server.capabilities import configured_versions @@ -1319,7 +1329,7 @@ if MCP_AVAILABLE: Extract mcp-session-id from ASGI scope headers. Returns None if not present. """ - scope_headers: Final[Sequence[tuple[bytes | str, bytes | str]]] = scope.get("headers", []) + scope_headers: Final = _raw_header_pairs(scope) for header_name, header_value in scope_headers: name = header_name if isinstance(header_name, bytes) else header_name.encode() if name.lower() == b"mcp-session-id": @@ -1669,7 +1679,7 @@ if MCP_AVAILABLE: Fixes https://github.com/BerriAI/litellm/issues/20992 """ _mcp_session_header: Final = b"mcp-session-id" - _headers: Final[Sequence[tuple[bytes | str, bytes | str]]] = scope.get("headers", []) + _headers: Final = _raw_header_pairs(scope) def _normalize_header_name(header_name: object) -> bytes | None: if isinstance(header_name, bytes): @@ -2869,7 +2879,7 @@ if MCP_AVAILABLE: mcp_servers, _sse_client_ip, scoped_server_endpoint=scoped_server_endpoint, - is_initialize=scope.get("method") == "GET", + is_initialize=asgi_scope_fields(scope).get("method", "") == "GET", ): async with ( sse.connect_sse(transport_scope, receive, send) as (read_stream, write_stream), @@ -3090,7 +3100,7 @@ if MCP_AVAILABLE: ) -> Send: async def wrapped_send(message: Message) -> None: if message.get("type") == "http.response.start": - response_headers: Final[Sequence[tuple[bytes | str, bytes | str]]] = message.get("headers", []) + response_headers: Final = _raw_header_pairs(message) for key, value in response_headers: header_name = key if isinstance(key, bytes) else str(key).encode() if header_name.lower() == b"mcp-session-id": diff --git a/litellm/proxy/_experimental/mcp_server/toolset_db.py b/litellm/proxy/_experimental/mcp_server/toolset_db.py index f3a78d206d7..35906c793b4 100644 --- a/litellm/proxy/_experimental/mcp_server/toolset_db.py +++ b/litellm/proxy/_experimental/mcp_server/toolset_db.py @@ -131,6 +131,10 @@ async def get_mcp_toolset_by_name( return _toolset_from_row(row) +def _set_fields(data: UpdateMCPToolsetRequest) -> Mapping[str, object]: + return data.model_dump(exclude_unset=True) + + async def update_mcp_toolset( prisma_client: PrismaClient, data: UpdateMCPToolsetRequest, @@ -143,7 +147,7 @@ async def update_mcp_toolset( data_dict: Final = dict( ( (field, json.dumps(value) if field == "tools" else value) - for field, value in data.model_dump(exclude_unset=True).items() + for field, value in _set_fields(data).items() if field != "toolset_id" and (field not in ("toolset_name", "tools") or value is not None) ), updated_by=touched_by, diff --git a/litellm/proxy/_experimental/mcp_server/utils.py b/litellm/proxy/_experimental/mcp_server/utils.py index 1684b9c5c30..ee8f3ea3b30 100644 --- a/litellm/proxy/_experimental/mcp_server/utils.py +++ b/litellm/proxy/_experimental/mcp_server/utils.py @@ -13,6 +13,8 @@ from collections.abc import Set as AbstractSet from typing import Any, Final, Literal, Protocol, overload from urllib.parse import quote +from pydantic import TypeAdapter + from litellm._logging import verbose_logger from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -21,6 +23,7 @@ if typing.TYPE_CHECKING: MCP_SERVERS_TARGET: Final = "mcp_servers" MCP_OAUTH_TOKENS_TARGET: Final = "mcp_oauth_tokens" +_DECODED_JSON: Final = TypeAdapter(object) class ASGIScopeFields(Protocol): @@ -243,7 +246,7 @@ def _parse_mcp_info_dict(mcp_info: object) -> Mapping[str, object] | None: return mcp_info if isinstance(mcp_info, str): try: - parsed: Final[object] = json.loads(mcp_info) + parsed: Final = _DECODED_JSON.validate_python(json.loads(mcp_info)) except (ValueError, TypeError): return None return parsed if isinstance(parsed, dict) else None diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 28c701e0fc2..aa67d36af4f 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -356,7 +356,7 @@ class LazyFeatureMiddleware: async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: # Short-circuit once every feature has loaded. if scope["type"] in ("http", "websocket") and len(self._loaded) < len(self._features): - path: str = scope.get("path", "") + path: str = str(_scope_value(scope, "path")) # Strip the request's root_path so prefix matching works under a # server root path. Without this, requests like /api/v1/policies/... # never match the registered prefixes (/policies/...) and lazy @@ -584,6 +584,7 @@ def _make_warmup_router(app: "FastAPI", features: tuple[LazyFeature, ...] = LAZY from fastapi import APIRouter, Depends, HTTPException from fastapi.openapi.utils import get_openapi + from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy.auth.user_api_key_auth import user_api_key_auth router: Final = APIRouter() @@ -603,17 +604,22 @@ def _make_warmup_router(app: "FastAPI", features: tuple[LazyFeature, ...] = LAZY await _force_load(app, feat, features) feat_routes: Final = [r for r in app.routes if feat.matches(getattr(r, "path", ""))] - full: Final = get_openapi(title=app.title, version=app.version, routes=feat_routes) + full: Final[Mapping[str, object]] = MappingProxyType( + get_openapi(title=app.title, version=app.version, routes=feat_routes) + ) + paths: Final = as_str_mapping(full.get("paths", {})) or {} + components: Final = as_str_mapping(full.get("components", {})) or {} # Force all operations under one tag so they group under a single Swagger # section — many lazy modules tag routes inconsistently. - for path_ops in full.get("paths", {}).values(): - for op in path_ops.values(): - if isinstance(op, dict): - op["tags"] = [feat.name] + for path_ops in paths.values(): + for op in (as_str_mapping(path_ops) or {}).values(): + operation = as_str_mapping(op) + if isinstance(operation, dict): + operation["tags"] = [feat.name] return { "stub_path": feat.path_prefixes[0], "paths": full.get("paths", {}), - "components": {"schemas": full.get("components", {}).get("schemas", {})}, + "components": {"schemas": components.get("schemas", {})}, } return router diff --git a/litellm/proxy/_lazy_openapi_snapshot.py b/litellm/proxy/_lazy_openapi_snapshot.py index 00a05307a89..c5a63b0f990 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.py +++ b/litellm/proxy/_lazy_openapi_snapshot.py @@ -16,6 +16,7 @@ import sys from collections.abc import Callable, Mapping from dataclasses import dataclass from pathlib import Path +from types import MappingProxyType from typing import TYPE_CHECKING, Final from typing_extensions import ReadOnly, TypedDict @@ -61,7 +62,7 @@ def load_snapshot() -> dict[str, dict] | None: return None -def _normalize_operation_ids(paths: dict[str, dict]) -> None: +def _normalize_operation_ids(paths: Mapping[str, object]) -> None: """Make FastAPI-generated operation IDs stable for multi-method routes. FastAPI derives the default operation ID suffix from the first item in the @@ -69,26 +70,30 @@ def _normalize_operation_ids(paths: dict[str, dict]) -> None: set iteration order can vary between processes, which makes the snapshot drift even when no routes changed. """ + from litellm.litellm_core_utils.core_helpers import as_str_mapping + for path_ops in paths.values(): - if not isinstance(path_ops, dict): + operations = as_str_mapping(path_ops) + if not isinstance(operations, dict): continue - methods = {method for method in path_ops if method in HTTP_METHOD_SUFFIXES} + methods = {method for method in operations if method in HTTP_METHOD_SUFFIXES} if not methods: continue - for method, operation in path_ops.items(): - if method not in HTTP_METHOD_SUFFIXES or not isinstance(operation, dict): + for method, operation in operations.items(): + entry = as_str_mapping(operation) + if method not in HTTP_METHOD_SUFFIXES or not isinstance(entry, dict): continue - operation_id = operation.get("operationId") + operation_id = entry.get("operationId") if not isinstance(operation_id, str): continue for suffix in methods: suffix_token = f"_{suffix}" if operation_id.endswith(suffix_token): - operation["operationId"] = operation_id[: -len(suffix_token)] + f"_{method}" + entry["operationId"] = operation_id[: -len(suffix_token)] + f"_{method}" break @@ -117,6 +122,7 @@ def _register_feature(app: "FastAPI", feat: "LazyFeature") -> str | None: def _feature_fragment(app: "FastAPI", feat: "LazyFeature", used_operation_ids: set[str]) -> SnapshotFragment | None: from fastapi.openapi.utils import get_openapi + from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy.proxy_server import ensure_unique_openapi_operation_ids feat_routes: Final = [r for r in app.routes if feat.matches(getattr(r, "path", ""))] @@ -124,22 +130,29 @@ def _feature_fragment(app: "FastAPI", feat: "LazyFeature", used_operation_ids: s return None _stabilize_multi_method_route_ids(feat_routes) full: Final = get_openapi(title=app.title, version=app.version, routes=feat_routes) - paths: Final = full.get("paths", {}) + openapi: Final[Mapping[str, object]] = MappingProxyType(full) + paths: Final = as_str_mapping(openapi.get("paths", {})) or {} _normalize_operation_ids(paths) for path_ops in paths.values(): - for method, op in path_ops.items(): - if isinstance(op, dict): - operation_id = op.get("operationId") + for method, op in (as_str_mapping(path_ops) or {}).items(): + operation = as_str_mapping(op) + if isinstance(operation, dict): + operation_id = operation.get("operationId") if isinstance(operation_id, str): for suffix in HTTP_METHOD_SUFFIXES: if operation_id.endswith(f"_{suffix}"): - op["operationId"] = operation_id[: -len(suffix)] + method + operation["operationId"] = operation_id[: -len(suffix)] + method break - op["tags"] = [feat.name] - unique: Final = ensure_unique_openapi_operation_ids(full, used_operation_ids) + operation["tags"] = [feat.name] + unique: Final[Mapping[str, object]] = MappingProxyType( + ensure_unique_openapi_operation_ids(full, used_operation_ids) + ) + components: Final = as_str_mapping(unique.get("components", {})) or {} return { - "paths": paths, - "components": {"schemas": unique.get("components", {}).get("schemas", {})}, + "paths": { + path: operations for path, entry in paths.items() if (operations := as_str_mapping(entry)) is not None + }, + "components": {"schemas": as_str_mapping(components.get("schemas", {})) or {}}, } diff --git a/litellm/proxy/a2a/discovery.py b/litellm/proxy/a2a/discovery.py index ff58c9c85ec..03d1be7d185 100644 --- a/litellm/proxy/a2a/discovery.py +++ b/litellm/proxy/a2a/discovery.py @@ -16,10 +16,13 @@ fetcher dispatches by ``discovery_mode``: from collections.abc import Mapping from enum import Enum -from typing import Any, Final +from typing import Final from urllib.parse import urlencode +from pydantic import TypeAdapter + from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.types.llms.custom_http import httpxSpecialProvider @@ -46,6 +49,8 @@ AGENT_CARD_WELL_KNOWN_PATHS: Final[tuple[str, ...]] = ( DEFAULT_DISCOVERY_TIMEOUT_SECONDS: Final = 10.0 +_DECODED_JSON: Final = TypeAdapter(object) + class AgentCardDiscoveryError(Exception): """Raised when none of the well-known paths returned a usable agent card.""" @@ -87,7 +92,7 @@ async def fetch_well_known_card( params: Mapping[str, object] | None = None, timeout: float = DEFAULT_DISCOVERY_TIMEOUT_SECONDS, headers: dict[str, str] | None = None, -) -> dict[str, Any]: +) -> dict[str, object]: """ Fetch an agent card from ``base_url`` using the strategy chosen by ``discovery_mode``. Returns the parsed JSON from the first path that @@ -137,13 +142,14 @@ async def fetch_well_known_card( continue try: - card = response.json() + decoded = _DECODED_JSON.validate_python(response.json()) except Exception as exc: last_error = f"{url}: invalid JSON ({exc})" continue + card = as_str_mapping(decoded) if not isinstance(card, dict): - last_error = f"{url}: expected JSON object, got {type(card).__name__}" + last_error = f"{url}: expected JSON object, got {type(decoded).__name__}" continue verbose_proxy_logger.debug("A2A discovery succeeded at %s", url) diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index c11d296837e..4691c7b0bc5 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -23,6 +23,7 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.url_utils import SSRFError, validate_url from litellm.llms.a2a.common_utils import resolve_a2a_hop_auth_header from litellm.proxy._types import UserAPIKeyAuth @@ -281,7 +282,7 @@ async def _a2a_sse_event_source( request_id: str | int | None = None, extra_headers: Mapping[str, str] | None = None, served_version: A2AVersion = "0.3", -) -> AsyncGenerator[Mapping[str, object], None]: +) -> AsyncGenerator[object, None]: """Stream an upstream A2A SSE response as parsed JSON-RPC event dicts. Upstream HTTP/JSON-RPC errors are surfaced as a single JSON-RPC error event @@ -327,12 +328,14 @@ async def _a2a_sse_event_source( if not payload: continue try: - event = json.loads(payload) + event = _DECODED_JSON.validate_python(json.loads(payload)) except Exception: continue - if isinstance(event, dict): - event = normalize_stream_event(event, served_version, request_id=request_id) - yield event + event_fields = as_str_mapping(event) + if isinstance(event_fields, dict): + yield normalize_stream_event(event_fields, served_version, request_id=request_id) + else: + yield event finally: await resp.aclose() @@ -741,7 +744,7 @@ async def invoke_agent_a2a( # Get backend URL and agent name agent_card_params: Final = agent.agent_card_params or {} agent_url: Final = agent_card_params.get("url") - agent_name: Final = agent_card_params.get("name", agent_id) + agent_name: Final = (as_str_mapping(agent_card_params) or {}).get("name", agent_id) # Get litellm_params (may include custom_llm_provider for completion bridge) litellm_params: dict[str, object] = agent.litellm_params or {} diff --git a/litellm/proxy/agent_endpoints/a2a_routing.py b/litellm/proxy/agent_endpoints/a2a_routing.py index 9a6f674db2c..fb727367caf 100644 --- a/litellm/proxy/agent_endpoints/a2a_routing.py +++ b/litellm/proxy/agent_endpoints/a2a_routing.py @@ -18,6 +18,7 @@ from litellm.a2a_protocol.litellm_completion_bridge.handler import ( bridge_model_name, ) from litellm.constants import A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.agent_endpoints.databricks_oauth import ( resolve_databricks_backend_auth, @@ -173,13 +174,14 @@ async def route_a2a_agent_request( return _call_route(route_type, _bridge_agent_request(data, agent, bridge_auth.litellm_params)) # Get API base URL from agent config - if not agent.agent_card_params or "url" not in agent.agent_card_params: + card_params: Final = as_str_mapping(agent.agent_card_params) + if not card_params or "url" not in card_params: verbose_proxy_logger.error("[A2A] Agent '%s' has no URL configured", agent_name) route_name = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type) raise ProxyModelNotFoundError(route=route_name, model_name=model_name, retryable_with_model_read_through=False) # Inject API base and route to litellm - data["api_base"] = agent.agent_card_params["url"] + data["api_base"] = card_params["url"] verbose_proxy_logger.debug("[A2A] Routing %s to %s", model_name, data["api_base"]) return _call_route(route_type, data) diff --git a/litellm/proxy/agent_endpoints/databricks_oauth.py b/litellm/proxy/agent_endpoints/databricks_oauth.py index 7425c240d10..bebba009d82 100644 --- a/litellm/proxy/agent_endpoints/databricks_oauth.py +++ b/litellm/proxy/agent_endpoints/databricks_oauth.py @@ -48,6 +48,7 @@ _DEFAULT_SCOPE: Final = "all-apis" _TOKEN_EXPIRY_BUFFER_SECONDS: Final = 60 _DEFAULT_TTL_SECONDS: Final = 3600 _OAUTH_BLOCK: Final = TypeAdapter(Mapping[str, object]) +_TOKEN_RESPONSE_BODY: Final = TypeAdapter(object) def _resolve_secret(value: object) -> str | None: @@ -275,7 +276,7 @@ class DatabricksAppOAuthTokenCache(InMemoryCache): except httpx.HTTPError as exc: raise ValueError(f"Databricks App OAuth token request failed: {exc}") from exc - body: Final[object] = response.json() + body: Final = _TOKEN_RESPONSE_BODY.validate_python(response.json()) if not isinstance(body, dict): raise ValueError( f"Databricks App OAuth token response returned non-object JSON (got {type(body).__name__})" diff --git a/litellm/proxy/agent_endpoints/managed_identity.py b/litellm/proxy/agent_endpoints/managed_identity.py index abab21901ee..fab7ebbc61d 100644 --- a/litellm/proxy/agent_endpoints/managed_identity.py +++ b/litellm/proxy/agent_endpoints/managed_identity.py @@ -119,6 +119,10 @@ def managed_write_fields( return AgentIdentityFailure(message=f"Invalid agent identity configuration: {exc}") +def _identity_fields(identity: EntraIdentityConfig) -> Mapping[str, object]: + return identity.model_dump() + + def _identity_write(identity: EntraIdentityConfig | None, existing: AgentResponse | None) -> ManagedWriteFields: if identity is None: unbind: Final[ManagedWriteFields] = { @@ -134,7 +138,7 @@ def _identity_write(identity: EntraIdentityConfig | None, existing: AgentRespons existing and existing.identity and existing.identity.active - and all(getattr(existing.identity, name) == value for name, value in identity.model_dump().items()) + and all(getattr(existing.identity, name) == value for name, value in _identity_fields(identity).items()) ): unchanged: Final[ManagedWriteFields] = {} return unchanged diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index 8e348423837..b0088d93112 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -21,6 +21,7 @@ import json import re from collections.abc import Mapping, Sequence from datetime import datetime, timezone +from types import MappingProxyType from typing import Annotated, Final, Protocol, TypedDict, get_args from urllib.parse import urlsplit @@ -299,7 +300,7 @@ def _validate_installation_preference(spec: PluginSpec) -> None: def _build_plugin_manifest(name: str, spec: PluginSpec) -> Mapping[str, object]: """Build the stored manifest dict shared by plugin create and update.""" - dumped: Final[Mapping[str, object]] = spec.model_dump(exclude_none=True) + dumped: Final[Mapping[str, object]] = MappingProxyType(spec.model_dump(exclude_none=True)) return {"name": name, **{key: value for key, value in dumped.items() if value and key != "name"}} diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 8887d86570a..2c6768b8423 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -393,7 +393,7 @@ def _member_allowed_models(membership: _TeamMembershipModelScope) -> Sequence[st class _RawCacheRead(Protocol): - async def async_get_cache(self, *, key: str) -> object: ... + async def async_get_cache(self, *, key: str, local_only: bool = ...) -> object: ... def _raw_cache(cache: _RawCacheRead) -> _RawCacheRead: @@ -2712,7 +2712,7 @@ async def get_team_membership( use_writer=True, ) - l1_cached: Final[object] = await user_api_key_cache.async_get_cache(key=_key, local_only=True) + l1_cached: Final = await _raw_cache(user_api_key_cache).async_get_cache(key=_key, local_only=True) l1_membership: Final = _membership_from_cached_payload(l1_cached) if not isinstance(l1_membership, _TeamMembershipCacheMiss): return l1_membership diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 7a186d84aa5..25f900a5f9f 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -924,8 +924,9 @@ def get_request_route(request: Request) -> str: scope: Final = request.scope if not isinstance(scope, dict): return str(request.url.path) - raw_path: Final[str] = str(scope.get("path", request.url.path)) - root_path: Final[str] = str(scope.get("app_root_path", scope.get("root_path", ""))).rstrip("/") + scope_fields: Final[Mapping[str, object]] = MappingProxyType(scope) + raw_path: Final[str] = str(scope_fields.get("path", request.url.path)) + root_path: Final[str] = str(scope_fields.get("app_root_path", scope_fields.get("root_path", ""))).rstrip("/") if not isinstance(raw_path, str): return str(request.url.path) # Strip root_path only when it matches whole path segments — guarding @@ -1520,8 +1521,15 @@ def get_model_rate_limit_from_metadata( metadata_accessor_key: Literal["team_metadata", "organization_metadata", "project_metadata"], rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit", "model_itpm_limit", "model_otpm_limit"], ) -> dict[str, int] | None: - if getattr(user_api_key_dict, metadata_accessor_key): - return getattr(user_api_key_dict, metadata_accessor_key).get(rate_limit_key) + metadata: Final = ( + user_api_key_dict.team_metadata + if metadata_accessor_key == "team_metadata" + else user_api_key_dict.organization_metadata + if metadata_accessor_key == "organization_metadata" + else user_api_key_dict.project_metadata + ) + if metadata: + return metadata.get(rate_limit_key) return None diff --git a/litellm/proxy/auth/ip_address_utils.py b/litellm/proxy/auth/ip_address_utils.py index f1ca2c70f3b..0b349e6a733 100644 --- a/litellm/proxy/auth/ip_address_utils.py +++ b/litellm/proxy/auth/ip_address_utils.py @@ -283,7 +283,7 @@ class IPAddressUtils: @staticmethod def get_mcp_client_ip( request: Request, - general_settings: dict[str, Any] | None = None, + general_settings: dict[str, object] | None = None, ) -> str | None: """ Extract client IP from a FastAPI request for MCP access control. @@ -371,4 +371,4 @@ class IPAddressUtils: return client_ip case _HopCountUnset(): pass - return get_request_ip_address(request, use_x_forwarded_for=use_xff) + return get_request_ip_address(request, use_x_forwarded_for=use_xff is True) diff --git a/litellm/proxy/auth/rds_iam_token.py b/litellm/proxy/auth/rds_iam_token.py index a9f349d1750..e291c78294c 100644 --- a/litellm/proxy/auth/rds_iam_token.py +++ b/litellm/proxy/auth/rds_iam_token.py @@ -3,7 +3,7 @@ import re from collections.abc import Mapping from dataclasses import dataclass from types import MappingProxyType -from typing import Any, Final +from typing import Final, Protocol import httpx @@ -223,11 +223,15 @@ def rds_region_from_hostname(db_host: str) -> str | None: return match.group("region").lower() +class _RdsAuthTokenClient(Protocol): + def generate_db_auth_token(self, *, DBHostname: str, Port: str, DBUsername: str, Region: str | None) -> str: ... + + def generate_iam_auth_token( db_host: str, db_port: str, db_user: str, - client: Any | None = None, + client: _RdsAuthTokenClient | None = None, *, region: str | None = None, ) -> str: diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 720bfabb95a..3ae671d40ac 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -160,7 +160,7 @@ class RouteChecks: if allowed_route in LiteLLMRoutes._member_names_: if RouteChecks.check_route_access( route=route, - allowed_routes=LiteLLMRoutes._member_map_[allowed_route].value, + allowed_routes=LiteLLMRoutes[allowed_route].value, ): if ( allowed_route in _AUTH_ENFORCED_PASS_THROUGH_ROUTE_GROUPS diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 27fa4731baf..b1e8bdbe787 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -13,6 +13,7 @@ import re import secrets from collections.abc import Mapping, Sequence from datetime import datetime, timezone +from types import MappingProxyType from typing import Final, NamedTuple, Protocol, Union, cast import fastapi @@ -177,6 +178,7 @@ from litellm.proxy.utils import ( normalize_route_for_root_path, ) from litellm.repositories.table_repositories import JWTKeyMappingRepository, TeamMembershipRepository +from litellm.repositories.user_repository import UserRepository from litellm.repositories.verification_token_repository import VerificationTokenRepository from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model from litellm.router_utils.fallback_event_handlers import ( @@ -1319,7 +1321,7 @@ async def fetch_global_spend_with_event_coordination( async def _load_global_spend() -> float | None: proxy_budget_row: Final = await bounded_db_lookup( - prisma_client.db.litellm_usertable.find_unique(where={"user_id": LITELLM_PROXY_BUDGET_NAME}), + UserRepository(prisma_client).table.find_unique(where={"user_id": LITELLM_PROXY_BUDGET_NAME}), name="proxy_budget", ) return float(proxy_budget_row.spend) if proxy_budget_row is not None else None @@ -1635,7 +1637,7 @@ async def _auto_register_jwt_mapping( if minted: try: async with db_span("delete_orphaned_jwt_key", "LiteLLM_VerificationToken"): - await prisma_client.db.litellm_verificationtoken.delete(where={"token": token_hash}) + await VerificationTokenRepository(prisma_client).table.delete(where={"token": token_hash}) except Exception as delete_err: # Don't fail the request if cleanup fails — the orphan is # unmapped and inert. Log so an operator can prune it later. @@ -2693,7 +2695,7 @@ async def user_api_key_auth_builder( and valid_token.last_refreshed_at is not None and team_obj.last_refreshed_at > valid_token.last_refreshed_at ): - team_obj_dict: Final = team_obj.__dict__ + team_obj_dict: Final[Mapping[str, object]] = MappingProxyType(team_obj.__dict__) for k, v in team_obj_dict.items(): field_name = f"team_{k}" diff --git a/litellm/proxy/caching_routes.py b/litellm/proxy/caching_routes.py index 597080be3b3..775d00ed53e 100644 --- a/litellm/proxy/caching_routes.py +++ b/litellm/proxy/caching_routes.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final from fastapi import APIRouter, Depends, HTTPException, Request @@ -19,6 +20,10 @@ router: Final = APIRouter( ) +def _copy_cache_params(cache_params: Mapping[str, object]) -> Mapping[str, object]: + return dict(**cache_params) + + def _extract_cache_params() -> dict[str, object]: """ Safely extracts and cleans cache parameters. @@ -41,7 +46,7 @@ def _extract_cache_params() -> dict[str, object]: try: cache_params: Final = vars(litellm.cache.cache) cleaned_params: Final = ( - HealthCheckCacheParams.model_validate(dict(**cache_params)).model_dump() if cache_params else {} + HealthCheckCacheParams.model_validate(_copy_cache_params(cache_params)).model_dump() if cache_params else {} ) return masker.mask_dict(cleaned_params) except (AttributeError, TypeError) as e: diff --git a/litellm/proxy/client/chat.py b/litellm/proxy/client/chat.py index c027bf2fe98..ccdef44f9e8 100644 --- a/litellm/proxy/client/chat.py +++ b/litellm/proxy/client/chat.py @@ -59,7 +59,7 @@ class ChatClient: frequency_penalty: float | None = None, user: str | None = None, return_request: bool = False, - ) -> dict[str, Any] | requests.Request: + ) -> dict[str, object] | requests.Request: """ Create a chat completion. @@ -184,7 +184,8 @@ class ChatClient: response.raise_for_status() # Parse SSE stream - for line in response.iter_lines(): + lines: Final[Iterator[object]] = response.iter_lines() + for line in lines: if line: line = _SSE_LINE.validate_python(line).decode("utf-8") if line.startswith("data: "): diff --git a/litellm/proxy/client/cli/commands/encryption.py b/litellm/proxy/client/cli/commands/encryption.py index ef73b4f0d2d..61c8db617af 100644 --- a/litellm/proxy/client/cli/commands/encryption.py +++ b/litellm/proxy/client/cli/commands/encryption.py @@ -4,9 +4,14 @@ from typing import Final import click import rich +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.core_helpers import as_str_mapping from ...http_client import HTTPClient -from .auth import cli_context_obj +from ._cli_context import cli_context_values + +_DECODED_JSON: Final = TypeAdapter(object) @click.group() @@ -40,23 +45,32 @@ def migrate(ctx: click.Context, check_only: bool, dry_run: bool): lite encryption migrate --check # attestation scan, no writes lite encryption migrate # perform the migration """ - client: Final = HTTPClient(cli_context_obj(ctx)["base_url"], ctx.obj["api_key"]) + context: Final = cli_context_values(ctx) + client: Final = HTTPClient(context["base_url"], context["api_key"]) - if check_only: - response = client.request("GET", "/credentials/migrate-encryption/check") - else: - response = client.request( + response: Final = _DECODED_JSON.validate_python( + client.request("GET", "/credentials/migrate-encryption/check") + if check_only + else client.request( "POST", "/credentials/migrate-encryption", json={}, params={"dry_run": "true"} if dry_run else None, ) + ) rich.print_json(data=response) - report: Final = response.get("report", {}) if isinstance(response, dict) else {} + report_value: Final = (as_str_mapping(response) or {}).get("report", {}) + report: Final = as_str_mapping(report_value) + if report is None: + raise click.ClickException(f"Unexpected report in the proxy response: {report_value!r}") residual: Final = report.get("residual_legacy") - if residual is not None and residual > 0: + if residual is None: + return + if not isinstance(residual, int | float): + raise click.ClickException(f"Unexpected residual_legacy in the proxy response: {residual!r}") + if residual > 0: rich.print(f"[yellow]Residual legacy values remaining: {residual}[/yellow]") elif residual == 0: rich.print("[green]No legacy values remaining (residual_legacy == 0).[/green]") diff --git a/litellm/proxy/client/cli/commands/statusline_script.py b/litellm/proxy/client/cli/commands/statusline_script.py index 5aeefa0c739..fef3e313ee5 100644 --- a/litellm/proxy/client/cli/commands/statusline_script.py +++ b/litellm/proxy/client/cli/commands/statusline_script.py @@ -89,6 +89,28 @@ class Fetch(Protocol): def __call__(self, credentials: Credentials, session_id: str) -> Fetched: ... +class _ResponseBody(Protocol): + def read(self) -> bytes: ... + + +class _OpenedResponse(Protocol): + def __enter__(self) -> _ResponseBody: ... + + def __exit__(self, *exc_info: object) -> None: ... + + +class _UrlOpener(Protocol): + def __call__(self, url: urllib.request.Request, /, *, timeout: float) -> _OpenedResponse: ... + + +def _url_opener() -> _UrlOpener: + return urllib.request.urlopen + + +def _json_decoder() -> Callable[[bytes | str], object]: + return json.loads + + def as_mapping(value: object) -> Mapping[str, object]: return value if isinstance(value, dict) else EMPTY @@ -106,7 +128,7 @@ def printable(value: object) -> str: def load_json(raw: bytes | str) -> object: try: - return json.loads(raw) + return _json_decoder()(raw) except ValueError: return None @@ -195,7 +217,7 @@ def fetch_session(credentials: Credentials, session_id: str) -> Fetched: }, ) try: - with urllib.request.urlopen(request, timeout=FETCH_TIMEOUT_SECONDS) as response: + with _url_opener()(request, timeout=FETCH_TIMEOUT_SECONDS) as response: raw: Final[bytes] = response.read() except urllib.error.HTTPError as error: return Fetched(session=None, definitive=400 <= error.code < 500) diff --git a/litellm/proxy/client/cli/interface.py b/litellm/proxy/client/cli/interface.py index f7c545ef48e..7c01732119e 100644 --- a/litellm/proxy/client/cli/interface.py +++ b/litellm/proxy/client/cli/interface.py @@ -189,7 +189,7 @@ def execute_command(path: ShellScope, ctx: click.Context, cli: click.Group) -> N try: if path[0].startswith("-"): cli.main( - ["--base-url", ctx.obj["base_url"], *path], + ["--base-url", cli_context_obj(ctx)["base_url"], *path], prog_name="lite", obj={"secret_vault": context_secret_vault(ctx)}, standalone_mode=False, diff --git a/litellm/proxy/client/cli/main.py b/litellm/proxy/client/cli/main.py index 3f1e7f4259c..a1964d8a1bb 100644 --- a/litellm/proxy/client/cli/main.py +++ b/litellm/proxy/client/cli/main.py @@ -4,6 +4,7 @@ from collections.abc import Callable, Iterator, Mapping, MutableMapping from typing import Final, Literal, Protocol import click +from pydantic import TypeAdapter from litellm._version import version as litellm_version from litellm.litellm_core_utils.cli_token_utils import load_cli_token @@ -89,10 +90,13 @@ class LazyCommands(MutableMapping[str, click.Command]): return len(self._loaders) +_MODULE_ATTRIBUTE: Final = TypeAdapter(object) + + def _command_in(module: str, attribute: str) -> Callable[[], click.Command]: @functools.cache def load() -> click.Command: - match getattr(importlib.import_module(module, __package__), attribute): + match _MODULE_ATTRIBUTE.validate_python(getattr(importlib.import_module(module, __package__), attribute)): case click.Command() as command: return command case _: diff --git a/litellm/proxy/client/credentials.py b/litellm/proxy/client/credentials.py index ebcd79457a6..649fe529936 100644 --- a/litellm/proxy/client/credentials.py +++ b/litellm/proxy/client/credentials.py @@ -75,7 +75,7 @@ class CredentialsManagementClient: credential_values: Mapping[str, object], return_request: bool = False, display_name: str | None = None, - ) -> dict[str, Any] | requests.Request: + ) -> dict[str, object] | requests.Request: """ Create a new credential. @@ -121,7 +121,7 @@ class CredentialsManagementClient: self, credential_name: str, return_request: bool = False, - ) -> dict[str, Any] | requests.Request: + ) -> dict[str, object] | requests.Request: """ Delete a credential by name. @@ -182,7 +182,7 @@ class CredentialsManagementClient: self, credential_name: str, return_request: bool = False, - ) -> dict[str, Any] | requests.Request: + ) -> dict[str, object] | requests.Request: """ Get a credential by name. diff --git a/litellm/proxy/client/http_client.py b/litellm/proxy/client/http_client.py index aa0b986b1ad..96a170c75cb 100644 --- a/litellm/proxy/client/http_client.py +++ b/litellm/proxy/client/http_client.py @@ -4,6 +4,11 @@ from collections.abc import Mapping from typing import Any, Final import requests +from typing_extensions import TypedDict, Unpack + + +class _RequestOptions(TypedDict, total=False, extra_items=object): + pass class HTTPClient: @@ -29,7 +34,7 @@ class HTTPClient: data: Mapping[str, object] | list | bytes | None = None, json: Mapping[str, object] | list | None = None, headers: dict[str, str] | None = None, - **kwargs: Any, + **kwargs: Unpack[_RequestOptions], ) -> Any: """Make an HTTP request to the LiteLLM proxy server. diff --git a/litellm/proxy/client/model_groups.py b/litellm/proxy/client/model_groups.py index fef307600c4..1555d170212 100644 --- a/litellm/proxy/client/model_groups.py +++ b/litellm/proxy/client/model_groups.py @@ -1,4 +1,4 @@ -from typing import Any, Final +from typing import Final import requests @@ -31,7 +31,7 @@ class ModelGroupsManagementClient: headers["Authorization"] = f"Bearer {self._api_key}" return headers - def info(self, return_request: bool = False) -> list[dict[str, Any]] | requests.Request: + def info(self, return_request: bool = False) -> list[dict[str, object]] | requests.Request: """ Get detailed information about all model groups from the server. diff --git a/litellm/proxy/client/models.py b/litellm/proxy/client/models.py index 1b50f8c24f3..7342974c0cc 100644 --- a/litellm/proxy/client/models.py +++ b/litellm/proxy/client/models.py @@ -158,7 +158,7 @@ class ModelsManagementClient: model_id: str | None = None, model_name: str | None = None, return_request: bool = False, - ) -> dict[str, Any] | requests.Request: + ) -> dict[str, object] | requests.Request: """ Get information about a specific model by its ID or name. diff --git a/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py b/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py index 40cdb117476..b2640d2a765 100644 --- a/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py +++ b/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py @@ -1,11 +1,15 @@ import asyncio import json -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import asdict, dataclass +from types import MappingProxyType from typing import TYPE_CHECKING, Final +from pydantic import TypeAdapter + from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy.common_utils.config_sync_pubsub import ( # noqa: F401 # legacy module exports ConfigSyncPubSub, _ConfigSyncPubSub, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -30,6 +34,7 @@ _pending_publishes: Final[set[asyncio.Task[None]]] = set() # mutable-ok: strong _in_flight_publishes: Final = asyncio.Semaphore(_MAX_IN_FLIGHT_PUBLISHES) _BACKOFF_INITIAL_SECONDS: Final = 5.0 _BACKOFF_MAX_SECONDS: Final = 60.0 +_DECODED_JSON: Final = TypeAdapter(object) def auth_cache_invalidation_channel(redis_cache: "RedisCache") -> str: @@ -46,7 +51,9 @@ class _CacheInvalidationMessage: def _cache_invalidation_message_json(cache_key: str, new_value: float | None = None, ttl: float | None = None) -> str: - message: Final = asdict(_CacheInvalidationMessage(cache_key=cache_key, new_value=new_value, ttl=ttl)) + message: Final[Mapping[str, object]] = MappingProxyType( + asdict(_CacheInvalidationMessage(cache_key=cache_key, new_value=new_value, ttl=ttl)) + ) return json.dumps({field: value for field, value in message.items() if value is not None}) @@ -62,10 +69,10 @@ def _message_from_data(data: object) -> _CacheInvalidationMessage | None: if not isinstance(data, str): return None try: - parsed: Final = json.loads(data) + parsed: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(data))) except json.JSONDecodeError: return None - if not isinstance(parsed, dict): + if parsed is None: return None cache_key: Final = parsed.get("cache_key") if not isinstance(cache_key, str): diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 46434a997b7..f889748c504 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from itertools import accumulate from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict, assert_never import litellm @@ -39,6 +40,7 @@ from litellm.types.utils import ( ) _CALLBACK_VAR_MASKER: Final = SensitiveDataMasker() +_LOADED_CALLBACK: Final = TypeAdapter(object) # Compound names that are credential-bearing but don't contain any of the # default sensitive segments (so SensitiveDataMasker won't flag them). _EXTRA_SENSITIVE_CALLBACK_KEYS: Final = {"gcs_path_service_account"} @@ -389,9 +391,11 @@ def initialize_callbacks_on_proxy( imported_list.append( _loaded_callback_or_raise( entry=callback, - loaded=get_instance_fn( - value=callback, - config_file_path=config_file_path, + loaded=_LOADED_CALLBACK.validate_python( + get_instance_fn( + value=callback, + config_file_path=config_file_path, + ) ), ) ) @@ -408,9 +412,11 @@ def initialize_callbacks_on_proxy( litellm.callbacks = [ _loaded_callback_or_raise( entry=value, - loaded=get_instance_fn( - value=value, - config_file_path=config_file_path, + loaded=_LOADED_CALLBACK.validate_python( + get_instance_fn( + value=value, + config_file_path=config_file_path, + ) ), ) ] diff --git a/litellm/proxy/common_utils/debug_utils.py b/litellm/proxy/common_utils/debug_utils.py index 33633150711..ec4baa76514 100644 --- a/litellm/proxy/common_utils/debug_utils.py +++ b/litellm/proxy/common_utils/debug_utils.py @@ -11,12 +11,14 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Annotated, Final, NamedTuple, Protocol, TypedDict from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import TypeAdapter from typing_extensions import ReadOnly from litellm import get_secret_str from litellm._logging import verbose_proxy_logger from litellm.constants import PYTHON_GC_THRESHOLD from litellm.litellm_core_utils.bug_report import EnvironmentReport +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.bug_report_config import build_proxy_environment_report @@ -26,6 +28,7 @@ if TYPE_CHECKING: from litellm.router import Router router: Final = APIRouter() +_DECODED_JSON: Final = TypeAdapter(object) # Configure garbage collection thresholds from environment variables @@ -885,8 +888,8 @@ def init_verbose_loggers(): return if os.path.isfile(worker_config): return - _settings: Final = json.loads(worker_config) - if not isinstance(_settings, dict): + _settings: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(worker_config))) + if _settings is None: return debug: Final = _settings.get("debug", None) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 245a56678eb..e34d03a04c3 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -164,7 +164,7 @@ def coerce_numeric_form_fields( def _parse_binary_body(body: bytes) -> dict: """JSON sent under a binary content type still parses; real binary (protobuf) carries no params -> {}.""" try: - parsed: Final = orjson.loads(body) + parsed: Final = _DECODED_JSON.validate_python(orjson.loads(body)) if isinstance(parsed, dict): return parsed except orjson.JSONDecodeError: diff --git a/litellm/proxy/common_utils/load_config_utils.py b/litellm/proxy/common_utils/load_config_utils.py index 4a082eb307b..640a947f9c5 100644 --- a/litellm/proxy/common_utils/load_config_utils.py +++ b/litellm/proxy/common_utils/load_config_utils.py @@ -15,6 +15,7 @@ if TYPE_CHECKING: from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase _BUCKET_CONFIG_ADAPTER: Final = TypeAdapter(dict[str, object]) +_DECODED_YAML: Final = TypeAdapter(object) class BucketObjectFetcher(Protocol): @@ -31,7 +32,7 @@ class SyncBucketObjectReader(Protocol): def _parsed_config(object_key: str, file_contents: str) -> object | None: try: - parsed: Final = yaml.safe_load(file_contents) + parsed: Final = _DECODED_YAML.validate_python(yaml.safe_load(file_contents)) except yaml.YAMLError as e: verbose_proxy_logger.error("Config object %s is not valid YAML: %s", object_key, e) return None diff --git a/litellm/proxy/common_utils/proxy_rate_limit_error.py b/litellm/proxy/common_utils/proxy_rate_limit_error.py index 089d0b7a226..02da6f706d1 100644 --- a/litellm/proxy/common_utils/proxy_rate_limit_error.py +++ b/litellm/proxy/common_utils/proxy_rate_limit_error.py @@ -37,7 +37,7 @@ This module provides a single proxy-side error class that: import json from collections.abc import Mapping -from typing import Any, Final +from typing import Final from fastapi import HTTPException from pydantic import TypeAdapter, ValidationError @@ -171,7 +171,7 @@ class ProxyRateLimitError(HTTPException, RateLimitError): def __init__( self, - detail: Any, + detail: object, headers: Mapping[str, object] | None = None, category: str | RateLimitErrorCategory = RateLimitErrorCategory.LITELLM_RATE_LIMIT, rate_limit_type: str | RateLimitType | None = None, diff --git a/litellm/proxy/container_endpoints/handler_factory.py b/litellm/proxy/container_endpoints/handler_factory.py index 35bd69ef0bd..a2ffdd4d59d 100644 --- a/litellm/proxy/container_endpoints/handler_factory.py +++ b/litellm/proxy/container_endpoints/handler_factory.py @@ -13,6 +13,7 @@ from typing import Final from fastapi import APIRouter, Depends, Request, Response from fastapi.responses import ORJSONResponse +from pydantic import TypeAdapter from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -26,6 +27,8 @@ from litellm.proxy.container_endpoints.ownership import ( get_container_forwarding_params, ) +_UPLOADED_FILE_FIELD: Final = TypeAdapter(object) + def _load_endpoints_config() -> dict: """Load the endpoints configuration from JSON file.""" @@ -304,7 +307,7 @@ async def _process_multipart_upload_request( raise HTTPException(status_code=400, detail="Missing required 'file' field") # convert_upload_files_to_file_data returns list of tuples, extract single file - file_list: Final = data["file"] + file_list: Final = _UPLOADED_FILE_FIELD.validate_python(data["file"]) if isinstance(file_list, list) and len(file_list) > 0: data["file"] = file_list[0] diff --git a/litellm/proxy/container_endpoints/ownership.py b/litellm/proxy/container_endpoints/ownership.py index aeb0bb1e3d0..9e4ab3c3c4e 100644 --- a/litellm/proxy/container_endpoints/ownership.py +++ b/litellm/proxy/container_endpoints/ownership.py @@ -1,13 +1,14 @@ import json from collections.abc import Awaitable, Callable, Mapping, Sequence from collections.abc import Set as AbstractSet -from typing import TYPE_CHECKING, Any, Final, TypeAlias +from typing import TYPE_CHECKING, Final, Protocol, TypeAlias from fastapi import HTTPException from pydantic import BaseModel from litellm._logging import verbose_proxy_logger from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.core_helpers import has_attributes from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.resource_ownership import ( get_primary_resource_owner_scope, @@ -111,12 +112,20 @@ def _get_response_id(response: object) -> str | None: return value if isinstance(value, str) else None -def _dump_response(response: Any) -> dict[str, object]: +class _SupportsModelDump(Protocol): + def model_dump(self) -> Mapping[str, object]: ... + + +class _SupportsPydanticDict(Protocol): + def dict(self) -> Mapping[str, object]: ... + + +def _dump_response(response: object) -> Mapping[str, object]: if isinstance(response, dict): return dict(response) - if hasattr(response, "model_dump"): + if has_attributes(response, _SupportsModelDump, "model_dump"): return response.model_dump() - if hasattr(response, "dict"): + if has_attributes(response, _SupportsPydanticDict, "dict"): return response.dict() return {"id": _get_response_id(response)} @@ -199,9 +208,11 @@ async def record_container_owner( original_container_id, resolved_provider = decode_container_id_for_ownership(container_id, custom_llm_provider) model_object_id: Final = _container_model_object_id(original_container_id, resolved_provider) - file_object: Final = _dump_response(response) - file_object["custom_llm_provider"] = resolved_provider - file_object["provider_container_id"] = original_container_id + file_object: Final = { + **_dump_response(response), + "custom_llm_provider": resolved_provider, + "provider_container_id": original_container_id, + } # Prisma Python requires Json fields to be serialized as a JSON string. file_object_json: Final[str] = json.dumps(file_object) diff --git a/litellm/proxy/db/autorouter_session_rollup.py b/litellm/proxy/db/autorouter_session_rollup.py index 75882bc3fb1..4ddcf7e600b 100644 --- a/litellm/proxy/db/autorouter_session_rollup.py +++ b/litellm/proxy/db/autorouter_session_rollup.py @@ -526,7 +526,7 @@ async def _upsert_turn_with_retry( except DB_RETRY_SAFE_ERROR_TYPES: if attempt >= n_retry_times: raise - await asyncio.sleep(2**attempt + random.uniform(0, 1)) + await asyncio.sleep(2.0**attempt + random.uniform(0, 1)) else: return diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index be5b307b5ae..b290e0f85b4 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -16,6 +16,7 @@ from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable, Mappi from contextlib import asynccontextmanager from contextvars import ContextVar from datetime import datetime, timedelta, timezone +from functools import reduce from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, overload from urllib.parse import quote, unquote @@ -104,6 +105,7 @@ else: RESPONSES_SESSION_CALL_TYPES: Final = frozenset({CallTypes.responses.value, CallTypes.aresponses.value}) _SPEND_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_CALL_KWARG_VALUE: Final = TypeAdapter(object) _JsonPayload: TypeAlias = Mapping[str, object] | Sequence[object] | str | int | float | bool | None _JSON_VALUE_ADAPTER: Final[TypeAdapter[_JsonPayload]] = TypeAdapter( _JsonPayload, config=ConfigDict(strict=True, hide_input_in_errors=True) @@ -313,6 +315,20 @@ def _spend_tables_left_to_send( return cast(DBSpendUpdateTransactions, remaining) if any(remaining.values()) else None +class _SpendRestore(TypedDict, total=False): + db_spend_update_transactions: ReadOnly[DBSpendUpdateTransactions] + daily_spend_update_transactions: ReadOnly[Mapping[str, BaseDailySpendTransaction]] + daily_team_spend_update_transactions: ReadOnly[Mapping[str, BaseDailySpendTransaction]] + daily_org_spend_update_transactions: ReadOnly[Mapping[str, BaseDailySpendTransaction]] + daily_end_user_spend_update_transactions: ReadOnly[Mapping[str, BaseDailySpendTransaction]] + daily_agent_spend_update_transactions: ReadOnly[Mapping[str, BaseDailySpendTransaction]] + window_spend_update_transactions: ReadOnly[Sequence[WindowSpendTransaction]] + + +def _merged_spend_restore(first: _SpendRestore, second: _SpendRestore) -> _SpendRestore: + return {**first, **second} + + def _report_spend_update_step_failure(step_name: str, *, payload: Mapping[str, object], exc: Exception) -> None: spend_log_error( "Spend tracking - %s failed, this request is missing from that spend rollup. request_id=%s api_key=%s", @@ -845,7 +861,7 @@ class DBSpendUpdateWriter: spend=payload["spend"], total_tokens=payload["total_tokens"], completion_response=completion_response, - realtime_tool_calls=(kwargs or {}).get("realtime_tool_calls"), + realtime_tool_calls=_CALL_KWARG_VALUE.validate_python((kwargs or {}).get("realtime_tool_calls")), ) if transaction is None: return @@ -1627,7 +1643,7 @@ class DBSpendUpdateWriter: ): verbose_proxy_logger.debug("acquired lock for spend updates") - uncommitted: dict[str, Any] = {} # mutable-ok: tracks popped categories still needing commit + uncommitted: dict[str, _SpendRestore] = {} # mutable-ok: tracks popped categories still needing commit committed_spend_tables: Final[list[_SpendTableName]] = [] # mutable-ok: filled as each table lands try: @@ -1642,13 +1658,41 @@ class DBSpendUpdateWriter: ) = await self.redis_update_buffer.get_all_transactions_from_redis_buffer_pipeline() uncommitted = { - "db_spend_update_transactions": db_spend_update_transactions, - "daily_spend_update_transactions": daily_spend_update_transactions, - "daily_team_spend_update_transactions": daily_team_spend_update_transactions, - "daily_org_spend_update_transactions": daily_org_spend_update_transactions, - "daily_end_user_spend_update_transactions": daily_end_user_spend_update_transactions, - "daily_agent_spend_update_transactions": daily_agent_spend_update_transactions, - "window_spend_update_transactions": window_spend_update_transactions, + "db_spend_update_transactions": ( + {} + if db_spend_update_transactions is None + else {"db_spend_update_transactions": db_spend_update_transactions} + ), + "daily_spend_update_transactions": ( + {} + if daily_spend_update_transactions is None + else {"daily_spend_update_transactions": daily_spend_update_transactions} + ), + "daily_team_spend_update_transactions": ( + {} + if daily_team_spend_update_transactions is None + else {"daily_team_spend_update_transactions": daily_team_spend_update_transactions} + ), + "daily_org_spend_update_transactions": ( + {} + if daily_org_spend_update_transactions is None + else {"daily_org_spend_update_transactions": daily_org_spend_update_transactions} + ), + "daily_end_user_spend_update_transactions": ( + {} + if daily_end_user_spend_update_transactions is None + else {"daily_end_user_spend_update_transactions": daily_end_user_spend_update_transactions} + ), + "daily_agent_spend_update_transactions": ( + {} + if daily_agent_spend_update_transactions is None + else {"daily_agent_spend_update_transactions": daily_agent_spend_update_transactions} + ), + "window_spend_update_transactions": ( + {} + if window_spend_update_transactions is None + else {"window_spend_update_transactions": window_spend_update_transactions} + ), } if db_spend_update_transactions is not None: @@ -1677,9 +1721,12 @@ class DBSpendUpdateWriter: on_table_committed=committed_spend_tables.append, ) except Exception as e: - uncommitted["db_spend_update_transactions"] = _spend_tables_left_to_send( + left_to_send: Final = _spend_tables_left_to_send( db_spend_update_transactions, committed_spend_tables, e ) + uncommitted["db_spend_update_transactions"] = ( + {} if left_to_send is None else {"db_spend_update_transactions": left_to_send} + ) raise uncommitted.pop("db_spend_update_transactions", None) @@ -1753,7 +1800,7 @@ class DBSpendUpdateWriter: exc=e, ) finally: - to_restore = {name: txns for name, txns in uncommitted.items() if txns is not None} + to_restore: Final = reduce(_merged_spend_restore, uncommitted.values(), _SpendRestore()) if to_restore: await self.redis_update_buffer.restore_transactions_to_redis(**to_restore) await self.pod_lock_manager.release_lock( diff --git a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py index 627b30e83f7..1116cd604a3 100644 --- a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py +++ b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py @@ -1,7 +1,7 @@ import asyncio import json import logging -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, TypeAlias from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger @@ -14,7 +14,9 @@ from litellm.types.services import ServiceTypes POD_LOCK_TARGET: Final = "pod_lock" if TYPE_CHECKING: - ProxyLogging = Any + from litellm.proxy.utils import ProxyLogging as _ProxyLogging + + ProxyLogging: TypeAlias = _ProxyLogging else: ProxyLogging = Any diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index 54d20bd4507..66833fd688a 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -11,6 +11,7 @@ from datetime import datetime from functools import reduce from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast +from pydantic import TypeAdapter from redis.exceptions import RedisError from litellm._internal_context import with_service_target @@ -98,6 +99,7 @@ _SPEND_TRANSACTION_FIELDS: Final[tuple[_SpendTransactionField, ...]] = ( ) _ValueT = TypeVar("_ValueT") +_DECODED_JSON: Final = TypeAdapter(object) def _spend_log_json_default(value: object) -> str: @@ -109,7 +111,7 @@ def _encode_spend_log_row(row: SpendLogRow) -> str: def _decode_spend_log_row(encoded: str) -> dict[str, object] | None: - decoded: Final = json.loads(encoded) + decoded: Final = _DECODED_JSON.validate_python(json.loads(encoded)) return decoded if isinstance(decoded, dict) else None diff --git a/litellm/proxy/db/prisma_query_span.py b/litellm/proxy/db/prisma_query_span.py index 75625c6c0f2..2747a765f2a 100644 --- a/litellm/proxy/db/prisma_query_span.py +++ b/litellm/proxy/db/prisma_query_span.py @@ -20,6 +20,8 @@ from dataclasses import dataclass from types import MappingProxyType from typing import Final +from pydantic import TypeAdapter + from litellm.integrations.otel.model.spans import PG_CATALOG, PRISMA_RELATIONS @@ -34,6 +36,7 @@ class PrismaQuery: UNKNOWN_PRISMA_QUERY: Final = PrismaQuery("prisma_query", None, None) +_DECODED_JSON: Final = TypeAdapter(object) _ROOT_FIELD: Final = re.compile(r"result:\s*(\w+)") _RAW_SQL: Final = re.compile(r'query:\s*"((?:[^"\\]|\\.)*)"') @@ -111,7 +114,7 @@ def sql_operation(sql: str) -> tuple[str | None, str | None]: def _query_text(content: str) -> str: try: - payload: Final[object] = json.loads(content) + payload: Final = _DECODED_JSON.validate_python(json.loads(content)) except ValueError: return content query: Final = payload.get("query") if isinstance(payload, dict) else None diff --git a/litellm/proxy/db/proxy_worker_heartbeat.py b/litellm/proxy/db/proxy_worker_heartbeat.py index 673b549b645..3214c0974b4 100644 --- a/litellm/proxy/db/proxy_worker_heartbeat.py +++ b/litellm/proxy/db/proxy_worker_heartbeat.py @@ -56,6 +56,7 @@ class _LiveWorkerCountRow(TypedDict): _COUNT_ROWS_ADAPTER: Final = TypeAdapter(tuple[_LiveWorkerCountRow, ...]) +_RAW_QUERY_RESULT: Final = TypeAdapter(object) class _HeartbeatExecute(Protocol): @@ -99,7 +100,9 @@ async def count_live_proxy_workers(prisma_client: PrismaClient) -> int | None: db: Final = prisma_client.db primary_db: Final = db.writer if isinstance(db, RoutingPrismaWrapper) else db async with db_span("count_live_proxy_workers", "LiteLLM_ProxyWorkerHeartbeat"): - rows: Final = await primary_db.query_raw(COUNT_SQL, PROXY_WORKER_LIVENESS_WINDOW_SECONDS) + rows: Final = _RAW_QUERY_RESULT.validate_python( + await primary_db.query_raw(COUNT_SQL, PROXY_WORKER_LIVENESS_WINDOW_SECONDS) + ) return _COUNT_ROWS_ADAPTER.validate_python(rows)[0]["live_workers"] except Exception as count_err: # noqa: BLE001 # an unknown count must degrade to "warn", never to a 503 verbose_proxy_logger.debug("Live proxy worker count unavailable: %s", count_err) diff --git a/litellm/proxy/db/routing_prisma_wrapper.py b/litellm/proxy/db/routing_prisma_wrapper.py index 48ad57fd06e..52a0be3d49e 100644 --- a/litellm/proxy/db/routing_prisma_wrapper.py +++ b/litellm/proxy/db/routing_prisma_wrapper.py @@ -298,7 +298,7 @@ class RoutingPrismaWrapper: def __getattr__(self, name: str) -> Any: if name in _TOP_LEVEL_READ_METHODS: - return getattr(self.read_target, name) + return _DELEGATED_ATTRIBUTE.validate_python(getattr(self.read_target, name)) writer_attr: Final = _DELEGATED_ATTRIBUTE.validate_python(getattr(self._writer, name)) # Per-model action accessors are non-callable instances that expose # both `find_many` and `create`. Methods like execute_raw / batch_ / diff --git a/litellm/proxy/db/spend_log_tool_index.py b/litellm/proxy/db/spend_log_tool_index.py index 7ac414bec00..591ad81a4e5 100644 --- a/litellm/proxy/db/spend_log_tool_index.py +++ b/litellm/proxy/db/spend_log_tool_index.py @@ -19,7 +19,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone from itertools import chain, groupby -from typing import TYPE_CHECKING, Any, Final, Literal, Protocol +from typing import TYPE_CHECKING, Final, Literal, Protocol from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import LiteralString @@ -122,7 +122,7 @@ def build_tool_usage_transaction( spend: float, total_tokens: int, completion_response: object, - realtime_tool_calls: Any = None, + realtime_tool_calls: object = None, ) -> ToolUsageTransaction | None: """None when the request invoked no tools. Realtime sessions carry invoked tools in kwargs["realtime_tool_calls"] (OpenAI tool_calls shape) rather than diff --git a/litellm/proxy/discovery_endpoints/agent_skills_archive.py b/litellm/proxy/discovery_endpoints/agent_skills_archive.py index 1f2fca3992e..94a0c3e2cf1 100644 --- a/litellm/proxy/discovery_endpoints/agent_skills_archive.py +++ b/litellm/proxy/discovery_endpoints/agent_skills_archive.py @@ -16,6 +16,7 @@ from types import MappingProxyType from typing import Final import yaml +from pydantic import TypeAdapter MAX_ARCHIVE_UNPACKED_BYTES: Final = 50 * 1024 * 1024 MAX_ARCHIVE_ENTRIES: Final = 1000 @@ -26,6 +27,7 @@ _ZIP_ENTRY_PERMISSIONS: Final = 0o644 << 16 _FRONTMATTER_PATTERN: Final = re.compile(r"^---\s*\n(.*?)\n---\s*(?:\n|$)", re.DOTALL) _WINDOWS_DRIVE_PATTERN: Final = re.compile(r"^[A-Za-z]:") _EMPTY_FRONTMATTER: Final[Mapping[str, object]] = MappingProxyType({}) +_DECODED_YAML: Final = TypeAdapter(object) @dataclass(frozen=True, slots=True) @@ -100,7 +102,7 @@ def _manifest_frontmatter(manifest: bytes) -> Mapping[str, object]: if match is None: return _EMPTY_FRONTMATTER try: - parsed: Final = yaml.safe_load(match.group(1)) + parsed: Final = _DECODED_YAML.validate_python(yaml.safe_load(match.group(1))) except yaml.YAMLError: return _EMPTY_FRONTMATTER if not isinstance(parsed, dict): diff --git a/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py b/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py index 2050cc40e6d..830f7b0334e 100644 --- a/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py +++ b/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py @@ -10,7 +10,6 @@ import asyncio import re from collections.abc import Sequence from itertools import groupby -from operator import itemgetter from types import MappingProxyType from typing import Final @@ -173,11 +172,15 @@ def _slugify(raw: str | None) -> str | None: return _NON_SLUG_PATTERN.sub("-", raw.lower()).strip("-")[:MAX_SKILL_NAME_LENGTH].rstrip("-") or None +def _entry_name(entry: tuple[int, str]) -> str: + return entry[1] + + def _deduplicated(names: Sequence[str]) -> tuple[str, ...]: ordinals: Final = MappingProxyType( { position: ordinal - for _, duplicates in groupby(sorted(enumerate(names), key=itemgetter(1)), key=itemgetter(1)) + for _, duplicates in groupby(sorted(enumerate(names), key=_entry_name), key=_entry_name) for ordinal, (position, _) in enumerate(duplicates) } ) diff --git a/litellm/proxy/example_config_yaml/custom_auth.py b/litellm/proxy/example_config_yaml/custom_auth.py index b7646ce5e3d..c53074638c6 100644 --- a/litellm/proxy/example_config_yaml/custom_auth.py +++ b/litellm/proxy/example_config_yaml/custom_auth.py @@ -30,7 +30,7 @@ async def generate_key_fn(data: GenerateKeyRequest): data_json: Final = data.json() # Unpacking variables - team_id: Final = data_json.get("team_id") + team_id: Final = data.team_id data_json.get("duration") data_json.get("models") data_json.get("aliases") diff --git a/litellm/proxy/example_config_yaml/custom_callbacks.py b/litellm/proxy/example_config_yaml/custom_callbacks.py index 2c689811dc8..5ae55060930 100644 --- a/litellm/proxy/example_config_yaml/custom_callbacks.py +++ b/litellm/proxy/example_config_yaml/custom_callbacks.py @@ -7,10 +7,14 @@ import traceback sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system path import inspect +from typing import Final + +from pydantic import TypeAdapter import litellm from litellm.integrations.custom_logger import CustomLogger -from typing import Final + +_ATTRIBUTE_VALUE: Final = TypeAdapter(object) # This file includes the custom callbacks for LiteLLM Proxy @@ -27,7 +31,11 @@ class MyCustomHandler(CustomLogger): print_verbose(f"{blue_color_code}Initialized LiteLLM custom logger") try: print_verbose("Logger Initialized with following methods:") - methods: Final = [method for method in dir(self) if inspect.ismethod(getattr(self, method))] + methods: Final = [ + method + for method in dir(self) + if inspect.ismethod(_ATTRIBUTE_VALUE.validate_python(getattr(self, method))) + ] # Pretty print_verbose the methods for method in methods: diff --git a/litellm/proxy/guardrails/_content_utils.py b/litellm/proxy/guardrails/_content_utils.py index 17bea031794..8581dca603b 100644 --- a/litellm/proxy/guardrails/_content_utils.py +++ b/litellm/proxy/guardrails/_content_utils.py @@ -9,8 +9,9 @@ every text fragment. """ from collections.abc import Callable, Iterator, Mapping, Sequence -from typing import Any, Final +from typing import Final +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.responses.litellm_completion_transformation.compaction_items import ( compaction_summary, input_items, @@ -231,7 +232,7 @@ def is_string_batch_input(data: Mapping[str, object]) -> bool: return isinstance(input_value, list) and bool(input_value) and all(isinstance(item, str) for item in input_value) -def apply_redacted_messages_back(data: dict[str, Any], redacted_messages: Sequence[object]) -> bool: +def apply_redacted_messages_back(data: dict[str, object], redacted_messages: Sequence[object]) -> bool: """Write redacted messages back to whichever field(s) the caller used. Mask/anonymize paths take a synthesised messages list (from @@ -249,9 +250,9 @@ def apply_redacted_messages_back(data: dict[str, Any], redacted_messages: Sequen carry exactly one message per inspected element: a partial rewrite would forward the remaining originals unredacted. Callers must block on False. """ + input_value: Final = data.get("input") if is_string_batch_input(data): - batch: Final = data["input"] - inspected_indices: Final = tuple(idx for idx, item in enumerate(batch) if item) + inspected_indices: Final = tuple(idx for idx, item in enumerate(iter_items(input_value)) if item) if len(redacted_messages) != len(inspected_indices): return False if any(not isinstance(message, Mapping) or message.get("content") is None for message in redacted_messages): @@ -259,12 +260,12 @@ def apply_redacted_messages_back(data: dict[str, Any], redacted_messages: Sequen redacted_texts: Final = tuple( "\n".join(_iter_text_parts_in_content(message["content"])) for message in redacted_messages ) - for idx, text in zip(inspected_indices, redacted_texts): - batch[idx] = text + if isinstance(input_value, list): + for idx, text in zip(inspected_indices, redacted_texts): + input_value[idx] = text return True if "messages" in data: data["messages"] = redacted_messages - input_value: Final = data.get("input") if isinstance(input_value, str): text_parts: Final[list[str]] = [] for msg in redacted_messages: diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 70e7aac12b5..a5822bfb0e3 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -90,6 +90,7 @@ if TYPE_CHECKING: router: Final = APIRouter() GUARDRAIL_REGISTRY: Final = GuardrailRegistry() _DECODED_JSON: Final = TypeAdapter(object) +_DECODED_YAML: Final = TypeAdapter(object) def _as_str_object_mapping(mapping: Mapping[str, object]) -> Mapping[str, object]: @@ -1682,7 +1683,7 @@ async def validate_blocked_words_file(request: dict[str, str]): if not file_content: return {"valid": False, "error": "No file content provided"} - data: Final = yaml.safe_load(file_content) + data: Final = _DECODED_YAML.validate_python(yaml.safe_load(file_content)) if not isinstance(data, dict) or "blocked_words" not in data: return { diff --git a/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py b/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py index ac0a6911184..aabac874ca2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py +++ b/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py @@ -63,6 +63,7 @@ _INVALID_ASSERTION_AADSTS_PREFIX: Final = "50027" _AADSTS_CODES_ADAPTER: Final = TypeAdapter(tuple[int, ...]) _MCP_CALL_TYPES: Final[tuple[str, ...]] = ("mcp_call", "call_mcp_tool") _TOOL_INPUT_SCHEMA_ADAPTER: Final = TypeAdapter(dict[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) _OBO_CACHE_MAX_ENTRIES: Final = 1000 _DEFAULT_TOKEN_TTL_SECONDS: Final = 3599.0 _TOKEN_EXPIRY_SLACK_SECONDS: Final = 60.0 @@ -343,7 +344,7 @@ class Agent365Guardrail(CustomGuardrail): latency_ms: float, ) -> dict: # mutable-ok: returns the request data dict per hook contract try: - parsed_verdict: Final = response.json() + parsed_verdict: Final = _DECODED_JSON.validate_python(response.json()) except ValueError: return self._handle_unavailable( data=data, @@ -486,7 +487,7 @@ class Agent365Guardrail(CustomGuardrail): response=response, ) try: - parsed_body: Final = response.json() + parsed_body: Final = _DECODED_JSON.validate_python(response.json()) except ValueError as exc: raise Agent365MalformedResponseError("the Entra token endpoint returned a non-JSON body") from exc if not isinstance(parsed_body, dict): diff --git a/litellm/proxy/guardrails/guardrail_hooks/airia/airia.py b/litellm/proxy/guardrails/guardrail_hooks/airia/airia.py index 993c1d30949..389706d7417 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/airia/airia.py +++ b/litellm/proxy/guardrails/guardrail_hooks/airia/airia.py @@ -2,12 +2,13 @@ import os import uuid from collections.abc import Mapping from typing import ( - Any, # noqa: TID251 # the only type CustomGuardrail.__init__ accepts for its open-ended kwargs Final, Literal, ) import httpx +from pydantic import TypeAdapter +from typing_extensions import TypedDict, Unpack from litellm._logging import verbose_proxy_logger from litellm.exceptions import GuardrailRaisedException @@ -36,6 +37,12 @@ DEFAULT_BLOCKED_MESSAGE: Final = "Blocked by your organization's content policy. SUPPORTED_EVENT_HOOKS: Final = (GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call) +_DECODED_JSON: Final = TypeAdapter(object) + + +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + pass + def _stream_cannot_carry_a_rewrite(request_data: Mapping[str, object] | None) -> bool: """Whether this is a streamed response on a route the framework can only block, not rewrite. @@ -97,7 +104,7 @@ class AiriaGuardrail(CustomGuardrail): timeout: float | None = None, supported_event_hooks: list[GuardrailEventHooks] | None = None, # mutable-ok: matches CustomGuardrail.__init__ async_handler: AsyncHTTPHandler | None = None, # dependency-injected in tests; defaults to a real client - **kwargs: Any, # kwargs-ok: passed straight through to CustomGuardrail.__init__ + **kwargs: Unpack[_CustomGuardrailOptions], # kwargs-ok: passed straight through to CustomGuardrail.__init__ ) -> None: resolved_timeout: Final = timeout or float(os.getenv("AIRIA_TIMEOUT", "10")) @@ -161,7 +168,7 @@ class AiriaGuardrail(CustomGuardrail): headers={"Authorization": f"Bearer {self.api_key}"}, ) response.raise_for_status() - raw_body: Final[object] = response.json() + raw_body: Final = _DECODED_JSON.validate_python(response.json()) body: Final = _as_object(raw_body) if body is None: raise TypeError(f"expected a JSON object in the response body, got {type(raw_body).__name__}") diff --git a/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/__init__.py index 64770fdf0f7..5bf471265c8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/__init__.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING, Final, Literal, cast +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.types.guardrails import GuardrailEventHooks, SupportedGuardrailIntegrations from .block_code_execution import BlockCodeExecutionGuardrail @@ -26,7 +27,7 @@ def _get_param( value: Final = getattr(litellm_params, key, default) if value is not None: return value - raw: Final = guardrail.get("litellm_params") + raw: Final = as_str_mapping(guardrail.get("litellm_params")) if isinstance(raw, dict) and key in raw: return raw[key] return default diff --git a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py index 936f862b10b..19b9bccd8aa 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py @@ -23,6 +23,7 @@ from litellm import DualCache from litellm._logging import verbose_proxy_logger from litellm._version import version as litellm_version from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, get_ssl_configuration, @@ -222,7 +223,7 @@ class CatoNetworksGuardrail(CustomGuardrail): return [] @staticmethod - def _iter_schema_string_refs(data: Mapping[str, Any]): + def _iter_schema_string_refs(data: Mapping[str, object]): """Yield ``(container, key)`` for every non-empty schema string the proxy forwards to the model inside tool/function and structured-output schemas: each ``tools[].function`` and legacy ``functions[]`` entry plus the @@ -235,10 +236,10 @@ class CatoNetworksGuardrail(CustomGuardrail): list_keys: Final = ("enum", "examples") stack: Final[list] = [] - for tool in data.get("tools") or []: + for tool in iter_items(data.get("tools") or ()): if isinstance(tool, dict) and isinstance(tool.get("function"), dict): stack.append(tool["function"]) - for function in data.get("functions") or []: + for function in iter_items(data.get("functions") or ()): if isinstance(function, dict): stack.append(function) response_format: Final = data.get("response_format") diff --git a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py index ab43d32e260..45e785fd53c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py @@ -35,6 +35,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -1743,7 +1744,7 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): @staticmethod def _extract_masked_entity_count( - rules: list[dict[str, Any]], + rules: Sequence[Mapping[str, object]], ) -> dict[str, int] | None: """Count entity-type detections per Cisco rule for the logging payload.""" if not rules: @@ -1752,8 +1753,7 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): for rule in rules: if not isinstance(rule, dict): continue - entity_types = rule.get("entity_types") or [] - for entity_type in entity_types: + for entity_type in iter_items(rule.get("entity_types") or ()): if not isinstance(entity_type, str): continue counts[entity_type] = counts.get(entity_type, 0) + 1 diff --git a/litellm/proxy/guardrails/guardrail_hooks/compresr/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/compresr/__init__.py index 498f9bf4099..c5ca97ff760 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/compresr/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/compresr/__init__.py @@ -2,6 +2,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, Final +from litellm.litellm_core_utils.core_helpers import attribute_or_none from litellm.types.guardrails import ( GuardrailEventHooks, Mode, @@ -26,7 +27,7 @@ def _coerce_event_hook( def _get_optional_value(litellm_params: LitellmParams, optional_params: object | None, attribute_name: str) -> object: if optional_params is not None: - value: Final = getattr(optional_params, attribute_name, None) + value: Final = attribute_or_none(optional_params, attribute_name) if value is not None: return value return getattr(litellm_params, attribute_name, None) diff --git a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py index 029feba89fb..4d910cde603 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py @@ -28,6 +28,7 @@ from urllib.parse import urlparse import httpx from fastapi import HTTPException from httpx import Response as HttpxResponse +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -72,6 +73,7 @@ if TYPE_CHECKING: ) BYPASS_HEADER: Final = "x-compresr-bypass" +_AGENTIC_LOOP_MODEL: Final = TypeAdapter(object) COMPRESR_RETRIEVE_TOOL_NAME: Final = "compresr_retrieve" DEFAULT_API_BASE: Final = "https://api.compresr.ai" DEFAULT_COMPRESSION_MODEL: Final = "latte_v2" @@ -1118,7 +1120,7 @@ class CompresrGuardrail(CustomGuardrail): full_model_name = model if logging_obj is not None: agentic_params: Final = getattr(logging_obj, "model_call_details", {}).get("agentic_loop_params", {}) - candidate: Final = agentic_params.get("model", model) + candidate: Final = _AGENTIC_LOOP_MODEL.validate_python(agentic_params.get("model", model)) if isinstance(candidate, str) and candidate: full_model_name = candidate diff --git a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py index 34725803c60..ee3a957f607 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py @@ -5,7 +5,7 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Annotated, Final, Literal, NamedTuple, Optional, cast from fastapi import HTTPException -from pydantic import ConfigDict, Field, ValidationError +from pydantic import ConfigDict, Field, TypeAdapter, ValidationError from typing_extensions import override from litellm._logging import verbose_proxy_logger @@ -37,6 +37,8 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +_DECODED_JSON: Final = TypeAdapter(object) + class CrowdStrikeAIDRGuardrailMissingSecrets(Exception): """Custom exception for missing CrowdStrike AIDR secrets.""" @@ -362,7 +364,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): assert response is not None response.raise_for_status() - response_body: Final[object] = response.json() + response_body: Final = _DECODED_JSON.validate_python(response.json()) raw_result: Final[object] = response_body.get("result") if isinstance(response_body, dict) else None blocked_signal: Final[object] = raw_result.get("blocked") if isinstance(raw_result, dict) else None diff --git a/litellm/proxy/guardrails/guardrail_hooks/custom_code/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/custom_code/__init__.py index 13d7178dfe9..0f9ce15e65b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/custom_code/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/custom_code/__init__.py @@ -39,7 +39,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" raise ValueError("Custom code guardrail requires a guardrail_name") # Get the custom code from litellm_params - custom_code: Final = getattr(litellm_params, "custom_code", None) + custom_code: Final = litellm_params.custom_code if not custom_code: raise ValueError("Custom code guardrail requires 'custom_code' in litellm_params") diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py index 199da7df30d..ecd23d86540 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py @@ -21,6 +21,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, get_async_httpx_client, @@ -126,6 +127,13 @@ def _sanitize_inbound_headers( return sanitized or None +def _logged_inbound_headers(details: Mapping[str, object]) -> Mapping[object, object] | None: + litellm_params: Final = as_str_mapping(details.get("litellm_params", {})) + metadata: Final = as_str_mapping(litellm_params.get("metadata", {})) if litellm_params is not None else None + headers: Final = metadata.get("headers", None) if metadata is not None else None + return headers if isinstance(headers, dict) else None + + def _extract_inbound_headers( request_data: dict, logging_obj: Optional["LiteLLMLoggingObj"], @@ -163,7 +171,7 @@ def _extract_inbound_headers( if logging_obj and getattr(logging_obj, "model_call_details", None): try: details: Final = logging_obj.model_call_details or {} - headers = details.get("litellm_params", {}).get("metadata", {}).get("headers", None) + headers = _logged_inbound_headers(details) if headers: return _sanitize_inbound_headers(headers, extra_allowlist=extra_allowlist) except Exception: @@ -283,7 +291,7 @@ class GenericGuardrailAPI(CustomGuardrail): ) self.num_retries: int = num_retries or 0 self.headers = headers or {} - self.extra_headers = extra_headers or [] + self.extra_headers: Sequence[object] = extra_headers or [] # If api_key is provided, add it as x-api-key header if api_key: diff --git a/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py b/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py index f32228a6204..7ff5aa2466b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py +++ b/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py @@ -34,6 +34,7 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj GRAYSWAN_BLOCK_ERROR_MSG: Final = "Blocked by Gray Swan Guardrail" +_DECODED_JSON: Final = TypeAdapter(object) class _GraySwanMonitorResponse(TypedDict): @@ -705,7 +706,9 @@ class GraySwanGuardrail(CustomGuardrail): {**existing_headers, **inbound_headers} if isinstance(existing_headers, dict) else inbound_headers ) if cleaned_litellm_metadata: - sanitized: Final[object] = safe_json_loads(safe_dumps(cleaned_litellm_metadata), default={}) + sanitized: Final = _DECODED_JSON.validate_python( + safe_json_loads(safe_dumps(cleaned_litellm_metadata), default={}) + ) if isinstance(sanitized, dict) and sanitized: payload["litellm_metadata"] = sanitized diff --git a/litellm/proxy/guardrails/guardrail_hooks/javelin/javelin.py b/litellm/proxy/guardrails/guardrail_hooks/javelin/javelin.py index d5edcc19a02..a1d0900ebed 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/javelin/javelin.py +++ b/litellm/proxy/guardrails/guardrail_hooks/javelin/javelin.py @@ -2,6 +2,7 @@ from datetime import datetime from typing import TYPE_CHECKING, Final from fastapi import HTTPException +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -23,6 +24,8 @@ from litellm.types.utils import CallTypesLiteral, GuardrailStatus if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +_DECODED_JSON: Final = TypeAdapter(object) + class JavelinGuardrail(CustomGuardrail): @classmethod @@ -113,7 +116,9 @@ class JavelinGuardrail(CustomGuardrail): json=dict(request), timeout=self.timeout, ) - verbose_proxy_logger.debug("Javelin Guardrail: Javelin guard API response: %s", response.json()) + verbose_proxy_logger.debug( + "Javelin Guardrail: Javelin guard API response: %s", _DECODED_JSON.validate_python(response.json()) + ) response_data: Final = response.json() # Ensure the response has the required assessments field if "assessments" not in response_data: diff --git a/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py b/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py index 8efd1dc79e3..7c0cdb0d36f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py @@ -7,6 +7,7 @@ from types import MappingProxyType from typing import Final, Literal from fastapi import HTTPException +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -40,6 +41,9 @@ from litellm.types.proxy.guardrails.guardrail_hooks.lakera_ai_v2 import ( ) from litellm.types.utils import CallTypesLiteral, GuardrailStatus, ModelResponse +_DECODED_JSON: Final = TypeAdapter(object) +_RESPONSE_FIELDS: Final = TypeAdapter(Mapping[str, object]) + _DETECTOR_CATEGORY_PHRASES: Final[Mapping[str, str]] = MappingProxyType( { "prompt_injection": "a potential prompt injection attempt", @@ -402,7 +406,9 @@ class LakeraAIGuardrail(CustomGuardrail): json=request, timeout=self.timeout, ) - verbose_proxy_logger.debug("Lakera AI v2 guard response: %s", response.json()) + verbose_proxy_logger.debug( + "Lakera AI v2 guard response: %s", _DECODED_JSON.validate_python(response.json()) + ) lakera_response = LakeraAIResponse(**response.json()) return lakera_response, masked_entity_count except Exception as e: @@ -801,7 +807,7 @@ class LakeraAIGuardrail(CustomGuardrail): choice_idx = choice_indices[idx] response_dict["choices"][choice_idx]["message"]["content"] = msg.get("content", "") add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name) - return ModelResponse(**response_dict) + return ModelResponse(**_RESPONSE_FIELDS.validate_python(response_dict)) # inject_system_message has nothing left to inject into once a response # already exists, so it is treated the same as monitor: log and allow. diff --git a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py index e05db5003fa..97b35769f63 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py @@ -9,7 +9,7 @@ import json import os import uuid from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict +from typing import TYPE_CHECKING, Final, Literal, TypedDict try: import ulid @@ -39,6 +39,7 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.integrations.custom_guardrail import dc as global_cache +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -397,13 +398,15 @@ class LassoGuardrail(CustomGuardrail): HTTPException: If blocking violations are detected """ raw_messages: Final[list[dict[str, object]]] = data.get("messages") or [] - messages: list[dict[str, Any]] = self._expand_messages_for_classification(raw_messages) if raw_messages else [] - messages_count: Final = len(messages) - if data.get("input") is not None: - # Responses-API payloads carry text in data["input"]. Inspect it - # alongside any "messages" array — otherwise a caller can attach - # benign messages and stash blocked content in input to bypass. - messages.extend(build_inspection_messages({"input": data["input"]})) + expanded_messages: Final = self._expand_messages_for_classification(raw_messages) if raw_messages else [] + messages_count: Final = len(expanded_messages) + # Responses-API payloads carry text in data["input"]. Inspect it + # alongside any "messages" array — otherwise a caller can attach + # benign messages and stash blocked content in input to bypass. + input_messages: Final = ( + build_inspection_messages({"input": data["input"]}) if data.get("input") is not None else [] + ) + messages: Final[Sequence[Mapping[str, object]]] = [*expanded_messages, *input_messages] if not messages: return data @@ -421,7 +424,7 @@ class LassoGuardrail(CustomGuardrail): data: dict, cache: DualCache, message_type: Literal["PROMPT", "COMPLETION"], - messages: list[dict[str, object]], + messages: Sequence[Mapping[str, object]], ) -> dict: """Handle classification without masking.""" try: @@ -439,7 +442,7 @@ class LassoGuardrail(CustomGuardrail): data: dict, cache: DualCache, message_type: Literal["PROMPT", "COMPLETION"], - messages: list[dict[str, object]], + messages: Sequence[Mapping[str, object]], messages_count: int, ) -> dict: """Handle masking with classifix endpoint. @@ -470,7 +473,7 @@ class LassoGuardrail(CustomGuardrail): text_parts = [msg["content"] for msg in masked_for_input if isinstance(msg.get("content"), str)] if text_parts: data["input"] = "\n".join(text_parts) - self._log_masking_applied(message_type, dict(response)) + self._log_masking_applied(message_type, response) return data except Exception as e: @@ -479,7 +482,7 @@ class LassoGuardrail(CustomGuardrail): def _map_masked_messages_back( self, - original_messages: list[dict[str, Any]], + original_messages: Sequence[Mapping[str, object]], masked_messages: Sequence[Mapping[str, object]], ) -> list[dict[str, object]]: """Map Lasso-format masked messages back onto the original OpenAI-format messages. @@ -492,7 +495,7 @@ class LassoGuardrail(CustomGuardrail): """ # Index masked content by type so we can look up by id without caring about order. masked_tool_use: Final[dict[object, dict[str, object]]] = {} - masked_tool_result: Final[dict[str, str]] = {} + masked_tool_result: Final[dict[object, str]] = {} masked_text: Final[list[str]] = [] for msg in masked_messages: @@ -567,12 +570,12 @@ class LassoGuardrail(CustomGuardrail): def _update_tool_calls_from_masked( self, - tool_calls: list[object], + tool_calls: object, masked_tool_use: Mapping[object, Mapping[str, object]], ) -> list[object]: """Replace tool_call arguments with masked values returned by Lasso.""" updated: Final = [] - for call in tool_calls: + for call in iter_items(tool_calls): call_id = self._get_field(call, "id") if call_id and call_id in masked_tool_use: masked_input = masked_tool_use[call_id].get("input") @@ -627,7 +630,7 @@ class LassoGuardrail(CustomGuardrail): def _log_masking_applied( self, message_type: Literal["PROMPT", "COMPLETION"], - response: dict[str, Any], + response: LassoResponse, ) -> None: """Log masking application with structured context.""" conversation_id: Final = getattr(self, "conversation_id", "unknown") @@ -637,12 +640,12 @@ class LassoGuardrail(CustomGuardrail): "guardrail_name": getattr(self, "guardrail_name", "unknown"), "message_type": message_type, "violations_count": len(response.get("findings", {})), - "masked_fields": len(response.get("messages", [])), + "masked_fields": len(response.get("messages") or []), "conversation_id": conversation_id, }, ) - def _expand_messages_for_classification(self, messages: list[dict[str, Any]]) -> list[dict[str, object]]: + def _expand_messages_for_classification(self, messages: Sequence[Mapping[str, object]]) -> list[dict[str, object]]: """ Convert raw OpenAI-format messages to Lasso API format with content blocks. @@ -701,7 +704,7 @@ class LassoGuardrail(CustomGuardrail): expanded.append({"role": role, "content": content}) if role == "assistant": - for call in msg.get("tool_calls") or []: + for call in iter_items(msg.get("tool_calls") or []): call_id, name, input_data = self._extract_tool_call_fields(call) if not call_id or not name: verbose_proxy_logger.warning( @@ -749,7 +752,7 @@ class LassoGuardrail(CustomGuardrail): def _prepare_payload( self, - messages: list[dict[str, object]], + messages: Sequence[Mapping[str, object]], data: dict, cache: DualCache, message_type: Literal["PROMPT", "COMPLETION"] = "PROMPT", diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/competitor_intent/airline.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/competitor_intent/airline.py index 5e2b85250d2..76c3a86d701 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/competitor_intent/airline.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/competitor_intent/airline.py @@ -13,6 +13,9 @@ import json from pathlib import Path from typing import Any, Final +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base import ( # noqa: F401 # legacy module exports BaseCompetitorIntentChecker, _compile_marker, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -114,6 +117,7 @@ AIRLINE_EXPLICIT_OTHER_MEANING_MARKER: Final = ( ) _MAJOR_AIRLINES_PATH: Final = Path(__file__).resolve().parent / "major_airlines.json" +_DECODED_JSON: Final = TypeAdapter(object) def _load_competitors_excluding_brand(brand_self: list[str]) -> list[str]: @@ -127,11 +131,11 @@ def _load_competitors_excluding_brand(brand_self: list[str]) -> list[str]: return [] try: with open(_MAJOR_AIRLINES_PATH, encoding="utf-8") as f: - airlines: Final = json.load(f) + airlines: Final = _DECODED_JSON.validate_python(json.load(f)) except (json.JSONDecodeError, OSError): return [] result: Final[list[str]] = [] - for entry in airlines: + for entry in iter_items(airlines): if not isinstance(entry, dict): continue match_str = entry.get("match") or "" diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py index 40e44151e65..f58c8705815 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py @@ -78,6 +78,7 @@ from .patterns import PATTERN_EXTRA_CONFIG, get_compiled_pattern MAX_KEYWORD_VALUE_GAP_WORDS: Final = 1 _DECODED_ARGUMENTS: Final = TypeAdapter(object) +_JSON_ENTRIES: Final = TypeAdapter(list[object]) GAP_WORD_TOKENIZER: Final = re.compile(r"\b\w+\b") SENTENCE_TERMINATORS: Final = re.compile(r"[.!?]+") @@ -699,9 +700,8 @@ class ContentFilterGuardrail(CustomGuardrail): Severity mapping: 4,3 -> high; 2 -> medium; 1 -> low. """ with open(file_path, "r") as f: - entries = json.load(f) - if not isinstance(entries, list): - entries = [entries] + decoded: Final = _DECODED_ARGUMENTS.validate_python(json.load(f)) + entries: Final = _JSON_ENTRIES.validate_python(decoded) if isinstance(decoded, list) else [decoded] # Derive category name from filename (e.g. harm_toxic_abuse.json -> harm_toxic_abuse) category_name: Final = os.path.splitext(os.path.basename(file_path))[0] severity_map: Final = {4: "high", 3: "high", 2: "medium", 1: "low"} @@ -806,7 +806,7 @@ class ContentFilterGuardrail(CustomGuardrail): """ try: with open(file_path, "r") as f: - data: Final = yaml.safe_load(f) + data: Final = _DECODED_ARGUMENTS.validate_python(yaml.safe_load(f)) if not isinstance(data, dict) or "blocked_words" not in data: raise ValueError("Invalid format: file must contain 'blocked_words' key with list of words") diff --git a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py index 7fb87080cd1..d2dafb04fe0 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py +++ b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py @@ -5,7 +5,7 @@ import uuid from collections.abc import AsyncGenerator, AsyncIterable, Iterator, Mapping, Sequence from typing import ( TYPE_CHECKING, - Any, # noqa: TID251 # the streaming hook and the flush copy pass provider chunks through untyped + Any, # noqa: TID251 # the streaming hook passes provider chunks through untyped ClassVar, Final, Literal, @@ -23,7 +23,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) -from litellm.litellm_core_utils.core_helpers import attribute_items +from litellm.litellm_core_utils.core_helpers import attribute_items, attribute_or_none, iter_items, require_attributes from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -99,13 +99,8 @@ class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): pass -class _ContentDelta(Protocol): - content: object - - -class _FinishingDelta(Protocol): - content: object - tool_calls: object +class _DeepCopyable(Protocol): + def model_copy(self, *, deep: bool) -> object: ... class LLMShieldProxyGuardrail(CustomGuardrail): @@ -528,7 +523,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail): one tool call's accumulating `arguments`. A shared window would splice the text held back for one stream onto another. """ - delta: Final = getattr(choice, "delta", None) + delta: Final = attribute_or_none(choice, "delta") index: Final = choice_index(choice) is_final: Final = bool(getattr(choice, "finish_reason", None)) if isinstance(choice, TextChoices): @@ -566,7 +561,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail): async def _restore_content_window( self, - delta: _ContentDelta, + delta: object, key: CarryKey, carries: CarryWindows, session_id: str, @@ -574,19 +569,19 @@ class LLMShieldProxyGuardrail(CustomGuardrail): ) -> None: """Restores one delta's content through its own window.""" carry: Final = carries.get(key, "") - text: Final = getattr(delta, "content", None) + text: Final = attribute_or_none(delta, "content") if not isinstance(text, str) or not text: if is_final and carry: flushed, remaining = await self._stream_step("", carry, True, session_id) carries[key] = remaining # rebind-ok: this stream's window advances. if flushed: - delta.content = flushed + write_field(delta, "content", flushed) return emitted, remaining = await self._stream_step(text, carry, is_final, session_id) carries[key] = remaining # rebind-ok: this stream's window advances. - delta.content = emitted + write_field(delta, "content", emitted) async def _restore_tool_call_window( self, @@ -618,7 +613,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail): async def _flush_finished_choice( self, - delta: _FinishingDelta, + delta: object, choice_index: int, carries: CarryWindows, session_id: str, @@ -643,12 +638,12 @@ class LLMShieldProxyGuardrail(CustomGuardrail): if not text: continue if tool_index is None: - delta.content = text + write_field(delta, "content", text) else: continuations.extend(continuation_delta(tool_index, text)) if continuations: - existing: Final = tuple(getattr(delta, "tool_calls", None) or ()) - delta.tool_calls = [*existing, *continuations] + existing: Final = tuple(attribute_items(delta, "tool_calls")) + write_field(delta, "tool_calls", [*existing, *continuations]) async def _flush_trailing( self, last_chunk: object, carries: CarryWindows, session_id: str @@ -676,7 +671,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail): chunk = self._chunk_for_choice(last_chunk, choice_index) if chunk is None: continue - choice: object = chunk.choices[0] + choice = read_list(chunk, "choices")[0] delta = read_field(choice, "delta") if isinstance(choice, TextChoices): choice.text = text @@ -688,26 +683,26 @@ class LLMShieldProxyGuardrail(CustomGuardrail): yield chunk @staticmethod - def _chunk_for_choice(last_chunk: Any, index: int) -> Any: + def _chunk_for_choice(last_chunk: object, index: int) -> object | None: """A single-choice copy of the last chunk, carrying only `index`. Emitting one choice per chunk keeps a flush from reading as content on a choice it does not belong to. """ - chunk: Final = last_chunk.model_copy(deep=True) - raw_choices: Final = getattr(chunk, "choices", None) + chunk: Final = require_attributes(last_chunk, _DeepCopyable, "model_copy").model_copy(deep=True) + raw_choices: Final = attribute_or_none(chunk, "choices") if not raw_choices: return None - choices: Final[tuple[object, ...]] = tuple(raw_choices) + choices: Final = tuple(iter_items(raw_choices)) position: Final = next((at for at, choice in enumerate(choices) if choice_index(choice) == index), 0) - kept: Final = raw_choices[position] - if getattr(kept, "delta", None) is None and not isinstance(kept, TextChoices): + kept: Final = choices[position] + if attribute_or_none(kept, "delta") is None and not isinstance(kept, TextChoices): return None - kept.index = index - kept.finish_reason = None - chunk.choices = [kept] + write_field(kept, "index", index) + write_field(kept, "finish_reason", None) + write_field(chunk, "choices", [kept]) if hasattr(chunk, "usage"): - del chunk.usage + delattr(chunk, "usage") return chunk async def _stream_step(self, text: str, carry: str, final: bool, session_id: str) -> tuple[str, str]: diff --git a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/stream_restorers.py b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/stream_restorers.py index 4ef8bbdbb3e..ef007155c46 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/stream_restorers.py +++ b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/stream_restorers.py @@ -7,9 +7,12 @@ from enum import Enum from types import MappingProxyType from typing import ( Final, + Protocol, TypeAlias, ) +from pydantic import TypeAdapter + from .payload import ( JsonBody, MutableRequest, @@ -24,6 +27,8 @@ from .payload import ( write_field, ) +_OPAQUE_VALUE: Final = TypeAdapter(object) + ANTHROPIC_DELTA_FIELDS: Final = MappingProxyType({"text_delta": "text", "input_json_delta": "partial_json"}) SSE_EVENT_BOUNDARY: Final = re.compile(rb"(\r?\n\r?\n)") @@ -80,6 +85,15 @@ def opens_like_sse(head: bytes) -> bool | None: return False +class _ValuedEnum(Protocol): + @property + def value(self) -> object: ... + + +def _enum_value(member: _ValuedEnum) -> object: + return member.value + + def responses_event_type(chunk: object) -> str | None: """The event type of a Responses API stream event, or None for any other chunk. @@ -90,7 +104,7 @@ def responses_event_type(chunk: object) -> str | None: if isinstance(chunk, (bytes, str)): return None kind: Final = read_field(chunk, "type") - value: Final = kind.value if isinstance(kind, Enum) else kind + value: Final = _enum_value(kind) if isinstance(kind, Enum) else kind return value if isinstance(value, str) and value.startswith("response.") else None @@ -173,7 +187,7 @@ class AnthropicSSERestorer: return block line: Final = lines[data_lines[0]] try: - parsed: Final[object] = json.loads(line[len("data:") :]) + parsed: Final = _OPAQUE_VALUE.validate_python(json.loads(line[len("data:") :])) except ValueError: return block event: Final = as_object(parsed) diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/__init__.py index 95db4bd4f77..fa64d306f22 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/__init__.py @@ -1,4 +1,6 @@ -from typing import TYPE_CHECKING, Any, Final, cast +from collections.abc import Mapping +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Protocol, cast from litellm.types.guardrails import SupportedGuardrailIntegrations @@ -8,13 +10,22 @@ if TYPE_CHECKING: from litellm.types.guardrails import Guardrail, LitellmParams +class _ParamsLookup(Protocol): + def get(self, key: str, /) -> object: ... + + +_NO_PARAMS: Final = MappingProxyType[str, object]({}) + + def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"): import litellm # Default to always-on. Only disable if the user explicitly sets default_on: false. # We check the raw guardrail dict because LitellmParams normalizes None → False, # making it impossible to distinguish "not set" from "explicitly false" via litellm_params. - _raw_default_on: Final = cast(dict[str, Any], guardrail).get("litellm_params", {}).get("default_on") + _raw_default_on: Final = ( + cast(Mapping[str, _ParamsLookup], guardrail).get("litellm_params", _NO_PARAMS).get("default_on") + ) _default_on: Final = _raw_default_on is not False _callback: Final = MCPEndUserPermissionGuardrail( diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py index 221c4b3752b..2e3f7120726 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py @@ -74,14 +74,14 @@ import os import re import time from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Optional +from typing import TYPE_CHECKING, Final, Optional import httpx import jwt from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, RSAPublicKey -from typing_extensions import NotRequired, ReadOnly, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack from litellm._logging import verbose_proxy_logger from litellm.caching import DualCache @@ -90,7 +90,6 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.proxy._types import UserAPIKeyAuth -from litellm.types.guardrail_base_init import GuardrailBaseInitKwargs from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import CallTypesLiteral @@ -98,6 +97,10 @@ if TYPE_CHECKING: from jwt.types import Options +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + pass + + class _OIDCDiscoveryDocument(TypedDict, total=False): jwks_uri: str @@ -279,11 +282,13 @@ class MCPJWTSigner(CustomGuardrail): debug_headers: bool = False, # FR-10: Configurable scopes allowed_scopes: list[str] | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ) -> None: - kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) - base_kwargs: Final[GuardrailBaseInitKwargs] = kwargs - super().__init__(**base_kwargs) + options: Final[_CustomGuardrailOptions] = { + "supported_event_hooks": list(self.get_supported_event_hooks()), + **kwargs, + } + super().__init__(**options) # --- Signing key setup --- key_material: Final = os.environ.get(self.SIGNING_KEY_ENV) diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py index 8482567b6da..3cb20a00e2f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py @@ -6,7 +6,7 @@ on the LiteLLM gateway. Blocks or alerts when unregistered servers are found. """ from collections.abc import Mapping -from typing import Any, Final, Literal +from typing import TYPE_CHECKING, Final, Literal from fastapi import HTTPException @@ -21,6 +21,9 @@ from litellm.responses.mcp.litellm_proxy_mcp_handler import ( ) from litellm.types.guardrails import GuardrailEventHooks +if TYPE_CHECKING: + from litellm.caching.caching import DualCache + class MCPSecurityGuardrail(CustomGuardrail): def __init__( @@ -40,7 +43,7 @@ class MCPSecurityGuardrail(CustomGuardrail): async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, - cache: Any, + cache: "DualCache", data: dict, call_type: str, ) -> Exception | str | dict | None: diff --git a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/__init__.py index ed955ac829d..06de4ef0e93 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/__init__.py @@ -1,5 +1,6 @@ from typing import TYPE_CHECKING, Final +from litellm.litellm_core_utils.core_helpers import attribute_or_none from litellm.types.guardrails import SupportedGuardrailIntegrations from .purview_dlp import MicrosoftPurviewDLPGuardrail @@ -11,8 +12,8 @@ if TYPE_CHECKING: def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"): import litellm - tenant_id: Final = getattr(litellm_params, "tenant_id", None) - client_id: Final = getattr(litellm_params, "client_id", None) + tenant_id: Final = attribute_or_none(litellm_params, "tenant_id") + client_id: Final = attribute_or_none(litellm_params, "client_id") # client_secret can be passed via the standard api_key field or as # a dedicated client_secret parameter. diff --git a/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py b/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py index 718401b6e68..43eb54d1464 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py +++ b/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py @@ -10,9 +10,12 @@ import json import os from collections.abc import Callable, Mapping from datetime import datetime -from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, TypeAlias, cast +from typing import TYPE_CHECKING, ClassVar, Final, Literal, Optional, TypeAlias, cast from urllib.parse import urlparse +from pydantic import TypeAdapter +from typing_extensions import TypedDict, Unpack + from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( CustomGuardrail, @@ -25,7 +28,6 @@ from litellm.llms.custom_httpx.http_handler import ( httpxSpecialProvider, ) from litellm.proxy.guardrails.guardrail_hooks.noma.noma import NomaBlockedMessage -from litellm.types.guardrail_base_init import GuardrailBaseInitKwargs from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.guardrails.guardrail_hooks.noma import NomaV2StreamingParams from litellm.types.utils import GenericGuardrailAPIInputs, GuardrailStatus @@ -39,6 +41,7 @@ _DEFAULT_API_BASE: Final = "https://api.noma.security/" _AIDR_SCAN_ENDPOINT: Final = "/litellm/guardrail" _INTERVENED_INPUT_FIELDS: Final = ("texts", "images", "tools", "tool_calls") _DEFAULT_API_BASE_HOSTNAME: Final = urlparse(_DEFAULT_API_BASE).hostname +_DECODED_JSON: Final = TypeAdapter(object) _GuardrailJsonResponse: TypeAlias = Exception | str | dict[str, object] @@ -50,6 +53,10 @@ _LOGGING_KEYS_DUPLICATING_SCAN_INPUTS: Final = _KEYS_DUPLICATING_SCAN_INPUTS + ( ) +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + pass + + class _Action(str, enum.Enum): BLOCKED = "BLOCKED" NONE = "NONE" @@ -74,7 +81,7 @@ class NomaV2Guardrail(CustomGuardrail): streaming_end_of_stream_only: bool | None = None, streaming_sampling_rate: int | None = None, streaming_buffer_release_on_scan: bool | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ) -> None: self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) @@ -115,10 +122,11 @@ class NomaV2Guardrail(CustomGuardrail): if streaming.streaming_buffer_release_on_scan is not None: self.streaming_buffer_release_on_scan: bool = streaming.streaming_buffer_release_on_scan - kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) - - base_kwargs: Final[GuardrailBaseInitKwargs] = kwargs - super().__init__(**base_kwargs) + options: Final[_CustomGuardrailOptions] = { + "supported_event_hooks": list(self.get_supported_event_hooks()), + **kwargs, + } + super().__init__(**options) @staticmethod def get_config_model() -> type["GuardrailConfigModel"] | None: @@ -219,7 +227,7 @@ class NomaV2Guardrail(CustomGuardrail): except (ValueError, TypeError): json_str = safe_dumps(payload) - safe_payload: Final[object] = safe_json_loads(json_str, default={}) + safe_payload: Final = _DECODED_JSON.validate_python(safe_json_loads(json_str, default={})) if safe_payload == {} and payload: verbose_proxy_logger.warning( "Noma v2 guardrail: payload serialization failed, falling back to empty payload" diff --git a/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py b/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py index 1a6feb47215..7c26c4c4c56 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py +++ b/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Final, Literal, Optional import httpx from fastapi import HTTPException +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( @@ -27,6 +28,8 @@ from litellm.types.utils import GenericGuardrailAPIInputs, ModelResponse if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +_DECODED_JSON: Final = TypeAdapter(object) + class OnyxGuardrail(CustomGuardrail): @classmethod @@ -115,7 +118,7 @@ class OnyxGuardrail(CustomGuardrail): try: response: Final = ModelResponse(**request_data) parsed: Final = response.json() - payload = parsed.get("response", {}) + payload = _DECODED_JSON.validate_python(parsed.get("response", {})) except Exception as e: verbose_proxy_logger.error( "Error in converting request_data to ModelResponse: %s", diff --git a/litellm/proxy/guardrails/guardrail_hooks/openai/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/openai/__init__.py index fe526b72ac3..4d120a2dd26 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/openai/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/openai/__init__.py @@ -1,6 +1,7 @@ from typing import TYPE_CHECKING, Final import litellm +from litellm.litellm_core_utils.core_helpers import attribute_or_none from litellm.proxy.guardrails.guardrail_hooks.openai.moderations import ( OpenAIModerationGuardrail, ) @@ -40,12 +41,12 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" return openai_moderation_guardrail -def _get_config_value(litellm_params, optional_params, attribute_name): +def _get_config_value(litellm_params: object, optional_params: object, attribute_name: str) -> object: if optional_params is not None: - value: Final = getattr(optional_params, attribute_name, None) + value: Final = attribute_or_none(optional_params, attribute_name) if value is not None: return value - return getattr(litellm_params, attribute_name, None) + return attribute_or_none(litellm_params, attribute_name) guardrail_initializer_registry: Final = { diff --git a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py index 7af5eaf1da9..98be35079c3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py +++ b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py @@ -8,6 +8,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, Optional from fastapi import HTTPException +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( @@ -40,6 +41,8 @@ if TYPE_CHECKING: from litellm.types.llms.openai import OpenAIModerationResponse from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +_DECODED_JSON: Final = TypeAdapter(object) + def _config_secret(value: object) -> object: return get_secret_str(value) if isinstance(value, str) and value.startswith("os.environ/") else value @@ -151,7 +154,9 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): timeout=self.timeout, ) - verbose_proxy_logger.debug("OpenAI Moderation guard response: %s", response.json()) + verbose_proxy_logger.debug( + "OpenAI Moderation guard response: %s", _DECODED_JSON.validate_python(response.json()) + ) if response.status_code != 200: raise HTTPException( diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index 8e03f7db4f5..6426bfb8788 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -18,6 +18,7 @@ from urllib.parse import urlparse import httpx from fastapi import HTTPException from pydantic import ConfigDict, TypeAdapter, ValidationError, field_validator +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid @@ -129,6 +130,21 @@ _RESPONSES_INPUT: Final[TypeAdapter[_ResponsesInput]] = TypeAdapter(_ResponsesIn _DECODED_ARGUMENTS: Final = TypeAdapter(object) +class _AirsToolEvent(TypedDict): + metadata: ReadOnly[Mapping[str, object]] + input: NotRequired[ReadOnly[str]] + + +def _mcp_tool_input(mcp_arguments: object) -> str | None: + if mcp_arguments is not None and mcp_arguments != "": + serialized_args: Final = ( + json.dumps(mcp_arguments) if isinstance(mcp_arguments, (dict, list)) else str(mcp_arguments) + ) + if serialized_args.strip(): + return serialized_args + return None + + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel @@ -339,7 +355,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): is_response: bool = False, metadata: Mapping[str, object] | None = None, call_id: object = None, - tool_event: dict[str, Any] | None = None, + tool_event: _AirsToolEvent | None = None, ) -> dict[str, object]: """Call PANW Prisma AIRS API to scan content or a tool_event.""" @@ -894,7 +910,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): if not line.startswith("data: "): continue try: - data = json.loads(line[6:]) + data = _DECODED_ARGUMENTS.validate_python(json.loads(line[6:])) except (json.JSONDecodeError, ValueError): continue if not isinstance(data, dict): @@ -1954,24 +1970,21 @@ class PanwPrismaAirsHandler(CustomGuardrail): # "mcp_tool_name"/"mcp_arguments". Check canonical first, then fallback. mcp_tool_name: Final = request_data.get("mcp_tool_name") or self._mcp_name_fallback(request_data) if mcp_tool_name and input_type == "request": - mcp_tool_event: Final[dict[str, object]] = { - "metadata": { - "ecosystem": "mcp", - "method": "tools/call", - "server_name": self._get_mcp_server_name(request_data, mcp_tool_name), - "tool_invoked": mcp_tool_name, - }, + mcp_tool_metadata: Final = { + "ecosystem": "mcp", + "method": "tools/call", + "server_name": self._get_mcp_server_name(request_data, mcp_tool_name), + "tool_invoked": mcp_tool_name, } mcp_arguments = request_data.get("mcp_arguments") if mcp_arguments is None: mcp_arguments = request_data.get("arguments") - if mcp_arguments is not None and mcp_arguments != "": - if isinstance(mcp_arguments, (dict, list)): - serialized_args = json.dumps(mcp_arguments) - else: - serialized_args = str(mcp_arguments) - if serialized_args.strip(): - mcp_tool_event["input"] = serialized_args + mcp_tool_input: Final = _mcp_tool_input(mcp_arguments) + mcp_tool_event: Final[_AirsToolEvent] = ( + {"metadata": mcp_tool_metadata} + if mcp_tool_input is None + else {"metadata": mcp_tool_metadata, "input": mcp_tool_input} + ) mcp_scan_result: Final = await self._call_panw_api( tool_event=mcp_tool_event, diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index cadc9ca689b..7e2f6bfec33 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -85,6 +85,7 @@ class _PresidioAnonymizeResponse(TypedDict): _PRESIDIO_ANONYMIZE_ADAPTER: Final[TypeAdapter[_PresidioAnonymizeResponse | None]] = TypeAdapter( _PresidioAnonymizeResponse | None ) +_DECODED_JSON: Final = TypeAdapter(object) class _JsonResponse(Protocol): @@ -1595,10 +1596,11 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if line.startswith("data: ") and line != "data: [DONE]": raw_json = line[6:] try: - event = json.loads(raw_json) + event = _DECODED_JSON.validate_python(json.loads(raw_json)) delta = event.get("delta") if isinstance(event, dict) else None if ( - isinstance(delta, dict) + isinstance(event, dict) + and isinstance(delta, dict) and event.get("type") == "content_block_delta" and delta.get("type") == "text_delta" and isinstance(delta.get("text"), str) diff --git a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py index a92e67046bd..6150fccac0c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py +++ b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py @@ -10,6 +10,7 @@ import os from typing import Any, Final, Literal from fastapi import HTTPException +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( @@ -29,6 +30,7 @@ from litellm.types.utils import GenericGuardrailAPIInputs GUARDRAIL_NAME: Final = "qualifire" DEFAULT_QUALIFIRE_API_BASE: Final = "https://proxy.qualifire.ai" +_DECODED_JSON: Final = TypeAdapter(object) class QualifireGuardrail(CustomGuardrail): @@ -188,7 +190,7 @@ class QualifireGuardrail(CustomGuardrail): args = function_info.get("arguments", {}) if isinstance(args, str): try: - args = json.loads(args) + args = _DECODED_JSON.validate_python(json.loads(args)) except json.JSONDecodeError: args = {} api_tool_calls.append( diff --git a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py index 1a571047977..342e5b64714 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py +++ b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py @@ -10,6 +10,7 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final import yaml +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.constants import DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD @@ -22,6 +23,7 @@ if TYPE_CHECKING: ROUTE_TEMPLATES_DIR: Final = os.path.join(os.path.dirname(__file__), "route_templates") +_DECODED_YAML: Final = TypeAdapter(object) class SemanticGuardRouteLoader: @@ -55,7 +57,7 @@ class SemanticGuardRouteLoader: if not os.path.exists(file_path): raise ValueError(f"SemanticGuard: custom routes file not found: {file_path}") with open(file_path, "r") as f: - data: Final = yaml.safe_load(f) + data: Final = _DECODED_YAML.validate_python(yaml.safe_load(f)) if isinstance(data, list): return data if isinstance(data, dict): diff --git a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/semantic_guard.py b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/semantic_guard.py index 2fbd50b5863..37acee28b53 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/semantic_guard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/semantic_guard.py @@ -6,13 +6,14 @@ via embedding similarity. Smarter than regex (understands intent), lighter than an LLM call (~20-50ms per request for embedding). """ -from typing import TYPE_CHECKING, Any, Final, Protocol +from typing import TYPE_CHECKING, Final, Protocol from litellm._logging import verbose_logger from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.litellm_core_utils.core_helpers import attribute_items, attribute_or_none, require_attributes from litellm.proxy.guardrails.guardrail_hooks.semantic_guard.route_loader import ( SemanticGuardRouteLoader, ) @@ -188,17 +189,21 @@ def _extract_user_text(messages: list) -> str: return "" -def _extract_response_text(response: Any) -> str: +class _MessageContent(Protocol): + @property + def content(self) -> object: ... + + +def _extract_response_text(response: object) -> str: """Extract text from every LLM response choice.""" - if hasattr(response, "choices") and response.choices: - text_parts: Final[list[str]] = [] - for choice in response.choices: - if hasattr(choice, "message") and choice.message: - text = _content_to_text(choice.message.content) - if text: - text_parts.append(text) - return "\n".join(text_parts) - return "" + text_parts: Final[list[str]] = [] + for choice in attribute_items(response, "choices"): + message = attribute_or_none(choice, "message") + if message: + text = _content_to_text(require_attributes(message, _MessageContent, "content").content) + if text: + text_parts.append(text) + return "\n".join(text_parts) def _content_to_text(content: object) -> str: diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_policy/tool_policy_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/tool_policy/tool_policy_guardrail.py index de0f8856522..fbf1f2576ff 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_policy/tool_policy_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_policy/tool_policy_guardrail.py @@ -105,7 +105,7 @@ def _resolve_tool_names_from_messages(messages: list[dict]) -> dict[str, str]: fn = (tc.get("function") or {}).get("name") else: tc_id = getattr(tc, "id", None) - fn_obj = getattr(tc, "function", None) + fn_obj: object = getattr(tc, "function", None) fn = getattr(fn_obj, "name", None) if fn_obj else None if tc_id and fn: mapping[tc_id] = fn diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index 3ad9d1a2c42..d713f1078a2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, cast, runtime import anyio from fastapi import HTTPException +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache @@ -50,6 +51,8 @@ A2A_CALL_TYPES: Final = (CallTypes.asend_message, CallTypes.send_message) GUARDRAIL_NAME: Final = "unified_llm_guardrails" +_DECODED_JSON: Final = TypeAdapter(object) + _RequestData: TypeAlias = dict[str, object] @@ -183,7 +186,7 @@ def _get_a2a_request_id(responses_so_far: Sequence[object], request_data: dict) return item.get("id") if isinstance(item, str): try: - obj = json.loads(item.strip()) + obj = _DECODED_JSON.validate_python(json.loads(item.strip())) if isinstance(obj, dict) and "id" in obj: return obj.get("id") except (json.JSONDecodeError, TypeError): @@ -729,7 +732,7 @@ class UnifiedLLMGuardrails(CustomLogger): user_api_key_dict: UserAPIKeyAuth, call_type: str, sampling_rate: int, - end_of_stream_only: bool, + end_of_stream_only: object, mappings: Mapping[CallTypes, type["BaseTranslation"]], ) -> AsyncGenerator[object, None]: """Emit guardrail text transformations as new deltas on the stream. @@ -1135,16 +1138,19 @@ class UnifiedLLMGuardrails(CustomLogger): guardrail_to_apply = request_data.pop("guardrail_to_apply", None) typed_request_data: Final[_RequestData] = request_data - def _streaming_flag(name: str, default: object) -> Any: + def _streaming_setting(name: str, default: object) -> object: return self.resolve_streaming_flag(guardrail_to_apply, name, default) + def _streaming_flag(name: str, default: object) -> Any: + return _streaming_setting(name, default) + sampling_rate: Final[int] = _streaming_flag("streaming_sampling_rate", 5) # Only apply the guardrail at end of stream (not per chunk). - end_of_stream_only: bool = _streaming_flag("streaming_end_of_stream_only", False) + end_of_stream_only: object = _streaming_setting("streaming_end_of_stream_only", False) # "block_only" (default) drops guardrail text rewrites on the streaming # path; "incremental_diff" emits them as synthetic deltas (see # _run_incremental_transform_stream). - streaming_transform_mode: Final[str] = _streaming_flag("streaming_transform_mode", "block_only") + streaming_transform_mode: Final = _streaming_setting("streaming_transform_mode", "block_only") # Withhold every chunk until end-of-stream moderation passes, then # release the original chunks (clean) or only the block message # (blocked) -- moderating the whole response *before* any content @@ -1152,10 +1158,10 @@ class UnifiedLLMGuardrails(CustomLogger): # release the original chunks are replayed as-is, so a # content-rewriting guardrail (e.g. PII masking) would leak # unredacted content. Guarded below via mask_response_content. - buffer_until_moderated: bool = _streaming_flag( + buffer_until_moderated: object = _streaming_setting( "streaming_buffer_until_moderated", buffer_until_moderated_default ) - release_on_scan: Final[bool] = _streaming_flag("streaming_buffer_release_on_scan", False) + release_on_scan: Final = _streaming_setting("streaming_buffer_release_on_scan", False) if ( buffer_until_moderated diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 7521c1e9f45..24b30e65e2b 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -102,6 +102,16 @@ def _as_json_array(value: object) -> list[object] | None: return _JSON_ARRAY.validate_python(value) if isinstance(value, list) else None +def _extra_param_items(litellm_params: LitellmParams) -> Iterable[tuple[str, object]]: + return ( + litellm_params.model_dump(exclude_none=True).items() + if hasattr(litellm_params, "model_dump") + else iter(litellm_params) + if litellm_params + else () + ) + + def contains_encrypted_marker(value: object, depth: int = 0) -> bool: """True if any string in value, at any JSON depth, starts with the encrypted-value prefix.""" if depth > DEFAULT_MAX_RECURSE_DEPTH: @@ -904,13 +914,7 @@ class InMemoryGuardrailHandler: # This matches the behavior of other guardrail initializers (e.g., initialize_lakera) # and aligns with the documented behavior for custom guardrails excluded_extra_param_keys: Final = frozenset(("guardrail", "mode", "default_on", "stream_scope")) - extra_params_items: Final = ( - litellm_params.model_dump(exclude_none=True).items() - if hasattr(litellm_params, "model_dump") - else iter(litellm_params) - if litellm_params - else () - ) + extra_params_items: Final = _extra_param_items(litellm_params) extra_params: Final = MappingProxyType( {key: value for key, value in extra_params_items if key not in excluded_extra_param_keys} ) diff --git a/litellm/proxy/guardrails/tool_name_extraction.py b/litellm/proxy/guardrails/tool_name_extraction.py index 526f58dcd36..c49835905fb 100644 --- a/litellm/proxy/guardrails/tool_name_extraction.py +++ b/litellm/proxy/guardrails/tool_name_extraction.py @@ -6,8 +6,8 @@ knowledge lives in one place. Uses guardrail translation handlers where availabl with standalone extractors for generate_content and MCP. """ -from collections.abc import Mapping -from typing import Any, Final +from collections.abc import Callable, Mapping +from typing import Final from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route from litellm.llms import load_guardrail_translation_mappings @@ -15,7 +15,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import list_field from litellm.types.utils import CallTypes # Call types that have no guardrail translation handler; we use standalone extractors -STANDALONE_EXTRACTORS: Final[dict[str, Any]] = {} +STANDALONE_EXTRACTORS: Final[dict[str, Callable[[Mapping[str, object]], list[str]]]] = {} def _extract_generate_content_tool_names(data: Mapping[str, object]) -> list[str]: @@ -28,7 +28,7 @@ def _extract_generate_content_tool_names(data: Mapping[str, object]) -> list[str return names -def _extract_mcp_tool_names(data: dict) -> list[str]: +def _extract_mcp_tool_names(data: Mapping[str, object]) -> list[str]: """MCP call_tool: name or mcp_tool_name in body""" names: Final[list[str]] = [] name: Final = data.get("name") or data.get("mcp_tool_name") diff --git a/litellm/proxy/guardrails/usage_endpoints.py b/litellm/proxy/guardrails/usage_endpoints.py index 2f466f22ef8..caa66bddf39 100644 --- a/litellm/proxy/guardrails/usage_endpoints.py +++ b/litellm/proxy/guardrails/usage_endpoints.py @@ -11,10 +11,11 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, overload from fastapi import APIRouter, Depends, Query -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, TypeAdapter from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.guardrails.usage_tracking import guardrail_status_to_action @@ -838,28 +839,41 @@ def _snippet(text: object, max_len: int = 200) -> str | None: return result -def _input_snippet_for_log(sl: "prisma_models.LiteLLM_SpendLogs") -> str | None: - """Snippet for request input: prefer messages, fall back to proxy_server_request (same as drawer).""" - out = _snippet(sl.messages) +_DECODED_JSON: Final = TypeAdapter(object) + + +def _request_dict(value: object) -> Mapping[str, object] | None: + mapping: Final = as_str_mapping(value) + return mapping if isinstance(mapping, dict) else None + + +def _proxy_request_snippet(psr: object) -> str | None: + request: Final = _request_dict(psr) + if request is None: + return _snippet(psr) + msgs: Final = request.get("messages") + body: Final = _request_dict(request.get("body")) if msgs is None else None + out: Final = _snippet(body.get("messages") if body is not None else msgs) if out: return out - psr = getattr(sl, "proxy_server_request", None) + return _snippet(psr) + + +def _input_snippet_for_log(sl: "prisma_models.LiteLLM_SpendLogs") -> str | None: + """Snippet for request input: prefer messages, fall back to proxy_server_request (same as drawer).""" + out: Final = _snippet(sl.messages) + if out: + return out + psr: Final[object] = getattr(sl, "proxy_server_request", None) if not psr: return None - if isinstance(psr, str): - try: - psr = json.loads(psr) - except Exception: - return _snippet(psr) - if isinstance(psr, dict): - msgs = psr.get("messages") - if msgs is None and isinstance(psr.get("body"), dict): - msgs = psr["body"].get("messages") - out = _snippet(msgs) - if out: - return out + if not isinstance(psr, str): + return _proxy_request_snippet(psr) + try: + decoded: Final = _DECODED_JSON.validate_python(json.loads(psr)) + except Exception: return _snippet(psr) - return _snippet(psr) + return _proxy_request_snippet(decoded) @router.get( diff --git a/litellm/proxy/hooks/key_management_event_hooks.py b/litellm/proxy/hooks/key_management_event_hooks.py index 47c3869fb64..8f9aebecf30 100644 --- a/litellm/proxy/hooks/key_management_event_hooks.py +++ b/litellm/proxy/hooks/key_management_event_hooks.py @@ -455,10 +455,10 @@ class KeyManagementEventHooks: try: from litellm.proxy import proxy_server as proxy_server_module + from litellm.proxy.proxy_server import prisma_client except ImportError: return None - prisma_client: Final = getattr(proxy_server_module, "prisma_client", None) user_api_key_cache: Final = getattr(proxy_server_module, "user_api_key_cache", None) if prisma_client is None or user_api_key_cache is None: diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index 47ea50debc9..05c64317a44 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -892,7 +892,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): {**_hidden_params, "additional_headers": _additional_headers}, ) - return await super().async_post_call_success_hook(data, user_api_key_dict, response) + await super().async_post_call_success_hook(data, user_api_key_dict, response) PROXY_MaxParallelRequestsHandler: Final = _PROXY_MaxParallelRequestsHandler diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index a2bb691b401..567b8452ae2 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -101,7 +101,7 @@ if TYPE_CHECKING: from litellm.types.agents import AgentResponse from litellm.types.caching import RedisPipelineIncrementOperation - Span = _Span | Any + Span = _Span InternalUsageCache = _InternalUsageCache else: Span = Any diff --git a/litellm/proxy/hooks/rate_limiter_utils.py b/litellm/proxy/hooks/rate_limiter_utils.py index bbe38068638..c4949834896 100644 --- a/litellm/proxy/hooks/rate_limiter_utils.py +++ b/litellm/proxy/hooks/rate_limiter_utils.py @@ -7,6 +7,7 @@ from typing import Final import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import PROXY_LLM_PROVIDER_FALLBACK +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.types.router import ModelGroupInfo from litellm.types.utils import PriorityReservationDict @@ -85,10 +86,10 @@ def _resolve_provider_from_router_alias( if llm_router is None: return None try: - model_list: Final = getattr(llm_router, "model_list", None) + model_list: Final[object] = getattr(llm_router, "model_list", None) if not model_list: return None - for deployment in model_list: + for deployment in iter_items(model_list): if not isinstance(deployment, dict): continue if deployment.get("model_name") != model: diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 5f31aa788e2..a87455e5a60 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -9,6 +9,7 @@ from typing_extensions import ReadOnly, TypedDict from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy._types import ( CommonProxyErrors, @@ -166,6 +167,16 @@ class _AccessGroupTx(Protocol): def litellm_agentstable(self) -> _AgentTable: ... +class _AccessGroupTransaction(Protocol): + async def __aenter__(self) -> _AccessGroupTx: ... + + async def __aexit__(self, *exc_info: object) -> None: ... + + +def _access_group_transaction(prisma_client: PrismaClient) -> _AccessGroupTransaction: + return prisma_client.db.tx() + + def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None: if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: raise HTTPException( @@ -191,6 +202,10 @@ def _tool_permissions_json(value: Mapping[str, Sequence[str]] | None) -> "Json": return Json(None) if value is None else Json({server: list(tools) for server, tools in value.items()}) +def _explicitly_set_fields(data: AccessGroupUpdateRequest) -> Mapping[str, object]: + return data.model_dump(exclude_unset=True) + + @dataclass(frozen=True, slots=True) class _ResourceNames: mcp_servers: Mapping[str, str] @@ -539,7 +554,7 @@ async def create_access_group( try: tx: _AccessGroupTx - async with prisma_client.db.tx() as tx: + async with _access_group_transaction(prisma_client) as tx: existing: Final = await tx.litellm_accessgrouptable.find_unique( where={"access_group_name": data.access_group_name} ) @@ -647,7 +662,7 @@ async def update_access_group( _require_proxy_admin(user_api_key_dict) prisma_client: Final = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) - update_fields: Final = data.model_dump(exclude_unset=True) + update_fields: Final = _explicitly_set_fields(data) update_data: Final[dict] = {"updated_by": user_api_key_dict.user_id} for field, value in update_fields.items(): if field in ( @@ -658,7 +673,7 @@ async def update_access_group( "access_agent_ids", "access_skill_names", ): - value = list(value or ()) + value = list(iter_items(value or ())) update_data[field] = ( _tool_permissions_json(data.mcp_tool_permissions) if field == "mcp_tool_permissions" else value ) @@ -672,7 +687,7 @@ async def update_access_group( try: tx: _AccessGroupTx - async with prisma_client.db.tx() as tx: + async with _access_group_transaction(prisma_client) as tx: # Read inside the transaction so delta computation is consistent with the write, # avoiding a TOCTOU race where a concurrent update could make deltas stale. existing: Final = await tx.litellm_accessgrouptable.find_unique(where={"access_group_id": access_group_id}) @@ -687,10 +702,10 @@ async def update_access_group( old_team_ids: Final[set[str]] = set(attached[access_group_id]) old_key_ids: Final[set[str]] = set(existing.assigned_key_ids or []) new_team_ids: Final[set[str]] = ( - set(update_fields["assigned_team_ids"] or []) if "assigned_team_ids" in update_fields else old_team_ids + set(data.assigned_team_ids or []) if "assigned_team_ids" in update_fields else old_team_ids ) new_key_ids: Final[set[str]] = ( - set(update_fields["assigned_key_ids"] or []) if "assigned_key_ids" in update_fields else old_key_ids + set(data.assigned_key_ids or []) if "assigned_key_ids" in update_fields else old_key_ids ) teams_to_add = list(new_team_ids - old_team_ids) @@ -747,7 +762,7 @@ async def delete_access_group( affected_key_tokens: list[str] = [] tx: _AccessGroupTx - async with prisma_client.db.tx() as tx: + async with _access_group_transaction(prisma_client) as tx: existing: Final = await tx.litellm_accessgrouptable.find_unique(where={"access_group_id": access_group_id}) if existing is None: raise HTTPException( diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index d40f64f435f..9c46e711cf6 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -54,6 +54,7 @@ from litellm.proxy.management_helpers.auto_router_permissions import ( from litellm.repositories.autorouter_session_repository import AutoRouterSessionRepository from litellm.repositories.base_repository import SupportsModelDump from litellm.repositories.daily_activity_sql import build_where_clause +from litellm.repositories.table_repositories import PrismaTableRepository from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository from litellm.repositories.verification_token_repository import VerificationTokenRepository @@ -179,7 +180,7 @@ class _ShadowEvalAttemptRow(Protocol): class _ShadowEvalFunnelTable(Protocol): - async def create_many(self, data: Sequence[Mapping[str, object]], skip_duplicates: bool) -> int: ... + async def create_many(self, data: Sequence[Mapping[str, object]], *, skip_duplicates: bool) -> int: ... class _ShadowEvalAttemptTable(Protocol): @@ -204,16 +205,28 @@ def _user_rows(prisma_client: "PrismaClient") -> _UserRowsTable: return UserRepository(prisma_client).table +class _ShadowEvalJobRepository(PrismaTableRepository[_ShadowEvalJobRow]): + table_name = "litellm_shadowevaljob" + + +class _ShadowEvalFunnelRepository(PrismaTableRepository[object]): + table_name = "litellm_shadowevalfunnel" + + +class _ShadowEvalAttemptRepository(PrismaTableRepository[_ShadowEvalAttemptRow]): + table_name = "litellm_shadowevalattempt" + + def _shadow_eval_jobs(prisma_client: "PrismaClient") -> _ShadowEvalJobTable: - return prisma_client.db.litellm_shadowevaljob + return _ShadowEvalJobRepository(prisma_client).table def _shadow_eval_funnel(prisma_client: "PrismaClient") -> _ShadowEvalFunnelTable: - return prisma_client.db.litellm_shadowevalfunnel # pyright: ignore[reportAttributeAccessIssue] # generated client + return _ShadowEvalFunnelRepository(prisma_client).table def _shadow_eval_attempts(prisma_client: "PrismaClient") -> _ShadowEvalAttemptTable: - return prisma_client.db.litellm_shadowevalattempt + return _ShadowEvalAttemptRepository(prisma_client).table async def _query_raw(prisma_client: "PrismaClient", query: str, *args: object) -> Sequence[Mapping[str, object]]: @@ -221,6 +234,10 @@ async def _query_raw(prisma_client: "PrismaClient", query: str, *args: object) - return await prisma_client.db.query_raw(query, *args) +async def _execute_raw(prisma_client: "PrismaClient", query: str, *args: object) -> int: + return await prisma_client.db.execute_raw(query, *args) + + async def _authorize_router_dry_run(user_api_key_dict: UserAPIKeyAuth, team_id: str | None) -> LiteLLM_TeamTable | None: from litellm.proxy.management_endpoints.model_management_endpoints import ( ModelManagementAuthChecks, @@ -1788,7 +1805,7 @@ async def start_shadow_eval( # deliberate. Sweep and claim filter on exact (target_type, id) pairs so a team id # that happens to equal a key hash never matches the other kind's slot. for target_type, ids in requested_by_type: - await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, list(ids), target_type) + await _execute_raw(prisma_client, _SWEEP_FINISHED_JOBS_SQL, list(ids), target_type) claimed: Final = await _shadow_eval_jobs(prisma_client).find_many( where={ "OR": [ @@ -2005,8 +2022,8 @@ async def stop_shadow_eval_job( raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) stamp: Final = datetime.now(timezone.utc) operator: Final = user_api_key_dict.user_id or "operator" - claimed: Final = await prisma_client.db.execute_raw( - _STOP_JOB_SQL, job_id, operator, stamp.replace(tzinfo=None).isoformat() + claimed: Final = await _execute_raw( + prisma_client, _STOP_JOB_SQL, job_id, operator, stamp.replace(tzinfo=None).isoformat() ) legs: Final = _LEG_ROWS.validate_python( await _shadow_eval_jobs(prisma_client).find_many(where={"group_id": job_id}) or () diff --git a/litellm/proxy/management_endpoints/credential_migration.py b/litellm/proxy/management_endpoints/credential_migration.py index b1f0002e143..14dd9ce6e8a 100644 --- a/litellm/proxy/management_endpoints/credential_migration.py +++ b/litellm/proxy/management_endpoints/credential_migration.py @@ -42,6 +42,7 @@ from litellm.proxy.db.db_span import db_span if TYPE_CHECKING: from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import PrismaClient + from litellm.repositories.prisma_protocols import TableActions from litellm.proxy.common_utils.encrypt_decrypt_utils import ( # noqa: F401 # legacy module exports _ALGO_AES_GCM, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _ENCRYPTION_ALGORITHM_SETTING, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -523,7 +524,7 @@ async def _scan_one_table( scalar_columns: tuple, ) -> LocationReport: report: Final = LocationReport(location=location) - table: Final = getattr(prisma_client.db, db_attr, None) + table: Final[TableActions[object] | None] = getattr(prisma_client.db, db_attr, None) if table is None: return report try: diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index eebc537509b..0ad4258fb04 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -211,6 +211,7 @@ if TYPE_CHECKING: from prisma import models as prisma_models _RepositoryModelT = TypeVar("_RepositoryModelT", bound=BaseModel) +_RawRowT = TypeVar("_RawRowT") class _UserRowLike(Protocol): @@ -234,6 +235,12 @@ class _TxTables(Protocol): litellm_proxymodeltable: TableActions[object] +class _ModelReencryptTransaction(Protocol): + async def __aenter__(self) -> _TxTables: ... + + async def __aexit__(self, *exc_info: object) -> None: ... + + class _ModelParamsUpdate(TypedDict): litellm_params: ReadOnly["prisma.Json"] @@ -334,6 +341,14 @@ def _credentials_table(prisma_client: PrismaClient) -> TableActions[CredentialIt ) +async def _query_raw_rows(prisma_client: PrismaClient, query: str, *args: object) -> Sequence[_RawRowT]: + return await prisma_client.db.query_raw(query, *args) + + +def _model_reencrypt_transaction(prisma_client: PrismaClient) -> _ModelReencryptTransaction: + return prisma_client.db.tx(timeout=timedelta(minutes=2)) + + def _config_table(prisma_client: PrismaClient, *, use_writer: bool = False) -> _ConfigTableActions: return cast( # cast-ok: ConfigRepository.table hides the write actions this module needs on that same object "_ConfigTableActions", ConfigRepository(prisma_client, use_writer=use_writer).table @@ -1313,11 +1328,6 @@ async def validate_team_id_used_in_service_account_request( return True -_BUDGET_NUMERIC_KEYS = frozenset( - ["max_budget", "soft_budget", "max_parallel_requests", "tpm_limit", "rpm_limit", "tpd_limit"] -) - - def _enforce_upperbound_key_params( data: GenerateKeyRequest | UpdateKeyRequest, fill_defaults: bool = True, @@ -1332,13 +1342,20 @@ def _enforce_upperbound_key_params( # (GHSA-2rv4-xv66-fpjg): float('nan') passes every `< 0` check because # nan < 0 is False, and spend >= nan is always False, permanently disabling # budget enforcement for any key that carries it. - for elem in data: - key, value = elem - if key in _BUDGET_NUMERIC_KEYS and value is not None: - if not math.isfinite(value): + budget_numeric_fields: Final = ( + ("max_budget", data.max_budget), + ("max_parallel_requests", data.max_parallel_requests), + ("tpm_limit", data.tpm_limit), + ("rpm_limit", data.rpm_limit), + ("tpd_limit", data.tpd_limit), + ("soft_budget", data.soft_budget), + ) + for field_name, field_value in budget_numeric_fields: + if field_value is not None: + if not math.isfinite(field_value): raise HTTPException( status_code=400, - detail={"error": f"{key} must be a finite number. Received: {value}"}, + detail={"error": f"{field_name} must be a finite number. Received: {field_value}"}, ) if litellm.upperbound_key_generate_params is None: @@ -5746,8 +5763,7 @@ async def rotate_master_key( ] ) verbose_proxy_logger.debug("Re-encrypting litellm_params on %s model rows", len(reencrypted_models)) - async with prisma_client.db.tx(timeout=timedelta(minutes=2)) as tx_ctx: - tx: Final[_TxTables] = tx_ctx + async with _model_reencrypt_transaction(prisma_client) as tx: for reencrypted_model in reencrypted_models: await tx.litellm_proxymodeltable.update_many( data=_ModelParamsUpdate(litellm_params=prisma.Json(reencrypted_model.litellm_params)), @@ -7344,7 +7360,7 @@ async def key_aliases( where_sql: Final = " AND ".join(where_parts) count_sql: Final = f'SELECT COUNT(*) AS count FROM "LiteLLM_VerificationToken" WHERE {where_sql}' - count_rows: Final[Sequence[Mapping[str, int]]] = await prisma_client.db.query_raw(count_sql, *query_params) + count_rows: Final[Sequence[Mapping[str, int]]] = await _query_raw_rows(prisma_client, count_sql, *query_params) total_count: Final = int(count_rows[0]["count"]) if count_rows else 0 aliases_params: Final = query_params + [size, (page - 1) * size] @@ -7357,7 +7373,9 @@ async def key_aliases( f" ORDER BY key_alias ASC" f" LIMIT ${limit_idx} OFFSET ${offset_idx}" ) - alias_rows: Final[Sequence[Mapping[str, str]]] = await prisma_client.db.query_raw(aliases_sql, *aliases_params) + alias_rows: Final[Sequence[Mapping[str, str]]] = await _query_raw_rows( + prisma_client, aliases_sql, *aliases_params + ) aliases: Final[list[str]] = [row["key_alias"] for row in alias_rows if row.get("key_alias")] total_pages: Final = -(-total_count // size) if total_count > 0 else 0 diff --git a/litellm/proxy/management_endpoints/management_v1/spend_logs.py b/litellm/proxy/management_endpoints/management_v1/spend_logs.py index b93c05c555f..da84c2d0013 100644 --- a/litellm/proxy/management_endpoints/management_v1/spend_logs.py +++ b/litellm/proxy/management_endpoints/management_v1/spend_logs.py @@ -1,10 +1,12 @@ """`/management/v1/spend_logs` facets.""" +from collections.abc import Sequence from datetime import datetime, timezone from functools import partial -from typing import Annotated, Final, Literal +from typing import Annotated, Final, Literal, Protocol from fastapi import APIRouter, Depends, Query, Request +from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth @@ -19,6 +21,7 @@ from litellm.proxy.list_api.common import ( reject_unknown_query_params, ) from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX +from litellm.proxy.utils import PrismaClient from litellm.types.proxy.management_endpoints.management_v1 import ( FacetListResponse, PageMeta, @@ -33,6 +36,19 @@ router: Final = APIRouter(prefix=MANAGEMENT_V1_PREFIX) SPEND_LOGS_FACET_SCAN_CAP: Final = 10000 +class _FacetRow(TypedDict, total=False): + end_user: ReadOnly[str | None] + user: ReadOnly[str | None] + + +class _FacetQuery(Protocol): + async def __call__(self, query: str, *args: object) -> Sequence[_FacetRow]: ... + + +def _facet_query(prisma_client: PrismaClient) -> _FacetQuery: + return prisma_client.db.query_raw + + def _as_utc(value: datetime) -> datetime: return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc) @@ -125,8 +141,8 @@ async def _list_spend_log_facet( f" ORDER BY {column_sql} ASC" f" LIMIT ${scan_idx + 1} OFFSET ${scan_idx + 2}" ) - rows: Final = await prisma_client.db.query_raw(facet_sql, *params) - values: Final[list[str]] = [row[column] for row in rows if row.get(column)] + rows: Final = await _facet_query(prisma_client)(facet_sql, *params) + values: Final[list[str]] = [value for row in rows if (value := row.get(column))] has_more: Final = len(values) > page_size return FacetListResponse( diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index a014b9cf8a5..776072c69be 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -104,6 +104,14 @@ class _HasServerId(Protocol): server_id: str +class _TemporaryServerCacheWriter(Protocol): + async def async_set_cache(self, *, key: str, value: str, ttl: int) -> object: ... + + +class _TemporaryServerCacheReader(Protocol): + async def async_get_cache(self, *, key: str) -> object: ... + + def _session_cookie_claims(claims: Mapping[str, object]) -> Mapping[str, object]: return claims @@ -527,7 +535,7 @@ if MCP_AVAILABLE: """ if litellm.cache is None or not hasattr(litellm.cache, "cache"): return - cache_backend: Final = getattr(litellm.cache, "cache", None) + cache_backend: Final[_TemporaryServerCacheWriter | None] = getattr(litellm.cache, "cache", None) if cache_backend is None or not hasattr(cache_backend, "async_set_cache"): return @@ -566,7 +574,7 @@ if MCP_AVAILABLE: """ if litellm.cache is None or not hasattr(litellm.cache, "cache"): return None - cache_backend: Final = getattr(litellm.cache, "cache", None) + cache_backend: Final[_TemporaryServerCacheReader | None] = getattr(litellm.cache, "cache", None) if cache_backend is None or not hasattr(cache_backend, "async_get_cache"): return None diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index fc0868a5fc9..11f616a5e18 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -3217,12 +3217,13 @@ async def update_useful_links( ) -def _validated_labeled_tiers( - tier_labels: dict[ComplexityTier, str], # mutable-ok: Pydantic materializes JSON object fields as dicts -) -> tuple[tuple[ComplexityTier, str], ...]: +_DECODED_JSON: Final = TypeAdapter(object) + + +def _validated_labeled_tiers(tier_labels: object) -> tuple[tuple[ComplexityTier, str], ...]: """Validate tier labels once for both prompt-preview transports.""" try: - return ComplexityRouterConfig(tier_labels=tier_labels).labeled_tiers() + return ComplexityRouterConfig.model_validate({"tier_labels": tier_labels}).labeled_tiers() except (TypeError, ValidationError) as e: raise ProxyException( message=f"tier_labels must be a JSON object of tier name to display name: {e}", @@ -3237,7 +3238,7 @@ def _labeled_tiers_from_query(tier_labels: str | None) -> tuple[tuple[Complexity if not tier_labels: return None try: - parsed: Final = json.loads(tier_labels) + parsed: Final = _DECODED_JSON.validate_python(json.loads(tier_labels)) except JSONDecodeError as e: raise ProxyException( message=f"tier_labels must be a JSON object of tier name to display name: {e}", diff --git a/litellm/proxy/management_endpoints/router_settings_endpoints.py b/litellm/proxy/management_endpoints/router_settings_endpoints.py index 64566221d4d..1f4ca442d20 100644 --- a/litellm/proxy/management_endpoints/router_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/router_settings_endpoints.py @@ -71,10 +71,11 @@ def _get_routing_strategies_from_router_class() -> list[str]: # Get the routing_strategy parameter routing_strategy_param: Final = sig.parameters.get("routing_strategy") + routing_strategy_annotation: Final[object] = routing_strategy_param.annotation if routing_strategy_param else None - if routing_strategy_param and routing_strategy_param.annotation: + if routing_strategy_param and routing_strategy_annotation: # Extract Literal values using get_args - literal_values: Final = get_args(routing_strategy_param.annotation) + literal_values: Final = get_args(routing_strategy_annotation) if literal_values: return list(literal_values) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index c7adae2ce8d..361778ab2ea 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -479,6 +479,14 @@ def _team_db(prisma_client: PrismaClient | None) -> "TableActions[prisma_models. return TeamRepository(prisma_client).table +class _ExecuteRaw(Protocol): + async def __call__(self, query: str, *args: object) -> int: ... + + +def _execute_raw(prisma_client: PrismaClient) -> _ExecuteRaw: + return prisma_client.db.execute_raw + + def _team_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_TeamTable]": return cast( # cast-ok: generated actions type Json columns as str; TableActions widens inputs to Mapping "TableActions[prisma_models.LiteLLM_TeamTable]", tx.litellm_teamtable @@ -3092,7 +3100,7 @@ async def _process_team_members( # Resolve allowed_models: explicit request value, or fall back to team's default_team_member_models member_allowed_models = data.allowed_models - team_default_member_models: Final = getattr(complete_team_data, "default_team_member_models", None) + team_default_member_models: Final = complete_team_data.default_team_member_models if member_allowed_models is None and team_default_member_models: member_allowed_models = team_default_member_models @@ -6572,7 +6580,7 @@ async def append_team_models( # When the team currently has models=[] (unrestricted access), the # CASE expression inserts the 'all-proxy-models' sentinel first. models_to_add: Final = list(data.models) - await prisma_client.db.execute_raw( + await _execute_raw(prisma_client)( 'UPDATE "LiteLLM_TeamTable" ' "SET models = (" " SELECT ARRAY(SELECT DISTINCT unnest(" diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index fd5eb672811..02494de8c87 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -278,6 +278,12 @@ class _CustomSsoCall(Protocol): async def __call__(self, sso_response: object) -> SSOUserDefinedValues | None: ... +def _configured_custom_sso() -> _CustomSsoCall | None: + from litellm.proxy.proxy_server import user_custom_sso + + return user_custom_sso + + class _ServicePrincipalAssignment(Protocol): def get(self, key: str) -> str: ... @@ -1051,6 +1057,11 @@ class _UserinfoFallbackCapableSSO(Protocol): ) -> CustomOpenID | None: ... +class _GenericSSOClient(_UserinfoFallbackCapableSSO, Protocol): + @property + def refresh_token(self) -> str | None: ... + + async def _verify_generic_sso_or_fall_back_to_id_token( generic_sso: _UserinfoFallbackCapableSSO, request: Request, @@ -1578,7 +1589,7 @@ async def get_generic_sso_response( team_mappings: Final = await _setup_team_mappings() generic_include_token_claims: Final = os.getenv("GENERIC_INCLUDE_TOKEN_CLAIMS", "false").lower() == "true" - def response_convertor(response: Mapping[str, object], httpx_session: object): + def response_convertor(response: Mapping[str, object], httpx_session: object) -> CustomOpenID: nonlocal received_response # return for user debugging response_id_token: Final = response.get("id_token") response_access_token: Final = response.get("access_token") @@ -1613,7 +1624,7 @@ async def get_generic_sso_response( discovery_document=discovery, response_convertor=response_convertor, ) - generic_sso: Final = SSOProvider( + generic_sso: Final[_GenericSSOClient] = SSOProvider( client_id=generic_client_id, client_secret=generic_client_secret, redirect_uri=redirect_url, @@ -1712,9 +1723,7 @@ async def get_generic_sso_response( access_token_str: str | None = combined_response.get("access_token") else: result = await _verify_generic_sso_or_fall_back_to_id_token( - generic_sso=cast( # cast-ok: fastapi-sso builds the provider class dynamically via create_provider - _UserinfoFallbackCapableSSO, generic_sso - ), + generic_sso=generic_sso, request=request, token_exchange_params=token_exchange_params, additional_headers=additional_generic_sso_headers_dict, @@ -2350,9 +2359,7 @@ async def _build_cli_sso_user_defined_values( result: OpenID | dict, parsed_openid_result: ParsedOpenIDResult, ) -> SSOUserDefinedValues | None: - from litellm.proxy.proxy_server import user_custom_sso - - custom_sso_handler: Final[_CustomSsoCall | None] = user_custom_sso + custom_sso_handler: Final = _configured_custom_sso() user_id: Final = parsed_openid_result.get("user_id") if custom_sso_handler is not None: if inspect.iscoroutinefunction(custom_sso_handler): @@ -2438,10 +2445,8 @@ async def _complete_cli_sso_callback_session( ): from fastapi.responses import HTMLResponse - from litellm.proxy.proxy_server import user_custom_sso - user_id: Final = parsed_openid_result.get("user_id") - if user_custom_sso is None: + if _configured_custom_sso() is None: _require_sso_user_id(user_id) user_email: Final = parsed_openid_result.get("user_email") user_info: Final = await get_user_info_from_db( @@ -3807,7 +3812,6 @@ class SSOAuthenticationHandler: proxy_logging_obj, redis_usage_cache, user_api_key_cache, - user_custom_sso, ) from litellm.proxy.utils import get_prisma_client_or_throw from litellm.types.proxy.ui_sso import ReturnedUITokenObject @@ -3832,7 +3836,7 @@ class SSOAuthenticationHandler: # But if it is, we want their models preferences user_defined_values: SSOUserDefinedValues | None = None - custom_sso_handler: Final[_CustomSsoCall | None] = user_custom_sso + custom_sso_handler: Final = _configured_custom_sso() if custom_sso_handler is not None: if inspect.iscoroutinefunction(custom_sso_handler): user_defined_values = await custom_sso_handler(result) diff --git a/litellm/proxy/management_helpers/object_permission_utils.py b/litellm/proxy/management_helpers/object_permission_utils.py index 3468396c6f6..920970427ba 100644 --- a/litellm/proxy/management_helpers/object_permission_utils.py +++ b/litellm/proxy/management_helpers/object_permission_utils.py @@ -310,7 +310,7 @@ async def _resolve_mcp_server_identifiers_to_ids( identifiers=identifiers, prisma_client=prisma_client, ): - server_id = getattr(server, "server_id", None) + server_id = server.server_id if not server_id: continue for identifier in identifiers: diff --git a/litellm/proxy/memory/memory_endpoints.py b/litellm/proxy/memory/memory_endpoints.py index 34c80bcdce3..4457df460d4 100644 --- a/litellm/proxy/memory/memory_endpoints.py +++ b/litellm/proxy/memory/memory_endpoints.py @@ -181,8 +181,8 @@ async def _assert_write_access( """ if _is_admin(user_api_key_dict): return - row_user_id: Final = getattr(row, "user_id", None) - row_team_id: Final = getattr(row, "team_id", None) + row_user_id: Final = row.user_id + row_team_id: Final = row.team_id # Personal ownership. if row_user_id and row_user_id == user_api_key_dict.user_id: diff --git a/litellm/proxy/moyai_endpoints.py b/litellm/proxy/moyai_endpoints.py index c4d8ad139f7..45f4502ec4e 100644 --- a/litellm/proxy/moyai_endpoints.py +++ b/litellm/proxy/moyai_endpoints.py @@ -20,9 +20,10 @@ from typing import TYPE_CHECKING, Annotated, Final from urllib.parse import urlencode, urlparse from fastapi import APIRouter, Depends, HTTPException, Request, status -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter from litellm._internal_context import with_service_target +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( @@ -114,6 +115,9 @@ def _sign_connect_code(master_key: str, moyai_url: str, user_id: str | None) -> return f"{_b64url(payload)}.{_b64url(signature)}" +_DECODED_CONNECT_PAYLOAD: Final = TypeAdapter(object) + + def _decode_connect_code(master_key: str, code: str) -> _MoyaiConnectCode: try: payload_b64, signature_b64 = code.split(".", 1) @@ -125,18 +129,23 @@ def _decode_connect_code(master_key: str, code: str) -> _MoyaiConnectCode: if not hmac.compare_digest(signature, expected): raise HTTPException(status_code=400, detail="Invalid Moyai connect code") try: - payload: Final = json.loads(payload_raw) + payload: Final = as_str_mapping(_DECODED_CONNECT_PAYLOAD.validate_python(json.loads(payload_raw))) except (ValueError, TypeError): raise HTTPException(status_code=400, detail="Invalid Moyai connect code") - if not isinstance(payload, dict) or not isinstance(payload.get("exp"), int) or payload["exp"] < int(time.time()): + if payload is None: raise HTTPException(status_code=400, detail="Invalid Moyai connect code") - if not isinstance(payload.get("moyai_origin"), str) or not isinstance(payload.get("nonce"), str): + exp: Final = payload.get("exp") + if not isinstance(exp, int) or exp < int(time.time()): + raise HTTPException(status_code=400, detail="Invalid Moyai connect code") + moyai_origin: Final = payload.get("moyai_origin") + nonce: Final = payload.get("nonce") + if not isinstance(moyai_origin, str) or not isinstance(nonce, str): raise HTTPException(status_code=400, detail="Invalid Moyai connect code") user_id: Final = payload.get("user_id") return _MoyaiConnectCode( - moyai_origin=payload["moyai_origin"], - nonce=payload["nonce"], - exp=payload["exp"], + moyai_origin=moyai_origin, + nonce=nonce, + exp=exp, user_id=user_id if isinstance(user_id, str) else None, ) diff --git a/litellm/proxy/openai_files_endpoints/batch_file_validation.py b/litellm/proxy/openai_files_endpoints/batch_file_validation.py index 1aa24eed4cd..5d596aece32 100644 --- a/litellm/proxy/openai_files_endpoints/batch_file_validation.py +++ b/litellm/proxy/openai_files_endpoints/batch_file_validation.py @@ -4,12 +4,15 @@ from dataclasses import dataclass from itertools import chain from typing import BinaryIO, Final, NoReturn +from pydantic import TypeAdapter from typing_extensions import assert_never +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import ProxyException from litellm.proxy.openai_files_endpoints.file_usage_caps import FileUsageLimit, describe_limit_source _MB: Final = 1024 * 1024 +_DECODED_LINE: Final = TypeAdapter(object) @dataclass(frozen=True, slots=True) @@ -95,10 +98,10 @@ def _iter_lines(file_source: bytes | BinaryIO) -> Iterator[bytes]: def _check_line(line_number: int, raw_line: bytes, line_shape: BatchLineShape) -> BatchFileValidationFailure | None: try: - parsed: Final = json.loads(raw_line) + parsed: Final = as_str_mapping(_DECODED_LINE.validate_python(json.loads(raw_line))) except (ValueError, RecursionError): return BatchFileInvalidJsonLine(line_number=line_number) - if not isinstance(parsed, dict): + if parsed is None: return BatchFileLineNotObject(line_number=line_number) missing: Final = next((key for key in line_shape.required_keys if key not in parsed), None) if missing is None: diff --git a/litellm/proxy/openai_files_endpoints/storage_backend_service.py b/litellm/proxy/openai_files_endpoints/storage_backend_service.py index afc1e1c96fa..1f507982e65 100644 --- a/litellm/proxy/openai_files_endpoints/storage_backend_service.py +++ b/litellm/proxy/openai_files_endpoints/storage_backend_service.py @@ -8,10 +8,11 @@ storage backends (e.g., Azure Blob Storage) and managing associated metadata. import base64 import time from collections.abc import Sequence -from typing import Any, Final, cast +from typing import Final from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid as uuid_module +from litellm.litellm_core_utils.core_helpers import require_attributes from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params from litellm.llms.base_llm.files.storage_backend import BaseFileStorageBackend from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend @@ -19,6 +20,7 @@ from litellm.llms.base_llm.files.transformation import BaseFileEndpoints from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.types.llms.openai import OpenAIFileObject, OpenAIFilesPurpose +from litellm.types.passthrough_endpoints.managed_id_rewriter import ManagedFileIdWriter from litellm.types.utils import ExtractedFileData, SpecialEnums @@ -232,11 +234,10 @@ class StorageBackendFileService: proxy_logging_obj: Proxy logging object user_api_key_dict: User API key authentication data """ - managed_files_obj = proxy_logging_obj.get_proxy_hook("managed_files") + managed_files_obj: Final = proxy_logging_obj.get_proxy_hook("managed_files") if not managed_files_obj or not isinstance(managed_files_obj, BaseFileEndpoints): verbose_proxy_logger.warning("Managed files hook not available, skipping managed files storage") return - managed_files_obj = cast(Any, managed_files_obj) # Create model mappings using storage URL model_mappings: Final = {model_name: storage_url for model_name in target_model_names} @@ -260,7 +261,10 @@ class StorageBackendFileService: ) # Store in managed files - await managed_files_obj.store_unified_file_id( + managed_files_writer: Final = require_attributes( + managed_files_obj, ManagedFileIdWriter, "store_unified_file_id" + ) + await managed_files_writer.store_unified_file_id( file_id=base64_unified_file_id, file_object=file_object, litellm_parent_otel_span=user_api_key_dict.parent_otel_span, diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 16fbd1de376..9eef4f0ef9d 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -1305,7 +1305,7 @@ async def handle_bedrock_count_tokens( verbose_proxy_logger.error("BedrockError in handle_bedrock_count_tokens: %s", e) from litellm.litellm_core_utils.llm_response_utils.get_headers import get_response_headers - provider_headers: Final = getattr(getattr(e, "response", None), "headers", None) + provider_headers: Final = e.response.headers raise HTTPException( status_code=e.status_code, detail={"error": e.message}, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/comprehend_medical_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/comprehend_medical_passthrough_logging_handler.py index 7527c504c59..3fb4fbd555d 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/comprehend_medical_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/comprehend_medical_passthrough_logging_handler.py @@ -27,11 +27,15 @@ COMPREHEND_MEDICAL_COST_PER_UNIT_USD: Final[Mapping[str, float]] = MappingProxyT COMPREHEND_MEDICAL_SUPPORTED_OPERATIONS: Final = frozenset(COMPREHEND_MEDICAL_COST_PER_UNIT_USD) +def _operation_from_headers(headers: Mapping[str, str]) -> str: + target: Final = headers.get("x-amz-target", "") + return target.split(".")[-1] + + class ComprehendMedicalPassthroughLoggingHandler: @staticmethod def _operation_from_response(httpx_response: httpx.Response) -> str: - target: Final[str] = httpx_response.request.headers.get("x-amz-target", "") - return target.split(".")[-1] + return _operation_from_headers(httpx_response.request.headers) @staticmethod def get_cost_for_operation(operation: str, text: str) -> float: diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py index e277655f1e1..2afc1d17bc2 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py @@ -36,6 +36,7 @@ from litellm.types.passthrough_endpoints.tinyfish import ( from litellm.types.utils import StandardPassThroughResponseObject _RUN_ADAPTER: Final = TypeAdapter(TinyfishRun) +_DECODED_JSON: Final = TypeAdapter(object) _EMPTY_KWARGS: Final[Mapping[str, object]] = MappingProxyType({}) @@ -323,7 +324,7 @@ class TinyFishPassthroughLoggingHandler: "TinyFish passthrough: GET /v1/runs/%s returned %s", safe_run_id, response.status_code ) return None - payload: Final[object] = response.json() # any-ok: httpx Response.json() -> Any + payload: Final = _DECODED_JSON.validate_python(response.json()) return _parse_run(payload) except Exception as e: # noqa: BLE001 # billing/logging must never break the relayed request verbose_proxy_logger.warning("[Non blocking logging error] TinyFish run fetch failed: %s", e) @@ -413,7 +414,7 @@ def _run_id_from_sse_chunks(all_chunks: Sequence[str]) -> str | None: if not line.startswith("data:"): continue try: - event_payload: object = json.loads(line[5:].strip()) # any-ok: json.loads -> Any + event_payload: object = _DECODED_JSON.validate_python(json.loads(line[5:].strip())) except json.JSONDecodeError: continue event = _parse_run(event_payload) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/transcribe_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/transcribe_passthrough_logging_handler.py index 79f6dee90f7..5b45316a44c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/transcribe_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/transcribe_passthrough_logging_handler.py @@ -492,6 +492,10 @@ def s3_media_url(media_uri: str, aws_region_name: str) -> str | None: return f"https://{bucket}.s3.{aws_region_name}.{dns_suffix}/{quote(key)}" +def _header_or_default(headers: Mapping[str, str], name: str, default: str) -> str: + return headers.get(name, default) + + def media_predates_job(headers: Mapping[str, str], job_created_at: float) -> bool: try: modified_at: Final = parsedate_to_datetime(headers["last-modified"]).timestamp() @@ -501,7 +505,7 @@ def media_predates_job(headers: Mapping[str, str], job_created_at: float) -> boo async def write_media_within_limit(response: httpx.Response, media_file: IO[bytes], max_bytes: int) -> bool: - if int(response.headers.get("content-length", "0")) > max_bytes: + if int(_header_or_default(response.headers, "content-length", "0")) > max_bytes: return False async for chunk in response.aiter_bytes(): _ = media_file.write(chunk) @@ -591,8 +595,7 @@ class TranscribePassthroughLoggingHandler: @staticmethod def _operation_from_response(httpx_response: httpx.Response) -> str: - headers: Final[Mapping[str, str]] = httpx_response.request.headers - target: Final[str] = headers.get("x-amz-target", "") + target: Final = _header_or_default(httpx_response.request.headers, "x-amz-target", "") return target.split(".")[-1] @staticmethod diff --git a/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py b/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py index 59b8d40577a..e70d583a027 100644 --- a/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py +++ b/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py @@ -7,6 +7,7 @@ records the reported values without recomputing them. """ import math +from collections.abc import Mapping from dataclasses import dataclass from typing import Final @@ -89,7 +90,7 @@ def _parse_total_tokens(raw_value: str | None) -> int | None: return total_tokens -def parse_upstream_reported_usage(headers: httpx.Headers) -> UpstreamReportedUsage | None: +def parse_upstream_reported_usage(headers: Mapping[str, str]) -> UpstreamReportedUsage | None: """Read the reported totals off an upstream pass-through response. Returns ``None`` when neither header is present, which is the normal case diff --git a/litellm/proxy/policy_engine/attachment_registry.py b/litellm/proxy/policy_engine/attachment_registry.py index c751091ff9b..15e7b7e8482 100644 --- a/litellm/proxy/policy_engine/attachment_registry.py +++ b/litellm/proxy/policy_engine/attachment_registry.py @@ -5,10 +5,10 @@ Attachments define WHERE policies apply, separate from the policy definitions. This allows the same policy to be attached to multiple scopes. """ -from collections.abc import Callable +from collections.abc import Callable, Mapping from datetime import datetime, timezone from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, TypedDict +from typing import TYPE_CHECKING, Final, TypedDict from litellm._logging import verbose_proxy_logger from litellm.repositories.table_repositories import PolicyAttachmentRepository @@ -102,7 +102,7 @@ class AttachmentRegistry: self._initialized = True verbose_proxy_logger.info("Loaded %s policy attachments", len(self._attachments)) - def _parse_attachment(self, attachment_data: dict[str, Any]) -> PolicyAttachment: + def _parse_attachment(self, attachment_data: Mapping[str, object]) -> PolicyAttachment: """ Parse an attachment from raw configuration data. @@ -112,15 +112,17 @@ class AttachmentRegistry: Returns: Parsed PolicyAttachment object """ - return PolicyAttachment( - policy=attachment_data.get("policy", ""), - scope=attachment_data.get("scope"), - teams=attachment_data.get("teams"), - keys=attachment_data.get("keys"), - models=attachment_data.get("models"), - tags=attachment_data.get("tags"), - priority=attachment_data.get("priority"), - default=attachment_data.get("default", False), + return PolicyAttachment.model_validate( + { + "policy": attachment_data.get("policy", ""), + "scope": attachment_data.get("scope"), + "teams": attachment_data.get("teams"), + "keys": attachment_data.get("keys"), + "models": attachment_data.get("models"), + "tags": attachment_data.get("tags"), + "priority": attachment_data.get("priority"), + "default": attachment_data.get("default", False), + } ) def get_attached_policies( diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 1b1b2302236..4ad29bd3478 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -11,7 +11,7 @@ from collections.abc import Callable, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, TypeVar, cast -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -52,6 +52,8 @@ try: except ImportError: HTTPException = None +_LEGACY_HOOK_REPLACEMENT: Final = TypeAdapter(object) + class UndeliverableStreamRewrite(Exception): def __init__(self, guardrail_name: str, reason: str) -> None: @@ -232,10 +234,12 @@ class _LegacyHookStreamAdapter(CustomGuardrail): logging_obj: "LiteLLMLoggingObj | None" = None, ) -> GenericGuardrailAPIInputs: hooked: Final = self.endpoint_translation.post_call_hook_response(request_data.get("response")) - replacement: Final = await self.inner.async_post_call_success_hook( - data=request_data, - user_api_key_dict=self.user_api_key_dict, - response=hooked, + replacement: Final = _LEGACY_HOOK_REPLACEMENT.validate_python( + await self.inner.async_post_call_success_hook( + data=request_data, + user_api_key_dict=self.user_api_key_dict, + response=hooked, + ) ) rewrite: Final = hooked if replacement is None else replacement if rewrite is None: diff --git a/litellm/proxy/policy_engine/policy_endpoints.py b/litellm/proxy/policy_engine/policy_endpoints.py index f4b38bea14e..68ac71c567b 100644 --- a/litellm/proxy/policy_engine/policy_endpoints.py +++ b/litellm/proxy/policy_engine/policy_endpoints.py @@ -623,7 +623,7 @@ async def test_pipeline( ``` """ try: - validated_pipeline: Final = GuardrailPipeline(**request.pipeline) + validated_pipeline: Final = GuardrailPipeline.model_validate(request.pipeline) except Exception as e: raise HTTPException(status_code=400, detail=f"Invalid pipeline: {e}") diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ade719ae0ed..44f43808dc8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -24,6 +24,7 @@ from collections.abc import ( Awaitable, Callable, Collection, + Container, Iterable, Mapping, MutableMapping, @@ -360,6 +361,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_parent_otel_span_from_kwargs, iter_items, normalize_drop_params, + require_attributes, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -474,6 +476,7 @@ from litellm.proxy.common_utils.codex_model_catalog import codex_model_list_body from litellm.proxy.common_utils.config_includes import resolve_include_file_path, resolve_includes from litellm.proxy.common_utils.config_sync_pubsub import ConfigSyncSubscriber from litellm.proxy.common_utils.credential_hydration import decrypted_or_stored +from litellm.proxy.common_utils.custom_openapi_spec import JsonObject from litellm.proxy.common_utils.debug_utils import init_verbose_loggers from litellm.proxy.common_utils.debug_utils import router as debugging_endpoints_router from litellm.proxy.common_utils.discoverable_model_filter import ( @@ -1140,7 +1143,7 @@ def _redact_worker_config_for_logging(worker_config: str | dict[str, JsonValue] return None if isinstance(worker_config, dict): return _redact_secret_values_in_obj(worker_config) - parsed: Final = safe_json_loads(worker_config, default=None) + parsed: Final[JsonValue] = safe_json_loads(worker_config, default=None) if isinstance(parsed, dict): return safe_dumps(_redact_secret_values_in_obj(parsed)) return worker_config @@ -1466,11 +1469,13 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState ) _module_path, _func_name = _hook_spec.rsplit(":", 1) _module = importlib.import_module(_module_path) - _hook_fn = getattr(_module, _func_name) + _hook_fn = _UNTYPED_VALUE.validate_python(getattr(_module, _func_name)) if inspect.iscoroutinefunction(_hook_fn): await _hook_fn() - else: + elif callable(_hook_fn): _hook_fn() + else: + raise TypeError(f"'{type(_hook_fn).__name__}' object is not callable") verbose_proxy_logger.info("Worker startup hook '%s' executed successfully", _hook_spec) except Exception as e: verbose_proxy_logger.error("Worker startup hook '%s' failed: %s", _hook_spec, e) @@ -1918,15 +1923,22 @@ def _strip_operation_id_method_suffix(operation_id: str) -> str: return operation_id +class _SupportsValues(Protocol): + def values(self) -> Iterable[JsonValue]: ... + + +def _openapi_path_items(openapi_schema: JsonObject) -> Iterable[JsonObject]: + paths: Final = require_attributes(openapi_schema.get("paths", {}), _SupportsValues, "values") + return (path_item for path_item in paths.values() if isinstance(path_item, dict)) + + def ensure_unique_openapi_operation_ids( - openapi_schema: dict[str, Any], + openapi_schema: JsonObject, reserved_operation_ids: set[str] | None = None, -) -> dict[str, Any]: +) -> JsonObject: operation_entries: Final = [] operation_id_counts: Final[dict[str, int]] = {} - for path_item in openapi_schema.get("paths", {}).values(): - if not isinstance(path_item, dict): - continue + for path_item in _openapi_path_items(openapi_schema): for method, operation in path_item.items(): if method not in _OPENAPI_HTTP_METHODS or not isinstance(operation, dict): continue @@ -2003,6 +2015,19 @@ from starlette.routing import BaseRoute from litellm.proxy.common_utils.openapi_route import OpenAPISchema, serve_openapi_off_the_event_loop +class _OpenAPIQueryParam(Protocol): + @property + def name(self) -> object: ... + + @property + def required(self) -> object: ... + + +def _query_param_name_and_required(param: object) -> tuple[object, object]: + fields: Final = require_attributes(param, _OpenAPIQueryParam, "name", "required") + return fields.name, fields.required + + def _inject_websocket_stubs_into_openapi_schema(openapi_schema: dict, websocket_routes: list) -> dict: """ Add a synthetic GET stub for each WebSocket route so it appears in Swagger UI. @@ -2019,14 +2044,15 @@ def _inject_websocket_stubs_into_openapi_schema(openapi_schema: dict, websocket_ try: if hasattr(route, "dependant") and route.dependant is not None: # Handle both FastAPI <0.120 and >=0.120 - query_params = getattr(route.dependant, "query_params", []) + query_params = _UNTYPED_VALUE.validate_python(getattr(route.dependant, "query_params", [])) if query_params: - for param in query_params: + for param in iter_items(query_params): + name, required = _query_param_name_and_required(param) parameters.append( { - "name": param.name, + "name": name, "in": "query", - "required": param.required, + "required": required, "schema": {"type": "string"}, } ) @@ -5576,6 +5602,13 @@ def _log_ignored_cost_map_copy(model_id: str, fields: tuple[str, ...]) -> None: _DB_CONFIG_CALLBACK_EVENT_TYPES: Final[Mapping[str, tuple[Literal["success", "failure"], ...]]] = MappingProxyType( {"success_callback": ("success",), "failure_callback": ("failure",), "callbacks": ("success", "failure")} ) +_DB_CONFIG_CALLBACK_LISTS: Final[Mapping[str, Callable[[], Container[object]]]] = MappingProxyType( + { + "success_callback": lambda: litellm.success_callback, + "failure_callback": lambda: litellm.failure_callback, + "callbacks": lambda: litellm.callbacks, + } +) _CALLBACK_LIST_NAMES: Final = ( "callbacks", "success_callback", @@ -5586,7 +5619,9 @@ _CALLBACK_LIST_NAMES: Final = ( def _callback_list_entries(list_name: str) -> tuple[tuple[str, object], ...]: - return tuple((list_name, entry) for entry in getattr(litellm, list_name)) + return tuple( + (list_name, entry) for entry in iter_items(_UNTYPED_VALUE.validate_python(getattr(litellm, list_name))) + ) def _registered_callback_entries() -> tuple[tuple[str, object], ...]: @@ -5615,7 +5650,7 @@ def _restore_displaced_callback(list_name: str, entry: object) -> None: def _unregister_db_callback(registration: _DbCallbackRegistration) -> None: for list_name, entry in registration.added: litellm.logging_callback_manager.remove_callback_from_list_by_object( - getattr(litellm, list_name), entry, require_self=False + _UNTYPED_VALUE.validate_python(getattr(litellm, list_name)), entry, require_self=False ) for list_name, entry in registration.displaced: _restore_displaced_callback(list_name, entry) @@ -7824,7 +7859,7 @@ class ProxyConfig: self, callback: str, event_types: list[Literal["success", "failure"]], - existing_callbacks: list, + existing_callbacks: Container[object], ) -> None: """ Helper method to add a single callback to litellm for specified event types. @@ -7867,7 +7902,7 @@ class ProxyConfig: self._add_callback_from_db_to_in_memory_litellm_callbacks( callback=callback, event_types=list(_DB_CONFIG_CALLBACK_EVENT_TYPES[setting_key]), - existing_callbacks=getattr(litellm, setting_key), + existing_callbacks=_DB_CONFIG_CALLBACK_LISTS[setting_key](), ) after: Final = _registered_callback_entries() return _DbCallbackRegistration( @@ -8362,8 +8397,12 @@ class ProxyConfig: environment_read: Final = next(self._environment_read_numbers) sections: Final = tuple(self._settings_stores) responses: Final = await asyncio.gather(*(get_config_param(prisma_client, section) for section in sections)) + response: _ConfigParamValueRow | None for section, response in zip(sections, responses): - if response is None or (param_value := getattr(response, "param_value", None)) is None: + if response is None: + continue + param_value: JsonValue = getattr(response, "param_value", None) + if param_value is None: continue verbose_proxy_logger.debug( @@ -8390,7 +8429,10 @@ class ProxyConfig: self, prisma_client: PrismaClient, row: object, environment_before_db: Mapping[str, str], read: int ) -> None: current: Final = await self._environment_row_not_behind_the_applied_one(prisma_client, row) - if current is None or (param_value := getattr(current, "param_value", None)) is None: + if current is None: + return + param_value: Final[object] = getattr(current, "param_value", None) + if param_value is None: return self._apply_db_environment_variables(param_value, _config_row_revision(current), environment_before_db, read) @@ -15997,7 +16039,23 @@ async def _get_caller_byok_team_scope( return key_team_scope | set(user_row.teams or []) -def _byok_row_outside_caller_teams(model_info_dict: dict[str, JsonValue], allowed_team_ids: set[str] | None) -> bool: +class _SupportsGet(Protocol): + def get(self, key: str, default: object = None, /) -> object: ... + + +def _with_get(value: object) -> _SupportsGet: + return require_attributes(value, _SupportsGet, "get") + + +class _SupportsLower(Protocol): + def lower(self) -> object: ... + + +def _lowered(value: object) -> object: + return require_attributes(value, _SupportsLower, "lower").lower() + + +def _byok_row_outside_caller_teams(model_info_dict: object, allowed_team_ids: set[str] | None) -> bool: """Whether a team BYOK row belongs to a team the caller is not a member of. `team_id` is only set on team BYOK rows; non-team rows fall through @@ -16005,7 +16063,7 @@ def _byok_row_outside_caller_teams(model_info_dict: dict[str, JsonValue], allowe """ if allowed_team_ids is None: return False - team_id: Final = model_info_dict.get("team_id") + team_id: Final = _with_get(model_info_dict).get("team_id") if team_id is None: return False return team_id not in allowed_team_ids @@ -16106,7 +16164,7 @@ async def _fetch_db_models_for_search( async def _apply_search_filter_to_models( - all_models: list[dict[str, Any]], + all_models: list[dict[str, object]], search: str, prisma_client: PrismaClient | None, proxy_config: ProxyConfig, @@ -16158,7 +16216,7 @@ async def _apply_search_filter_to_models( prisma_client=prisma_client, ) - def _is_byok_outside_caller_teams(model_info_dict: dict[str, JsonValue]) -> bool: + def _is_byok_outside_caller_teams(model_info_dict: object) -> bool: return _byok_row_outside_caller_teams(model_info_dict, allowed_team_ids) def _model_matches_search(m: dict[str, Any]) -> bool: @@ -16195,7 +16253,7 @@ async def _apply_search_filter_to_models( db_model_ids_in_router: Final = set() for m in filtered_router_models: - model_info = m.get("model_info", {}) + model_info = _with_get(m.get("model_info", {})) is_db_model = model_info.get("db_model", False) model_id = model_info.get("id") @@ -16306,7 +16364,7 @@ def _sort_models( reverse: Final = sort_order.lower() == "desc" def get_sort_key(model: dict[str, Any]) -> Any: - model_info: Final = model.get("model_info", {}) + model_info: Final = _with_get(_UNTYPED_VALUE.validate_python(model.get("model_info", {}))) if sort_by == "model_name": # Team BYOK models persist an internal `model_name` (e.g. @@ -16318,7 +16376,7 @@ def _sort_models( team_public_model_name: Final = model_info.get("team_public_model_name") if team_public_model_name: return str(team_public_model_name).lower() - return model.get("model_name", "").lower() + return _lowered(_UNTYPED_VALUE.validate_python(model.get("model_name", ""))) elif sort_by == "created_at": created_at: Final = model_info.get("created_at") @@ -16344,8 +16402,8 @@ def _sort_models( return normalized_dt elif sort_by == "costs": - input_cost: Final = model_info.get("input_cost_per_token", 0) or 0 - output_cost: Final = model_info.get("output_cost_per_token", 0) or 0 + input_cost: Final = model.get("model_info", {}).get("input_cost_per_token", 0) or 0 + output_cost: Final = model.get("model_info", {}).get("output_cost_per_token", 0) or 0 total_cost: Final = input_cost + output_cost # Put 0 or None costs at the end for asc, at the start for desc if total_cost == 0: @@ -16580,7 +16638,7 @@ async def _authorize_team_id_query( async def _filter_models_by_team_id( - all_models: list[dict[str, Any]], + all_models: list[dict[str, object]], team_id: str, prisma_client: PrismaClient, llm_router: Router, @@ -16624,7 +16682,7 @@ async def _filter_models_by_team_id( # every public model the admin can call. filtered_models: Final = [] for _model in all_models: - model_info = _model.get("model_info", {}) + model_info = _with_get(_model.get("model_info", {})) model_id = model_info.get("id", None) # BYOK rows owned by this team are always accessible to it, even if diff --git a/litellm/proxy/public_endpoints/public_v1/model_hub.py b/litellm/proxy/public_endpoints/public_v1/model_hub.py index cd634e490b6..e58149fba0b 100644 --- a/litellm/proxy/public_endpoints/public_v1/model_hub.py +++ b/litellm/proxy/public_endpoints/public_v1/model_hub.py @@ -107,10 +107,14 @@ def _features(row: ModelGroupInfoProxy) -> tuple[str, ...]: The hub's feature control has always been a multi-select over the `supports_*` flags. One boolean filter per flag would AND them, which is the opposite of what it does. """ + return _enabled_features(row.model_dump()) + + +def _enabled_features(fields: Mapping[str, object]) -> tuple[str, ...]: return tuple( sorted( name.removeprefix(FEATURE_PREFIX) - for name, value in row.model_dump().items() + for name, value in fields.items() if name.startswith(FEATURE_PREFIX) and value is True ) ) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 803df9f83ca..0b91e57f01e 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -728,7 +728,9 @@ async def _get_team_member_budget_counter( membership_cache_key: Final = team_membership_reservation_cache_key( user_id=valid_token.user_id, team_id=team_object.team_id ) - cached_team_membership: Final = await user_api_key_cache.async_get_cache(key=membership_cache_key) + cached_team_membership: Final = _CACHED_VALUE.validate_python( + await user_api_key_cache.async_get_cache(key=membership_cache_key) + ) team_membership: LiteLLM_TeamMembership | None = None if isinstance(cached_team_membership, LiteLLM_TeamMembership): team_membership = cached_team_membership diff --git a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py index ea543aeae6e..4abc79febe1 100644 --- a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py +++ b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py @@ -21,6 +21,8 @@ from itertools import chain, groupby from types import MappingProxyType from typing import TYPE_CHECKING, Final, Protocol +from pydantic import TypeAdapter + from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.constants import ( @@ -135,6 +137,9 @@ def _public_model_name(row: object, model_info: Mapping[str, object]) -> str: return str(getattr(row, "model_name", "") or "") +_DECODED_JSON: Final = TypeAdapter(object) + + def _decode_model_info(raw: object) -> "Mapping[str, object] | None": """A deployment's model_info as a mapping, decoding a JSON string, else None. @@ -143,7 +148,7 @@ def _decode_model_info(raw: object) -> "Mapping[str, object] | None": """ if isinstance(raw, str): try: - decoded: Final[object] = json.loads(raw) + decoded: Final = _DECODED_JSON.validate_python(json.loads(raw)) except (TypeError, ValueError): return None return decoded if isinstance(decoded, dict) else None diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 84278f4619d..3c65fc0bf06 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -3045,7 +3045,7 @@ async def ui_view_spend_logs( ) sql_params.extend([page_size, skip]) - data: Final = await prisma_client.db.query_raw(sql_query, *sql_params) + data: Final[Sequence[Mapping[str, object]]] = await _query_raw(prisma_client, sql_query, *sql_params) if request_id is not None and not is_v2 and not is_admin_view: await _assert_user_owns_fetched_spend_rows( @@ -3064,7 +3064,7 @@ async def ui_view_spend_logs( return await _build_ui_spend_logs_response( prisma_client, - data, + list(data), total_records, page, page_size, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index efc83132568..6cf274884c7 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -19,6 +19,7 @@ from collections.abc import ( Awaitable, Callable, Coroutine, + Iterable, Iterator, Mapping, Sequence, @@ -51,7 +52,7 @@ from typing import ( import anyio import httpx -from typing_extensions import Never, ReadOnly, TypedDict +from typing_extensions import Never, NotRequired, ReadOnly, TypedDict from litellm import _custom_logger_compatible_callbacks_literal from litellm.constants import ( @@ -323,10 +324,22 @@ class _RelTuplesRow(TypedDict): reltuples: ReadOnly[int] +class _TeamMemberRow(TypedDict): + role: ReadOnly[Literal["admin", "user"]] + user_id: ReadOnly[NotRequired[str | None]] + user_email: ReadOnly[NotRequired[str | None]] + + +class _TeamMembershipRow(TypedDict): + user_id: ReadOnly[str | None] + team_members_with_roles: ReadOnly[Iterable[_TeamMemberRow] | None] + + _VIEW_SETUP_POLL_INTERVAL_SECONDS: Final = 5.0 _VIEW_SETUP_DEADLINE_SECONDS: Final = 15 * 60.0 _VIEW_SETUP_GATE_TABLE: Final = '"LiteLLM_SpendLogs"' _VIEW_SETUP_GATE_PROBE_ROWS: Final = TypeAdapter(tuple[Mapping[str, bool], ...]) +_DELEGATED_ATTRIBUTE: Final = TypeAdapter(object) _ViewSetupOutcome: TypeAlias = Literal["ready", "timed_out"] _ViewSetupAttempt: TypeAlias = Literal["ready", "table_missing"] | Exception @@ -364,6 +377,47 @@ class _UpdateManyBatch(Protocol): async def commit(self) -> None: ... +def _update_many_batch(db: PrismaWrapper | RoutingPrismaWrapper) -> _UpdateManyBatch: + return db.batch_() + + +def _query_raw_rows( + db: PrismaWrapper | RoutingPrismaWrapper, query: str, *args: object +) -> Awaitable[Sequence[Mapping[str, object]]]: + return db.query_raw(query, *args) + + +def _query_first_row( + db: PrismaWrapper | RoutingPrismaWrapper, query: str, *args: object +) -> Awaitable[Mapping[str, object] | None]: + return db.query_first(query, *args) + + +def _team_member_for_user(row: _TeamMembershipRow) -> Member | None: + members_with_roles: Final = row["team_members_with_roles"] + if members_with_roles is None: + return None + user_id: Final = row["user_id"] + if user_id is None: + return None + matches: Final = [ + Member(**member) + for member in members_with_roles + if member.get("user_id") is not None and user_id == member.get("user_id") + ] + return matches[-1] if matches else None + + +class _EndUserSpendTransaction(Protocol): + def batch_(self) -> contextlib.AbstractAsyncContextManager[_EndUserSpendBatch]: ... + + +def _end_user_spend_transaction( + db: PrismaWrapper | RoutingPrismaWrapper, timeout: timedelta +) -> contextlib.AbstractAsyncContextManager[_EndUserSpendTransaction]: + return db.tx(timeout=timeout) + + unified_guardrail: Final = UnifiedLLMGuardrails() NON_OPENAI_STREAM_GUARDRAIL_TRANSLATION_CALL_TYPES: "frozenset[CallTypes]" = frozenset({CallTypes.anthropic_messages}) @@ -543,7 +597,7 @@ class _UpstreamStreamBoundary(Generic[_T]): self.failure: BaseException | None = None def __getattr__(self, name: str) -> object: - return getattr(self._source, name) + return _DELEGATED_ATTRIBUTE.validate_python(getattr(self._source, name)) def __aiter__(self) -> "_UpstreamStreamBoundary[_T]": return self @@ -5494,7 +5548,9 @@ class PrismaClient: raise e - async def _query_first_with_cached_plan_fallback(self, sql_query: str, *args) -> dict | None: + async def _query_first_with_cached_plan_fallback( + self, sql_query: str, *args: object + ) -> Mapping[str, object] | None: """ Execute a query, recovering once from PostgreSQL's "cached plan must not change result type" error. @@ -5745,7 +5801,7 @@ class PrismaClient: LIMIT $1 OFFSET $2 """ - response = await self.db.query_raw(sql_query, limit, offset) + response = await _query_raw_rows(self.db, sql_query, limit, offset) return response elif table_name == "spend": verbose_proxy_logger.debug("PrismaClient: get_data: table_name == 'spend'") @@ -5889,7 +5945,7 @@ class PrismaClient: """ response = ( - await self.writer_db.query_first(sql_query, hashed_token) + await _query_first_row(self.writer_db, sql_query, hashed_token) if use_writer else await self._query_first_with_cached_plan_fallback(sql_query, hashed_token) ) @@ -5925,31 +5981,14 @@ class PrismaClient: if response["team_blocked"] is None: response["team_blocked"] = False - team_member: Member | None = None - if response["team_members_with_roles"] is not None and response["user_id"] is not None: - ## find the team member corresponding to user id - """ - [ - { - "role": "admin", - "user_id": "default_user_id", - "user_email": null - }, - { - "role": "user", - "user_id": null, - "user_email": "test@email.com" - } - ] - """ - for tm in response["team_members_with_roles"]: - if tm.get("user_id") is not None and response["user_id"] == tm.get("user_id"): - team_member = Member(**tm) + ## find the team member corresponding to user id + team_member: Final = _team_member_for_user(response) response["team_member"] = team_member - response = LiteLLM_VerificationTokenView(**response, last_refreshed_at=time.time()) + token_view: Final = LiteLLM_VerificationTokenView(**response, last_refreshed_at=time.time()) # for prisma we need to cast the expires time to str - if response.expires is not None and isinstance(response.expires, datetime): - response.expires = response.expires.isoformat() + if token_view.expires is not None and isinstance(token_view.expires, datetime): + token_view.expires = token_view.expires.isoformat() + response = token_view return response except Exception as e: import traceback @@ -6239,7 +6278,7 @@ class PrismaClient: """ Batch write update queries """ - batcher: _UpdateManyBatch = self.db.batch_() + batcher: _UpdateManyBatch = _update_many_batch(self.db) for idx, t in enumerate(data_list): # check if plain text or hash if t.token.startswith("sk-"): @@ -6264,7 +6303,7 @@ class PrismaClient: """ Batch write update queries """ - batcher = self.db.batch_() + batcher = _update_many_batch(self.db) for user in sorted(data_list, key=attrgetter("user_id")): try: data_json = self.jsonify_object(data=user.model_dump(exclude_none=True)) @@ -6289,7 +6328,7 @@ class PrismaClient: """ Batch write update queries """ - batcher = self.db.batch_() + batcher = _update_many_batch(self.db) for enduser in data_list: try: data_json = self.jsonify_object(data=enduser.model_dump(exclude_none=True)) @@ -6314,7 +6353,7 @@ class PrismaClient: """ Batch write update queries """ - batcher = self.db.batch_() + batcher = _update_many_batch(self.db) for budget in data_list: try: data_json = self.jsonify_object(data=budget.model_dump(exclude_none=True)) @@ -6337,7 +6376,7 @@ class PrismaClient: and isinstance(data_list, list) ): # Batch write update queries - batcher = self.db.batch_() + batcher = _update_many_batch(self.db) for idx, team in enumerate(data_list): try: data_json = self.jsonify_team_object(db_data=team.model_dump(exclude_none=True)) @@ -7002,7 +7041,7 @@ class PrismaClient: # caller's own retried query, a stronger check than SELECT 1, # and a reader that fails to come back sets `_reader_unavailable` # so reads fall through to the writer just recreated here. - await self.writer_db.query_raw("SELECT 1") + await _query_raw_rows(self.writer_db, "SELECT 1") # A recreate can decline: the optimistic-lock guard no-ops when # the writer generation moved since cycle entry, and the routing # wrapper then leaves the reader untouched as well. Callers that @@ -7350,7 +7389,7 @@ class PrismaClient: async def _view_setup_gate_table_present(self) -> bool: rows: Final = _VIEW_SETUP_GATE_PROBE_ROWS.validate_python( - await self.db.query_raw("SELECT to_regclass($1) IS NOT NULL AS present", _VIEW_SETUP_GATE_TABLE) + await _query_raw_rows(self.db, "SELECT to_regclass($1) IS NOT NULL AS present", _VIEW_SETUP_GATE_TABLE) ) return rows[0]["present"] @@ -7360,7 +7399,7 @@ class PrismaClient: await asyncio.sleep(self._db_health_watchdog_interval_seconds) async with db_span("db_health_watchdog", None): await asyncio.wait_for( - self.db.query_raw("SELECT 1"), + _query_raw_rows(self.db, "SELECT 1"), timeout=self._db_health_watchdog_probe_timeout_seconds, ) if isinstance(self.db, RoutingPrismaWrapper) and self.db.writer_unavailable: @@ -7420,7 +7459,7 @@ class PrismaClient: async def _writer_is_read_only(self, writer: PrismaWrapper) -> bool: """True iff the pooled writer session answers reads but rejects writes (SQLSTATE 25006).""" rows: Final = _WRITER_WRITABILITY_PROBE_ROWS.validate_python( - await writer.query_raw(_WRITER_WRITABILITY_PROBE_SQL) + await _query_raw_rows(writer, _WRITER_WRITABILITY_PROBE_SQL) ) return any(row.get("transaction_read_only") == "on" for row in rows) @@ -7447,7 +7486,7 @@ class PrismaClient: """ sql_query: Final = "SELECT 1" async with db_span("health_check", None): - response: Final[object] = await wrapper.query_raw(sql_query) + response: Final[object] = await _query_raw_rows(wrapper, sql_query) return response async def _probe_answers_now(self, wrapper: PrismaWrapper) -> bool: @@ -8000,7 +8039,7 @@ class ProxyUpdateSpend: try: async with ( db_span("update_end_user_spend", "LiteLLM_EndUserTable"), - prisma_client.db.tx(timeout=timedelta(seconds=60)) as transaction, + _end_user_spend_transaction(prisma_client.db, timedelta(seconds=60)) as transaction, ): batcher: _EndUserSpendBatch async with transaction.batch_() as batcher: diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index d4643994043..6d9249d5572 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -4,6 +4,7 @@ from typing import Final from fastapi import APIRouter, Depends, File, Form, Request, Response, UploadFile from fastapi.responses import ORJSONResponse +from pydantic import TypeAdapter from starlette.datastructures import UploadFile as StarletteUploadFile from litellm.proxy._types import * @@ -36,6 +37,7 @@ from litellm.types.videos.utils import ( ) router: Final = APIRouter() +_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object) @router.post( @@ -582,7 +584,9 @@ async def video_create_character( if video_file: data["video"] = video_file[0] - target_model_name: Final = extract_model_from_target_model_names(data.get("target_model_names")) + target_model_name: Final = extract_model_from_target_model_names( + _OPAQUE_VALUE_ADAPTER.validate_python(data.get("target_model_names")) + ) if target_model_name and not data.get("model"): data["model"] = target_model_name diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index a2be42f1b76..56123e4bee3 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: from litellm.router import Router -def extract_model_from_target_model_names(target_model_names: Any) -> str | None: +def extract_model_from_target_model_names(target_model_names: object) -> str | None: if isinstance(target_model_names, str): model_names: Final = [m.strip() for m in target_model_names.split(",") if m.strip()] return model_names[0] if model_names else None diff --git a/litellm/rag/ingestion/bedrock_ingestion.py b/litellm/rag/ingestion/bedrock_ingestion.py index 1e7aa6ac1c6..5520912dd41 100644 --- a/litellm/rag/ingestion/bedrock_ingestion.py +++ b/litellm/rag/ingestion/bedrock_ingestion.py @@ -106,11 +106,11 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): ) self.embedding_model = self.vector_store_config.get("embedding_model") or "amazon.titan-embed-text-v2:0" - self.wait_for_ingestion = self.vector_store_config.get("wait_for_ingestion", False) + self.wait_for_ingestion: object = self.vector_store_config.get("wait_for_ingestion", False) self.ingestion_timeout: int = _get_int(self.vector_store_config.get("ingestion_timeout"), 300) # Get AWS region using BaseAWSLLM method - _aws_region: Final = self.vector_store_config.get("aws_region_name") + _aws_region: Final[object] = self.vector_store_config.get("aws_region_name") self.aws_region_name = self.get_aws_region_name_for_non_llm_api_calls( aws_region_name=str(_aws_region) if _aws_region else None ) diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 088dfb54155..7b9068ff95e 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -33,7 +33,7 @@ from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes, LlmProviders from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference -from ..litellm_core_utils.core_helpers import normalize_drop_params +from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params from ..litellm_core_utils.credential_accessor import CredentialAccessor from ..litellm_core_utils.get_litellm_params import get_litellm_params from ..litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -83,6 +83,13 @@ def _with_resolved_session_model(session: dict[str, object], model_name: str) -> return {**session, "model": model_name} +def _resolved_request_session(request_data: Mapping[str, object], model_name: str) -> Mapping[str, object] | None: + session: Final = as_str_mapping(request_data.get("session")) + if not isinstance(session, dict): + return None + return _with_resolved_session_model(session, model_name) + + def _build_litellm_metadata(kwargs: dict) -> dict[str, object]: """Build the litellm_metadata dict for guardrail checking (internal only, not forwarded to provider).""" metadata: Final[dict[str, object]] = {**(kwargs.get("litellm_metadata") or {})} @@ -189,8 +196,9 @@ async def acreate_realtime_client_secret( custom_llm_provider=custom_llm_provider, ) request_data: Final = req.model_dump(exclude_none=True, exclude={"model"}) - if isinstance(request_data.get("session"), dict): - request_data["session"] = _with_resolved_session_model(request_data["session"], model_name) + resolved_session: Final = _resolved_request_session(request_data, model_name) + if resolved_session is not None: + request_data["session"] = resolved_session return await base_llm_http_handler.async_realtime_client_secret_handler( api_base=resolved_api_base, api_key=await _realtime_http_bearer_token( diff --git a/litellm/repositories/base_repository.py b/litellm/repositories/base_repository.py index 81fba770b70..92f8ba984f4 100644 --- a/litellm/repositories/base_repository.py +++ b/litellm/repositories/base_repository.py @@ -8,6 +8,7 @@ from typing import Any, Final, Generic, Protocol, TypeVar, runtime_checkable from pydantic import BaseModel +from litellm.litellm_core_utils.core_helpers import attribute_or_none from litellm.repositories.chunked_in import find_many_in from litellm.repositories.prisma_protocols import TableActions @@ -131,4 +132,4 @@ def is_unique_violation(exc: BaseException) -> bool: return "P2002" in str(exc) or "unique constraint" in str(exc).lower() if isinstance(exc, UniqueViolationError): return True - return getattr(exc, "code", None) == "P2002" + return attribute_or_none(exc, "code") == "P2002" diff --git a/litellm/repositories/team_repository.py b/litellm/repositories/team_repository.py index 57f6fd33c11..b89d87aa354 100644 --- a/litellm/repositories/team_repository.py +++ b/litellm/repositories/team_repository.py @@ -75,6 +75,7 @@ class _PrismaClientView(Protocol): _MEMBERS_WITH_ROLES_ADAPTER: Final = TypeAdapter(list[Member]) +_RAW_COLUMN: Final = TypeAdapter(object) _JSON_ENCODED_TEAM_FIELDS: Final = ( "metadata", "model_spend", @@ -142,7 +143,7 @@ class TeamRepository(BaseRepository[LiteLLM_TeamTable]): ) if not rows: return None - raw_value: Final[object] = rows[0]["members_with_roles"] + raw_value: Final = _RAW_COLUMN.validate_python(rows[0]["members_with_roles"]) parsed: Final[object] = json.loads(raw_value) if isinstance(raw_value, str) else raw_value if not parsed: return [] diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index f7e2616324f..65d9b89cf79 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -1,6 +1,6 @@ import asyncio import contextvars -from collections.abc import Awaitable +from collections.abc import Awaitable, Coroutine from functools import partial from typing import Final, Literal @@ -76,7 +76,7 @@ async def arerank( func_with_context: Final = partial(ctx.run, func) init_response: Final = await loop.run_in_executor(None, func_with_context) - if asyncio.iscoroutine(init_response): + if isinstance(init_response, Coroutine): response = await init_response else: response = init_response diff --git a/litellm/rerank_api/rerank_utils.py b/litellm/rerank_api/rerank_utils.py index f365ae749d1..4924625b011 100644 --- a/litellm/rerank_api/rerank_utils.py +++ b/litellm/rerank_api/rerank_utils.py @@ -1,4 +1,4 @@ -from typing import Any, Final +from typing import Final from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig @@ -8,7 +8,7 @@ def get_optional_rerank_params( model: str, drop_params: bool, query: str, - documents: list[str | dict[str, Any]], + documents: list[str | dict[str, object]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, diff --git a/litellm/responses/file_search/emulated_handler.py b/litellm/responses/file_search/emulated_handler.py index 7121908517b..f29f6c83ef0 100644 --- a/litellm/responses/file_search/emulated_handler.py +++ b/litellm/responses/file_search/emulated_handler.py @@ -15,7 +15,7 @@ import json import time import uuid from collections.abc import Iterable, Sequence -from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast # noqa: TID251 # see kwargs-ok / cast-ok markers +from typing import TYPE_CHECKING, Final, TypeAlias, cast # noqa: TID251 # see cast-ok markers from typing_extensions import NotRequired, ReadOnly, TypedDict @@ -517,7 +517,7 @@ async def aresponses_with_emulated_file_search( model: str, tools: Iterable[ToolParam] | None = None, # Pass-through params — forwarded as-is to the underlying aresponses call - **kwargs: Any, # kwargs-ok: `object` would surface the caller's partially-unknown dict at its call site + **kwargs: object, # kwargs-ok: pass-through params forwarded as-is to the underlying aresponses call ) -> ResponsesAPIResponse: """ Emulated file_search for providers that don't support it natively. diff --git a/litellm/responses/litellm_completion_transformation/custom_tools.py b/litellm/responses/litellm_completion_transformation/custom_tools.py index fcda6b31b00..98e1984550c 100644 --- a/litellm/responses/litellm_completion_transformation/custom_tools.py +++ b/litellm/responses/litellm_completion_transformation/custom_tools.py @@ -23,6 +23,7 @@ from typing import Final, Literal, TypeAlias from pydantic import TypeAdapter, ValidationError from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ( ChatCompletionToolParam, @@ -30,6 +31,7 @@ from litellm.types.llms.openai import ( ) _MAX_ARGUMENTS_LEN: Final = 1_000_000 +_DECODED_JSON: Final = TypeAdapter(object) TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE: Final = MappingProxyType( {"function_call": "fc", "custom_tool_call": "ctc", "tool_search_call": "tsc"} @@ -111,8 +113,8 @@ def unwrap_custom_tool_arguments(arguments: str) -> str: if len(arguments) > _MAX_ARGUMENTS_LEN: return arguments try: - parsed: Final = json.loads(arguments) - if isinstance(parsed, dict) and "content" in parsed: + parsed: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(arguments))) + if parsed is not None and "content" in parsed: return str(parsed["content"]) except (json.JSONDecodeError, TypeError, ValueError): pass diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index c25dd4a405e..b5ef9e95741 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -5,6 +5,7 @@ from itertools import filterfalse from typing import Any, Final, cast import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend from litellm.main import stream_chunk_builder @@ -84,14 +85,18 @@ def _output_items_with_id(items: tuple[Any, ...], item_type: str, item_id: str | ) +def _is_signed_thinking_block(block: object) -> bool: + fields: Final = as_str_mapping(block) + return fields is not None and isinstance(block, dict) and bool(fields.get("signature") or fields.get("data")) + + def _streams_compaction(chunk: ModelResponseStream) -> bool: delta_fields: Final = chunk.choices[0].delta.provider_specific_fields if chunk.choices else None return delta_fields is not None and "compaction_blocks" in delta_fields def _delta_has_signed_thinking_block(delta: object) -> bool: - blocks: Final = getattr(delta, "thinking_blocks", None) or () - return any(isinstance(b, dict) and (b.get("signature") or b.get("data")) for b in blocks) + return any(_is_signed_thinking_block(block) for block in attribute_items(delta, "thinking_blocks")) def _delta_carries_output(delta: ChatCompletionDelta) -> bool: @@ -1427,7 +1432,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _is_unstreamed_empty_message(self, item: object) -> bool: if getattr(item, "type", None) != "message" or self._message_output_index is not None: return False - return not any(getattr(part, "text", None) for part in getattr(item, "content", None) or ()) + return not any(getattr(part, "text", None) for part in attribute_items(item, "content")) def finished_round_event(self) -> ResponseCompletedEvent | ResponseIncompleteEvent | None: if not isinstance(self.litellm_model_response, ModelResponse): diff --git a/litellm/responses/main.py b/litellm/responses/main.py index e7380c26529..1ff80b0d71d 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -191,6 +191,7 @@ def mock_responses_api_response( _STR_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_FORWARDED_VALUE: Final = TypeAdapter(object) def _client_trace_id(request_kwargs: Mapping[str, object]) -> str | None: @@ -866,7 +867,7 @@ async def aresponses( stream: bool | None = None, temperature: float | None = None, text: Optional["ResponseText"] = None, - text_format: type["BaseModel"] | dict | None = None, + text_format: type["BaseModel"] | dict[str, object] | None = None, tool_choice: ToolChoice | None = None, tools: Iterable[ToolParam] | None = None, top_p: float | None = None, @@ -1355,7 +1356,7 @@ def _responses_try_dispatch_emulated_file_search( user: str | None, service_tier: str | None, safety_identifier: str | None, - text_format: type[BaseModel] | dict | None, + text_format: type[BaseModel] | dict[str, object] | None, allowed_openai_params: list[str] | None, extra_headers: dict[str, object] | None, extra_query: dict[str, object] | None, @@ -1390,7 +1391,7 @@ def _responses_try_dispatch_emulated_file_search( "background": background, "stream": stream, "temperature": temperature, - "text": text, + "text": _FORWARDED_VALUE.validate_python(text), "tool_choice": tool_choice, "top_p": top_p, "truncation": truncation, @@ -1435,7 +1436,7 @@ def responses( stream: bool | None = None, temperature: float | None = None, text: Optional["ResponseText"] = None, - text_format: type["BaseModel"] | dict | None = None, + text_format: type["BaseModel"] | dict[str, object] | None = None, tool_choice: ToolChoice | None = None, tools: Iterable[ToolParam] | None = None, top_p: float | None = None, diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index b100858b306..cd072825ef5 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -4,6 +4,7 @@ import logging from types import MappingProxyType from typing import TYPE_CHECKING, Final, cast +from pydantic import TypeAdapter from typing_extensions import TypedDict, Unpack from litellm._internal_context import internal_sub_call @@ -24,6 +25,8 @@ from litellm.utils import CustomStreamWrapper if TYPE_CHECKING: from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth +_DELEGATED_ATTRIBUTE: Final = TypeAdapter(object) + class _MCPCompletionKwargs(TypedDict, total=False, extra_items=object): """Extra keywords forwarded verbatim to ``litellm.acompletion``, which owns their contract.""" @@ -566,7 +569,7 @@ async def acompletion_with_mcp( def __getattr__(self, name: str) -> object: # Delegate all other attributes to original wrapper - return getattr(self._original_wrapper, name) + return _DELEGATED_ATTRIBUTE.validate_python(getattr(self._original_wrapper, name)) # Helper class to wrap async iterator for sync iteration class _SyncIteratorWrapper: diff --git a/litellm/responses/sse_output_recovery.py b/litellm/responses/sse_output_recovery.py index cdabcb504d3..e7d0a540022 100644 --- a/litellm/responses/sse_output_recovery.py +++ b/litellm/responses/sse_output_recovery.py @@ -11,9 +11,12 @@ import json from collections.abc import Mapping from typing import Final, SupportsInt, TypeAlias, cast # noqa: TID251 # int() re-checks the cast below at runtime +from pydantic import TypeAdapter + from litellm.constants import STREAM_SSE_DONE_STRING _MAX_CONTENT_INDEX: Final = 1024 +_DECODED_JSON: Final = TypeAdapter(object) _ConvertibleToInt: TypeAlias = SupportsInt | str @@ -33,7 +36,7 @@ def parse_sse_json_chunk(chunk: str) -> dict[str, object] | None: if not stripped_chunk or stripped_chunk == STREAM_SSE_DONE_STRING or stripped_chunk.startswith("event:"): return None try: - parsed_chunk: Final[object] = json.loads(stripped_chunk) + parsed_chunk: Final = _DECODED_JSON.validate_python(json.loads(stripped_chunk)) except json.JSONDecodeError: return None if not isinstance(parsed_chunk, dict): diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 0d59c93786c..50ab3e4f75a 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -28,7 +28,7 @@ from typing import ( import httpx from openai._streaming import SSEDecoder -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import TypeIs import litellm @@ -166,6 +166,7 @@ def _typed_gets_litellm_params(fn: _GetsLitellmParams) -> _GetsLitellmParams: _SHOULD_STORE_RESULT_IN_CACHE_ATTR: Final = "_should_store_result_in_cache" _UNMASK_PII_TEXT_ATTR: Final = "_unmask_pii_text" +_DECODED_JSON: Final = TypeAdapter(object) def _load_json_object(payload: str | bytes) -> dict[str, object]: @@ -175,7 +176,7 @@ def _load_json_object(payload: str | bytes) -> dict[str, object]: def _load_json_value(payload: str | bytes) -> object: """Parse a JSON payload whose top-level shape the caller narrows itself.""" - return json.loads(payload) + return _DECODED_JSON.validate_python(json.loads(payload)) def _model_id_from_metadata(litellm_metadata: Mapping[str, object] | None) -> str | None: diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index f256480d952..dc75e77d685 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -5,10 +5,12 @@ from functools import reduce from typing import Any, Final, Optional, TypeVar, Union, cast, get_type_hints, overload from pydantic import BaseModel, TypeAdapter +from typing_extensions import TypedDict as ExtraItemsTypedDict from typing_extensions import TypeIs # noqa: TID251 # narrows untyped wire payloads without a runtime conversion import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, iter_items from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel from litellm.types.llms.openai import ( @@ -33,6 +35,10 @@ from litellm.types.utils import ( ) +class _UsageExtraFields(ExtraItemsTypedDict, total=False, extra_items=object): + pass + + def _apply_nested_drop_params(params: dict[str, object], additional_drop_params: list[str] | None) -> dict[str, object]: nested_paths: Final = tuple(path for path in additional_drop_params or () if is_nested_path(path)) return reduce(lambda acc, path: delete_nested_value(acc, path), nested_paths, params) @@ -64,6 +70,28 @@ def _is_object_dict( return isinstance(value, dict) +def _prompt_tokens_details_wrapper(details: object) -> PromptTokensDetailsWrapper: + if _is_object_dict(details): + input_details: Final = InputTokensDetails.model_validate(details) + return PromptTokensDetailsWrapper( + **input_details.model_dump(exclude={"cache_write_tokens_details"}), + cache_creation_token_details=_cache_write_lifetimes(input_details.cache_write_tokens_details), + ) + return PromptTokensDetailsWrapper( + cached_tokens=getattr(details, "cached_tokens", None), + audio_tokens=getattr(details, "audio_tokens", None), + text_tokens=getattr(details, "text_tokens", None), + image_tokens=getattr(details, "image_tokens", None), + cached_tokens_details=getattr(details, "cached_tokens_details", None), + video_tokens=getattr(details, "video_tokens", None), + cache_write_tokens=getattr(details, "cache_write_tokens", None), + cache_creation_token_details=_cache_write_lifetimes(getattr(details, "cache_write_tokens_details", None)), + cache_type=getattr(details, "cache_type", None), + web_search_requests=getattr(details, "web_search_requests", None), + google_maps_grounding_requests=getattr(details, "google_maps_grounding_requests", None), + ) + + def normalize_responses_api_stream_options( stream_options: object, ) -> ResponsesAPIStreamOptions | None: @@ -1060,7 +1088,7 @@ class ResponsesAPIRequestUtils: @staticmethod def _update_container_ids_in_response( - responses_api_response: ResponsesAPIResponse | dict[str, Any], + responses_api_response: ResponsesAPIResponse | dict[str, object], custom_llm_provider: str | None, litellm_metadata: dict[str, Any] | None = None, ) -> ResponsesAPIResponse | dict[str, object]: @@ -1074,15 +1102,16 @@ class ResponsesAPIRequestUtils: model_id: Final = model_info.get("id") # Get the output list - if isinstance(responses_api_response, dict): - output = responses_api_response.get("output", []) - else: - output = getattr(responses_api_response, "output", []) + output: Final = ( + responses_api_response.get("output", []) + if isinstance(responses_api_response, dict) + else attribute_or_none(responses_api_response, "output") + ) if not output: return responses_api_response - for item in output: + for item in iter_items(output): ResponsesAPIRequestUtils.encode_container_id_on_output_item( item=item, custom_llm_provider=custom_llm_provider, @@ -1257,34 +1286,11 @@ class ResponseAPILoggingUtils: response_api_usage = usage_input prompt_tokens: Final[int] = response_api_usage.input_tokens or 0 completion_tokens: Final[int] = response_api_usage.output_tokens or 0 - prompt_tokens_details: PromptTokensDetailsWrapper | None = None - if response_api_usage.input_tokens_details: - if isinstance(response_api_usage.input_tokens_details, dict): - input_details: Final = InputTokensDetails.model_validate(response_api_usage.input_tokens_details) - prompt_tokens_details = PromptTokensDetailsWrapper( - **input_details.model_dump(exclude={"cache_write_tokens_details"}), - cache_creation_token_details=_cache_write_lifetimes(input_details.cache_write_tokens_details), - ) - else: - prompt_tokens_details = PromptTokensDetailsWrapper( - cached_tokens=getattr(response_api_usage.input_tokens_details, "cached_tokens", None), - audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None), - text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None), - image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None), - cached_tokens_details=getattr( - response_api_usage.input_tokens_details, "cached_tokens_details", None - ), - video_tokens=getattr(response_api_usage.input_tokens_details, "video_tokens", None), - cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None), - cache_creation_token_details=_cache_write_lifetimes( - getattr(response_api_usage.input_tokens_details, "cache_write_tokens_details", None) - ), - cache_type=getattr(response_api_usage.input_tokens_details, "cache_type", None), - web_search_requests=getattr(response_api_usage.input_tokens_details, "web_search_requests", None), - google_maps_grounding_requests=getattr( - response_api_usage.input_tokens_details, "google_maps_grounding_requests", None - ), - ) + prompt_tokens_details: Final = ( + _prompt_tokens_details_wrapper(response_api_usage.input_tokens_details) + if response_api_usage.input_tokens_details + else None + ) completion_tokens_details: CompletionTokensDetailsWrapper | None = None output_tokens_details: Final[OutputTokensDetails | None] = getattr( response_api_usage, "output_tokens_details", None @@ -1308,20 +1314,22 @@ class ResponseAPILoggingUtils: audio_tokens=audio_tokens, ) - extra_usage_fields: Final = { - key: value - for key, value in (response_api_usage.model_extra or {}).items() - if key - not in ( - "input_token_details", - "output_token_details", - "prompt_tokens", - "completion_tokens", - "total_tokens", - "prompt_tokens_details", - "completion_tokens_details", - ) - } + extra_usage_fields: Final = _UsageExtraFields( + **{ + key: value + for key, value in (as_str_mapping(response_api_usage.model_extra) or {}).items() + if key + not in ( + "input_token_details", + "output_token_details", + "prompt_tokens", + "completion_tokens", + "total_tokens", + "prompt_tokens_details", + "completion_tokens_details", + ) + } + ) chat_usage: Final = Usage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, diff --git a/litellm/router.py b/litellm/router.py index 705accbcf8d..40c5c8afa6c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -934,7 +934,7 @@ MAX_HELD_PRE_OUTPUT_RESPONSES_EVENTS: Final = 200 def _retry_policy_ceiling(policy: RetryPolicy) -> int: """The most retries any error class under this policy can be granted.""" - retry_counts: Final[Iterable[object]] = policy.model_dump().values() + retry_counts: Final[tuple[object, ...]] = tuple(policy.model_dump().values()) return max((retries for retries in retry_counts if isinstance(retries, int)), default=0) @@ -6504,7 +6504,7 @@ class Router: has_generated_content: bool, buffered_lifecycle_chunks: tuple[bytes, ...], model: str, - initial_kwargs: dict[str, Any], # mutable-ok: handed to _aanthropic_messages_fallback_attempt, which mutates it + initial_kwargs: dict[str, object], # mutable-ok: _aanthropic_messages_fallback_attempt mutates it wrapper: "FallbackAwareAnthropicMessagesStream", close_superseded: Callable[[], Awaitable[None]], ) -> AsyncGenerator[bytes, None]: diff --git a/litellm/router_strategy/quality_router/quality_router.py b/litellm/router_strategy/quality_router/quality_router.py index 1d119375778..d8c52b4c8b6 100644 --- a/litellm/router_strategy/quality_router/quality_router.py +++ b/litellm/router_strategy/quality_router/quality_router.py @@ -47,13 +47,13 @@ class QualityRouter(CustomLogger): model_name: str, litellm_router_instance: "Router", default_model: str | None = None, - quality_router_config: dict[str, Any] | None = None, + quality_router_config: Mapping[str, object] | None = None, ): self.model_name = model_name self.litellm_router_instance = litellm_router_instance if quality_router_config: - self.config = QualityRouterConfig(**quality_router_config) + self.config = QualityRouterConfig.model_validate(dict(**quality_router_config)) else: self.config = QualityRouterConfig() @@ -185,7 +185,7 @@ class QualityRouter(CustomLogger): if isinstance(raw_prefs, RoutingPreferences): prefs = raw_prefs elif isinstance(raw_prefs, dict): - prefs = RoutingPreferences(**raw_prefs) + prefs = RoutingPreferences.model_validate(dict(**raw_prefs)) else: # A Pydantic object of some other shape — coerce via its dict. prefs = RoutingPreferences( diff --git a/litellm/sandbox/__init__.py b/litellm/sandbox/__init__.py index 1974859fb1e..bcbb7bc5743 100644 --- a/litellm/sandbox/__init__.py +++ b/litellm/sandbox/__init__.py @@ -7,7 +7,10 @@ They resolve lazily so `import litellm` does not pull in litellm.harness. import importlib from typing import Final +from pydantic import TypeAdapter + _HARNESS_SANDBOX_MODULE: Final = "litellm.harness.sandbox" +_HARNESS_EXPORT: Final = TypeAdapter(object) _HARNESS_EXPORTS: Final = frozenset( { "local", @@ -23,5 +26,5 @@ _HARNESS_EXPORTS: Final = frozenset( def __getattr__(name: str) -> object: if name in _HARNESS_EXPORTS: - return getattr(importlib.import_module(_HARNESS_SANDBOX_MODULE), name) + return _HARNESS_EXPORT.validate_python(getattr(importlib.import_module(_HARNESS_SANDBOX_MODULE), name)) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/litellm/secret_managers/base_secret_manager.py b/litellm/secret_managers/base_secret_manager.py index e537dd1d998..9c25086c451 100644 --- a/litellm/secret_managers/base_secret_manager.py +++ b/litellm/secret_managers/base_secret_manager.py @@ -1,6 +1,6 @@ import re from abc import ABC, abstractmethod -from typing import Any, Final +from typing import Final import httpx @@ -73,7 +73,7 @@ class BaseSecretManager(ABC): optional_params: dict | None = None, timeout: float | httpx.Timeout | None = None, tags: dict | list | None = None, - ) -> dict[str, Any]: + ) -> dict[str, object]: """ Asynchronously write a secret to the secret manager. diff --git a/litellm/secret_managers/cyberark_secret_manager.py b/litellm/secret_managers/cyberark_secret_manager.py index a1ec7c0a386..abfa9649e91 100644 --- a/litellm/secret_managers/cyberark_secret_manager.py +++ b/litellm/secret_managers/cyberark_secret_manager.py @@ -1,7 +1,7 @@ import asyncio import base64 import os -from typing import Any, Final +from typing import Final from urllib.parse import quote import httpx @@ -278,7 +278,7 @@ class CyberArkSecretManager(BaseSecretManager): optional_params: dict | None = None, timeout: float | httpx.Timeout | None = None, tags: dict | list | None = None, - ) -> dict[str, Any]: + ) -> dict[str, object]: """ Writes a secret to CyberArk Conjur using an async HTTPX client. diff --git a/litellm/secret_managers/main.py b/litellm/secret_managers/main.py index f4234e26c60..b33cf0e6b7e 100644 --- a/litellm/secret_managers/main.py +++ b/litellm/secret_managers/main.py @@ -24,6 +24,7 @@ from litellm.types.llms.base import LiteLLMBaseModel oidc_cache: Final = DualCache() _PARSED_LITERAL: Final = TypeAdapter(object) +_SECRET_MANAGER_CLIENT: Final = TypeAdapter(object) @lru_cache(maxsize=128) @@ -383,7 +384,7 @@ def get_secret( try: if _should_read_secret_from_secret_manager() and litellm.secret_manager_client is not None: try: - client: Final = litellm.secret_manager_client + client: Final = _SECRET_MANAGER_CLIENT.validate_python(litellm.secret_manager_client) key_manager = "local" if key_management_system is not None: key_manager = key_management_system.value diff --git a/litellm/utils.py b/litellm/utils.py index bb94ca6d40a..1db5f7129df 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1641,7 +1641,7 @@ def _deployment_provider(kwargs: Mapping[str, object]) -> str: return "" -async def async_pre_call_deployment_hook(kwargs: dict[str, Any], call_type: str) -> dict[str, Any]: +async def async_pre_call_deployment_hook(kwargs: Mapping[str, object], call_type: str) -> dict[str, Any]: """ Allow modifying the request just before it's sent to the deployment. @@ -4287,7 +4287,7 @@ def get_optional_params_embeddings( # retrieve all parameters passed to the function passed_params: Final = locals() - custom_llm_provider = passed_params.pop("custom_llm_provider", None) + passed_params.pop("custom_llm_provider", None) special_params: Final = passed_params.pop("kwargs") passed_params.pop("drop_params", None) @@ -4325,7 +4325,7 @@ def get_optional_params_embeddings( provider_config: BaseEmbeddingConfig | None = None optional_params = {} - if custom_llm_provider is not None and custom_llm_provider in LlmProviders._member_map_.values(): + if custom_llm_provider in LlmProviders._member_map_.values(): provider_config = ProviderConfigManager.get_provider_embedding_config( model=model, provider=LlmProviders(custom_llm_provider), @@ -9060,8 +9060,9 @@ _get_bundled_model_cost_map = get_bundled_model_cost_map def _get_model_cost_entry_for_provider_config( model: str, provider: LlmProviders, -) -> dict[str, object]: +) -> Mapping[str, object]: candidate_keys: Final = (model, f"{provider.value}/{model}") + model_info: Mapping[str, object] | None for model_key in candidate_keys: model_info = litellm.model_cost.get(model_key) if model_info is not None: @@ -10886,18 +10887,12 @@ def extract_duration_from_srt_or_vtt(srt_or_vtt_content: str) -> float | None: # Regular expression to match timestamps in the format "hh:mm:ss,ms" or "hh:mm:ss.ms" timestamp_pattern: Final = r"(\d{2}):(\d{2}):(\d{2})[.,](\d{3})" - timestamps: Final[Sequence[tuple[str, str, str, str]]] = re.findall(timestamp_pattern, srt_or_vtt_content) - - if not timestamps: - return None + timestamps: Final = re.finditer(timestamp_pattern, srt_or_vtt_content) # Convert timestamps to seconds and find the max (end time) - durations: Final = [] - match: tuple[str, str, str, str] - for match in timestamps: - hours, minutes, seconds, milliseconds = map(int, match) - total_seconds = hours * 3600 + minutes * 60 + seconds + milliseconds / 1000.0 - durations.append(total_seconds) + durations: Final = [ + int(match[1]) * 3600 + int(match[2]) * 60 + int(match[3]) + int(match[4]) / 1000.0 for match in timestamps + ] return max(durations) if durations else None diff --git a/litellm/vector_stores/main.py b/litellm/vector_stores/main.py index 8b33b1ce12c..9f53accfa0b 100644 --- a/litellm/vector_stores/main.py +++ b/litellm/vector_stores/main.py @@ -5,7 +5,7 @@ LiteLLM SDK Functions for Creating and Searching Vector Stores import asyncio import builtins import contextvars -from collections.abc import Coroutine, Mapping +from collections.abc import Coroutine, Iterator, Mapping from functools import partial from typing import TYPE_CHECKING, Final @@ -53,6 +53,10 @@ def _direct_vector_store_embedding_executor( ) +def _items_without_key(values: Mapping[str, object], excluded_key: str) -> Iterator[tuple[str, object]]: + return ((key, value) for key, value in values.items() if key != excluded_key) + + def mock_vector_store_search_response( mock_results: builtins.list[VectorStoreSearchResult] | None = None, ): @@ -307,7 +311,7 @@ async def asearch( embedding_executor: Final = _direct_vector_store_embedding_executor( kwargs.pop("_direct_vector_store_embedding_executor", None), router, kwargs ) - local_vars: Final = {key: value for key, value in locals().items() if key != "embedding_executor"} + local_vars: Final = dict(_items_without_key(locals(), "embedding_executor")) try: loop: Final = asyncio.get_event_loop() @@ -391,7 +395,7 @@ def search( embedding_executor: Final = _direct_vector_store_embedding_executor( kwargs.pop("_direct_vector_store_embedding_executor", None), router, kwargs ) - local_vars: Final = {key: value for key, value in locals().items() if key != "embedding_executor"} + local_vars: Final = dict(_items_without_key(locals(), "embedding_executor")) try: litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None) @@ -694,7 +698,7 @@ def list( timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs, -): +) -> Mapping[str, object] | Coroutine[object, object, Mapping[str, object]]: """ List vector stores. @@ -992,7 +996,7 @@ def delete( timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs, -): +) -> Mapping[str, object] | Coroutine[object, object, Mapping[str, object]]: """ Delete a vector store. diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 7610d602fff..a5bb5e0b340 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -42,7 +42,7 @@ IGNORE_FUNCTIONS = [ "contains_encrypted_marker", # max depth set. "_rotate_guardrail_row", # bounded by attempts_left. "_delete_nested_value_custom", # max depth set (bounded by number of path segments). - "filter_exceptions_from_params", # max depth set (default 20) to prevent infinite recursion. + "_filter_exceptions_from_params", # max depth set (default 20) to prevent infinite recursion. "__getattr__", # lazy loading pattern in litellm/__init__.py with proper caching to prevent infinite recursion. "_validate_inheritance_chain", # max depth set (default 100) to prevent infinite recursion in policy inheritance validation. "_basic_json_schema_validate", # max depth set. diff --git a/tests/unit/a2a_protocol/test_utils.py b/tests/unit/a2a_protocol/test_utils.py index 8a219211c62..dfa8449f6e6 100644 --- a/tests/unit/a2a_protocol/test_utils.py +++ b/tests/unit/a2a_protocol/test_utils.py @@ -1,5 +1,8 @@ """Tests for litellm/a2a_protocol/utils.py token/usage extraction.""" +from types import SimpleNamespace +from typing import Final + import pytest pytest.importorskip("a2a.compat.v0_3.types") @@ -7,6 +10,7 @@ pytest.importorskip("a2a.compat.v0_3.types") from a2a.compat.v0_3.types import MessageSendParams, SendMessageRequest from litellm.a2a_protocol.utils import A2ARequestUtils +from tests.unit.litellm_core_utils.attribute_forwarder import AttributeForwarder def _request(user_text: str) -> SendMessageRequest: @@ -39,3 +43,13 @@ def test_calculate_usage_counts_input_tokens_from_request_object(): assert prompt_tokens > 0 assert completion_tokens > 0 assert total_tokens == prompt_tokens + completion_tokens + + +def test_calculate_usage_counts_input_tokens_from_a_message_that_forwards_its_fields(): + request: Final = _request("count these input tokens please") + forwarded_request: Final = SimpleNamespace(params=SimpleNamespace(message=AttributeForwarder(request.params.message))) + response_dict: Final = {"result": {"kind": "message", "parts": [{"kind": "text", "text": "ok"}]}} + + assert A2ARequestUtils.calculate_usage_from_request_response( + request=forwarded_request, response_dict=response_dict + ) == A2ARequestUtils.calculate_usage_from_request_response(request=request, response_dict=response_dict) diff --git a/tests/unit/containers/test_container_proxy_ownership.py b/tests/unit/containers/test_container_proxy_ownership.py index 780ae9bf6d1..ac5adeee8c1 100644 --- a/tests/unit/containers/test_container_proxy_ownership.py +++ b/tests/unit/containers/test_container_proxy_ownership.py @@ -1,6 +1,7 @@ import json import sys from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -10,6 +11,7 @@ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.container_endpoints import ownership from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.containers.main import ContainerListResponse, ContainerObject +from tests.unit.litellm_core_utils.attribute_forwarder import AttributeForwarder @pytest.fixture(autouse=True) @@ -97,6 +99,23 @@ async def test_should_not_mutate_dict_container_response_when_recording_owner( assert file_obj["provider_container_id"] == "cntr_provider" +@pytest.mark.asyncio +async def test_should_store_the_full_container_of_a_response_that_forwards_its_fields(monkeypatch): + table: Final = AsyncMock() + table.find_unique.return_value = None + prisma_client: Final = SimpleNamespace(db=SimpleNamespace(litellm_managedobjecttable=table)) + monkeypatch.setattr(ownership, "_get_prisma_client", AsyncMock(return_value=prisma_client)) + + await ownership.record_container_owner( + response=AttributeForwarder(_container("cntr_provider")), + user_api_key_dict=UserAPIKeyAuth(user_id="user-1"), + custom_llm_provider="openai", + ) + + file_obj: Final = json.loads(table.create.await_args.kwargs["data"]["file_object"]) + assert file_obj["status"] == "active" + + @pytest.mark.asyncio async def test_should_record_team_owner_for_keys_without_user_id(monkeypatch): table = AsyncMock() diff --git a/tests/unit/enterprise/proxy/hooks/test_managed_files.py b/tests/unit/enterprise/proxy/hooks/test_managed_files.py index 20ed8ac0c10..7f6e955e831 100644 --- a/tests/unit/enterprise/proxy/hooks/test_managed_files.py +++ b/tests/unit/enterprise/proxy/hooks/test_managed_files.py @@ -31,6 +31,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( encode_file_id_with_model, ) from litellm.types.llms.openai import OpenAIFileObject +from tests.unit.litellm_core_utils.attribute_forwarder import AttributeForwarder class _InMemoryManagedFileTable: @@ -451,6 +452,20 @@ async def test_async_pre_call_deployment_hook_prefers_top_level_model_info(): ), "Should prefer top-level model_info over litellm_metadata" +@pytest.mark.asyncio +async def test_async_pre_call_deployment_hook_reads_model_info_that_forwards_its_fields(): + proxy_managed_files: Final = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=MagicMock()) + kwargs: Final = { + "input_file_id": "managed-file-abc", + "model_file_id_mapping": {"managed-file-abc": {"deployment-xyz": "file-provider-123"}}, + "model_info": AttributeForwarder({"id": "deployment-xyz"}), + } + + result: Final = await proxy_managed_files.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.acreate_batch) + + assert result["input_file_id"] == "file-provider-123" + + @pytest.mark.asyncio async def test_async_pre_call_deployment_hook_no_model_info_leaves_file_id_unchanged(): """ diff --git a/tests/unit/integrations/dotprompt/test_prompt_manager.py b/tests/unit/integrations/dotprompt/test_prompt_manager.py index dbd4e4a4c4c..cbf19fe7985 100644 --- a/tests/unit/integrations/dotprompt/test_prompt_manager.py +++ b/tests/unit/integrations/dotprompt/test_prompt_manager.py @@ -7,6 +7,7 @@ from fastapi.testclient import TestClient +from typing import Final from unittest.mock import MagicMock, Mock, patch import httpx @@ -362,6 +363,22 @@ def test_load_prompts_from_json_data(): assert rendered == "Dynamic prompt: test" +def test_load_prompts_from_json_data_skips_malformed_entries(): + manager: Final = PromptManager(prompt_directory=str(Path(__file__).parent)) + initial_count: Final = len(manager.prompts) + + manager.load_prompts_from_json_data( + { + "valid_prompt": {"content": "Valid: {{value}}", "metadata": {"model": "gpt-4"}}, + "string_entry": "not a prompt", + "null_entry": None, + } + ) + + assert len(manager.prompts) == initial_count + 1 + assert manager.render("valid_prompt", {"value": "ok"}) == "Valid: ok" + + def test_prompt_file_to_json_conversion(): """Test converting .prompt files to JSON format.""" # Create a temporary prompt file with frontmatter @@ -661,6 +678,27 @@ def test_prompt_initializer_registers_flat_db_prompt_under_base_id(): assert template.content == "AHOY {{name}}" +def test_prompt_initializer_accepts_params_without_dotprompt_content(): + from types import SimpleNamespace + + from litellm.integrations.dotprompt import prompt_initializer + from litellm.types.prompts.init_prompts import PromptLiteLLMParams, PromptSpec + + litellm_params: Final = SimpleNamespace( + prompt_data={"content": "AHOY {{name}}", "metadata": {"model": "gpt-4o-mini"}}, + ) + prompt_spec: Final = PromptSpec( + prompt_id="agent-prompt", + litellm_params=PromptLiteLLMParams(prompt_integration="dotprompt"), + ) + + dotprompt_manager: Final = prompt_initializer(litellm_params, prompt_spec) + + template: Final = dotprompt_manager.prompt_manager.get_prompt("agent-prompt") + assert template is not None + assert template.content == "AHOY {{name}}" + + def _swap_prompt_manager_and_spec(ignore_prompt_manager_model: bool) -> tuple[DotpromptManager, PromptSpec]: manager = DotpromptManager( prompt_data={"content": "You are a pirate assistant.", "metadata": {"model": "gpt-4o-mini"}}, diff --git a/tests/unit/integrations/newrelic/test_newrelic.py b/tests/unit/integrations/newrelic/test_newrelic.py index 00a115dd8f2..d09732c6eb2 100644 --- a/tests/unit/integrations/newrelic/test_newrelic.py +++ b/tests/unit/integrations/newrelic/test_newrelic.py @@ -1,6 +1,7 @@ import os import sys from datetime import datetime, timezone +from typing import Final from unittest.mock import MagicMock, patch import pytest @@ -491,6 +492,18 @@ class TestExtractAllMessagesTimestamps: input_msg = next(m for m in messages if not m.get("is_response")) assert input_msg["timestamp"] == int(1_000_000.0 * 1000.0) + def test_input_messages_read_a_start_time_that_only_exposes_timestamp(self): + class EpochSeconds: + def timestamp(self) -> float: + return 1_000_000.0 + + kwargs: Final = {**make_kwargs(messages=[{"role": "user", "content": "Hi"}]), "start_time": EpochSeconds()} + + messages: Final = self.logger._extract_all_messages(kwargs, make_response(), response_model="gpt-4", vendor="openai") + + input_msg: Final = next(m for m in messages if not m.get("is_response")) + assert input_msg["timestamp"] == int(1_000_000.0 * 1000.0) + def test_output_messages_get_end_time_timestamp(self): kwargs = make_kwargs(messages=[{"role": "user", "content": "Hi"}]) response = make_response() diff --git a/tests/unit/litellm_core_utils/attribute_forwarder.py b/tests/unit/litellm_core_utils/attribute_forwarder.py new file mode 100644 index 00000000000..3056057fc97 --- /dev/null +++ b/tests/unit/litellm_core_utils/attribute_forwarder.py @@ -0,0 +1,6 @@ +class AttributeForwarder: + def __init__(self, target: object) -> None: + self._target = target + + def __getattr__(self, name: str) -> object: + return getattr(self._target, name) diff --git a/tests/unit/llms/compactifai/test_compactifai.py b/tests/unit/llms/compactifai/test_compactifai.py index 1367c703fda..56d5374f383 100644 --- a/tests/unit/llms/compactifai/test_compactifai.py +++ b/tests/unit/llms/compactifai/test_compactifai.py @@ -2,7 +2,7 @@ import json import os import sys from unittest.mock import AsyncMock, patch -from typing import Optional +from typing import Final, Optional import httpx import pytest @@ -271,3 +271,38 @@ async def test_compactifai_async_completion(respx_mock): assert response.choices[0].message.content == "Async response from CompactifAI" assert response.usage.total_tokens == 23 + + +@pytest.mark.parametrize( + ("body", "json_mode", "error", "message"), + [ + ([], None, TypeError, r"argument after \*\* must be a mapping, not list"), + ({"choices": None}, True, TypeError, "'NoneType' object is not iterable"), + ({"choices": [{"message": "hi"}]}, True, AttributeError, "'str' object has no attribute 'get'"), + ({"choices": [{"message": {"tool_calls": [5]}}]}, True, TypeError, "'int' object is not subscriptable"), + ], +) +def test_compactifai_malformed_response_body_raises_the_error_of_the_failing_lookup(body, json_mode, error, message): + from unittest.mock import MagicMock + + from litellm.llms.compactifai.chat.transformation import CompactifAIChatConfig + + raw_response: Final = httpx.Response( + 200, json=body, request=httpx.Request("POST", "https://api.compactif.ai/v1/chat/completions") + ) + + with pytest.raises(error, match=message) as raised: + CompactifAIChatConfig().transform_response( + model="cai-llama-3-1-8b-slim", + raw_response=raw_response, + model_response=ModelResponse(), + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + json_mode=json_mode, + ) + + assert raised.type is error diff --git a/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py b/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py index aff90549a7b..0951ba6ba92 100644 --- a/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py +++ b/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py @@ -1,6 +1,7 @@ import io import os import pathlib +from typing import Final from unittest.mock import MagicMock import httpx @@ -594,3 +595,24 @@ def test_transform_response_wraps_malformed_words_with_the_raw_body(words: objec DeepgramAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) assert str(exc_info.value).endswith(f"\nResponse: {raw_response.text}") + + +@pytest.mark.parametrize( + ("payload", "error"), + [ + ({"results": None}, "'NoneType' object is not subscriptable"), + (_deepgram_payload({"transcript": "hello", "words": 5}, {}), "object of type 'int' has no len()"), + ( + {**_deepgram_payload({"transcript": "hello"}, {}), "metadata": None}, + "'NoneType' object is not subscriptable", + ), + ], +) +def test_transform_response_wraps_the_error_of_the_failing_lookup_in_a_malformed_body(payload: object, error: str): + raw_response: Final = httpx.Response(200, json=payload) + + with pytest.raises(ValueError, match="Error transforming Deepgram response: ") as exc_info: + DeepgramAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert exc_info.type is ValueError + assert str(exc_info.value) == f"Error transforming Deepgram response: {error}\nResponse: {raw_response.text}" diff --git a/tests/unit/llms/edenai/embedding/test_edenai_embedding_transformation.py b/tests/unit/llms/edenai/embedding/test_edenai_embedding_transformation.py index efdb428db32..ee6ee187d64 100644 --- a/tests/unit/llms/edenai/embedding/test_edenai_embedding_transformation.py +++ b/tests/unit/llms/edenai/embedding/test_edenai_embedding_transformation.py @@ -98,6 +98,12 @@ class TestEmbedding: class TestErrors: + def test_a_non_object_body_fails_as_an_invalid_response_object(self, eden_key, respx_mock): + respx_mock.post(EDEN_EMBEDDINGS_URL).mock(return_value=httpx.Response(200, json=["unexpected"])) + + with pytest.raises(litellm.APIConnectionError, match="Invalid response object"): + litellm.embedding(model=MODEL, input="hello") + def test_middleware_401_maps_to_authentication_error(self, eden_key, respx_mock): respx_mock.post(EDEN_EMBEDDINGS_URL).mock(return_value=httpx.Response(401, json={"detail": "Invalid token."})) diff --git a/tests/unit/llms/edenai/image_generation/test_edenai_image_generation_transformation.py b/tests/unit/llms/edenai/image_generation/test_edenai_image_generation_transformation.py index d7b32affc70..4e0673b8c27 100644 --- a/tests/unit/llms/edenai/image_generation/test_edenai_image_generation_transformation.py +++ b/tests/unit/llms/edenai/image_generation/test_edenai_image_generation_transformation.py @@ -117,6 +117,12 @@ class TestImageGeneration: class TestErrors: + def test_a_non_object_body_fails_as_an_invalid_response_object(self, eden_key, respx_mock): + respx_mock.post(EDEN_IMAGES_URL).mock(return_value=httpx.Response(200, json=["unexpected"])) + + with pytest.raises(litellm.APIConnectionError, match="Invalid response object"): + litellm.image_generation(model=MODEL, prompt="a red square") + def test_middleware_401_maps_to_authentication_error(self, eden_key, respx_mock): respx_mock.post(EDEN_IMAGES_URL).mock(return_value=httpx.Response(401, json={"detail": "Invalid token."})) diff --git a/tests/unit/llms/gemini/google_genai/test_google_genai_transformation.py b/tests/unit/llms/gemini/google_genai/test_google_genai_transformation.py index da8ab138416..175ce040d9f 100644 --- a/tests/unit/llms/gemini/google_genai/test_google_genai_transformation.py +++ b/tests/unit/llms/gemini/google_genai/test_google_genai_transformation.py @@ -106,6 +106,26 @@ def test_generate_content_response_parses_function_call() -> None: } +@pytest.mark.parametrize( + ("body", "message"), + [ + ([], r"argument after \*\* must be a mapping, not list"), + (["candidates"], "list indices must be integers or slices, not str"), + ], +) +def test_generate_content_response_non_object_body_raises_the_error_of_the_failing_lookup( + body: JsonValue, message: str +) -> None: + with pytest.raises(TypeError, match=message) as raised: + GoogleGenAIConfig().transform_generate_content_response( + model="gemini-2.5-flash", + raw_response=httpx.Response(200, json=body), + logging_obj=SimpleNamespace(model_call_details={}), + ) + + assert raised.type is TypeError + + def test_generate_content_request_preserves_tool_declarations() -> None: tools: Final = [ { diff --git a/tests/unit/llms/gemini/image_edit/test_gemini_image_edit_transformation.py b/tests/unit/llms/gemini/image_edit/test_gemini_image_edit_transformation.py index bd9b7006e58..c21dec9b6bb 100644 --- a/tests/unit/llms/gemini/image_edit/test_gemini_image_edit_transformation.py +++ b/tests/unit/llms/gemini/image_edit/test_gemini_image_edit_transformation.py @@ -241,6 +241,26 @@ class TestGeminiImageEditTransformation: ) assert logging_usage["completion_tokens_details"]["image_tokens"] == 1716 + @pytest.mark.parametrize( + ("body", "error", "message"), + [ + (["not", "an", "object"], AttributeError, "'list' object has no attribute 'get'"), + ({"candidates": None}, TypeError, "'NoneType' object is not iterable"), + ({"candidates": [{"content": {"parts": None}}]}, TypeError, "'NoneType' object is not iterable"), + ], + ) + def test_transform_image_edit_response_malformed_body_raises_the_error_of_the_failing_lookup( + self, body: object, error: type[Exception], message: str + ) -> None: + with pytest.raises(error, match=message) as raised: + self.config.transform_image_edit_response( + model=self.model, + raw_response=httpx.Response(200, json=body), + logging_obj=self.logging_obj, + ) + + assert raised.type is error + def test_transform_image_edit_request_without_image_raises(self) -> None: optional_params = {} diff --git a/tests/unit/llms/gemini/test_gemini_common_utils.py b/tests/unit/llms/gemini/test_gemini_common_utils.py index 196b912752b..20c36878fef 100644 --- a/tests/unit/llms/gemini/test_gemini_common_utils.py +++ b/tests/unit/llms/gemini/test_gemini_common_utils.py @@ -355,6 +355,25 @@ class TestGeminiModelInfo: assert result == expected +@pytest.mark.parametrize( + ("body", "error", "message"), + [ + ([], TypeError, "list indices must be integers or slices, not str"), + ({"models": None}, TypeError, "'NoneType' object is not iterable"), + ({"models": [{"name": 1}]}, AttributeError, "'int' object has no attribute 'replace'"), + ], +) +def test_get_models_malformed_listing_raises_the_error_of_the_failing_lookup( + body: JsonValue, error: type[Exception], message: str, respx_mock: respx.MockRouter +) -> None: + respx_mock.get("https://models.test/v1beta/models").respond(200, json=body) + + with pytest.raises(error, match=message) as raised: + GeminiModelInfo().get_models(api_key="test-key", api_base="https://models.test") + + assert raised.type is error + + class TestGoogleAIStudioTokenCounter: """Test suite for GoogleAIStudioTokenCounter class""" diff --git a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py index 0a5039791ee..8995b3cb068 100644 --- a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py @@ -8,6 +8,7 @@ from litellm.llms.anthropic.pass_through.responses_adapters.transformation impor from litellm.llms.openai.responses.count_tokens.transformation import ( OpenAICountTokensConfig, ) +from tests.unit.litellm_core_utils.attribute_forwarder import AttributeForwarder def test_transform_basic_request(): @@ -207,6 +208,23 @@ def test_messages_to_responses_input_with_tool(): } +def test_messages_to_responses_input_reads_a_tool_call_that_forwards_its_fields(): + tool_call: Final = {"id": "call_123", "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}} + messages: Final = [ + {"role": "user", "content": "What's the weather?"}, + {"role": "assistant", "content": None, "tool_calls": [AttributeForwarder(tool_call)]}, + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items[1] == { + "type": "function_call", + "call_id": "call_123", + "name": "get_weather", + "arguments": '{"city": "Paris"}', + } + + def test_messages_to_responses_input_preserves_images(): """An image block must survive the round trip, or OpenAI counts only the text. diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 2f10fd6fd0d..0d1b1aca9bc 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -47,6 +47,7 @@ from litellm.types.responses.main import ( OutputToolSearchCall, ) from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs +from tests.unit.litellm_core_utils.attribute_forwarder import AttributeForwarder class MockGuardrail(CustomGuardrail): @@ -1783,6 +1784,22 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: assert events[4]["item"]["content"][0]["text"] == "hello [MASKED]" assert events[5]["response"]["output"][0]["content"][0]["text"] == "hello [MASKED]" + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_syncs_events_that_forward_their_fields(self): + handler: Final = OpenAIResponsesHandler() + plain_events: Final = self._ended_stream_events() + forwarded_delta: Final = AttributeForwarder(plain_events[0]) + events: Final = [forwarded_delta, *plain_events[1:]] + + await handler.process_output_streaming_response( + responses_so_far=events, + guardrail_to_apply=self._masking_guardrail(), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert forwarded_delta.delta == "hello [MASKED]" + @staticmethod def _ended_function_call_stream_events() -> List[dict]: def item(arguments: str, status: str) -> dict: diff --git a/tests/unit/llms/runwayml/image_generation/__init__.py b/tests/unit/llms/runwayml/image_generation/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/runwayml/image_generation/test_runway_image_generation_transformation.py b/tests/unit/llms/runwayml/image_generation/test_runway_image_generation_transformation.py new file mode 100644 index 00000000000..7eec9541b98 --- /dev/null +++ b/tests/unit/llms/runwayml/image_generation/test_runway_image_generation_transformation.py @@ -0,0 +1,87 @@ +from typing import Final +from unittest.mock import MagicMock + +import httpx +import pytest + +from litellm.llms.runwayml.image_generation.transformation import RunwayMLImageGenerationConfig +from litellm.types.utils import ImageResponse + +TASK_URL = "https://api.dev.runwayml.com/v1/tasks/task_1" + + +@pytest.mark.respx() +def test_output_dict_with_non_string_values_still_yields_the_image(respx_mock): + respx_mock.get(TASK_URL).respond( + json={ + "id": "task_1", + "status": "SUCCEEDED", + "output": [{"url": "https://cdn.example/image.png", "b64_json": None, "width": 1024}], + }, + status_code=200, + ) + created: Final = httpx.Response( + 200, + json={"id": "task_1", "status": "PENDING"}, + request=httpx.Request( + "POST", "https://api.dev.runwayml.com/v1/text_to_image", headers={"Authorization": "Bearer k"} + ), + ) + + image_response: Final = RunwayMLImageGenerationConfig().transform_image_generation_response( + model="gen4_image", + raw_response=created, + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert [image.url for image in image_response.data] == ["https://cdn.example/image.png"] + + +def test_non_object_task_body_raises_the_base_error_class(): + created: Final = httpx.Response( + 200, + json=["unexpected"], + request=httpx.Request( + "POST", "https://api.dev.runwayml.com/v1/text_to_image", headers={"Authorization": "Bearer k"} + ), + ) + + with pytest.raises(AttributeError): + RunwayMLImageGenerationConfig().transform_image_generation_response( + model="gen4_image", + raw_response=created, + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + +@pytest.mark.asyncio +async def test_async_non_object_task_body_raises_the_base_error_class(): + created: Final = httpx.Response( + 200, + json=["unexpected"], + request=httpx.Request( + "POST", "https://api.dev.runwayml.com/v1/text_to_image", headers={"Authorization": "Bearer k"} + ), + ) + + with pytest.raises(AttributeError): + await RunwayMLImageGenerationConfig().async_transform_image_generation_response( + model="gen4_image", + raw_response=created, + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) diff --git a/tests/unit/proxy/auth/test_mcp_ip_filtering.py b/tests/unit/proxy/auth/test_mcp_ip_filtering.py index 4c219760762..c403daf733f 100644 --- a/tests/unit/proxy/auth/test_mcp_ip_filtering.py +++ b/tests/unit/proxy/auth/test_mcp_ip_filtering.py @@ -6,6 +6,7 @@ external callers only see servers with available_on_public_internet=True. """ import logging +from typing import Final from unittest.mock import MagicMock, patch import pytest @@ -176,6 +177,19 @@ class TestMCPClientIPExtraction: assert result == "203.0.113.5" + def test_non_boolean_use_x_forwarded_for_never_trusts_xff(self): + request: Final = _make_request("10.0.0.5", "203.0.113.9, 10.0.0.5") + + result: Final = IPAddressUtils.get_mcp_client_ip( + request, + general_settings={ + "use_x_forwarded_for": "false", + "mcp_trusted_proxy_ranges": ["10.0.0.0/8"], + }, + ) + + assert result == "10.0.0.5" + def _make_request(client_host, xff): request = MagicMock(spec=Request) diff --git a/tests/unit/proxy/client/cli/test_encryption_commands.py b/tests/unit/proxy/client/cli/test_encryption_commands.py index 3a86eb82593..6685d29fb7e 100644 --- a/tests/unit/proxy/client/cli/test_encryption_commands.py +++ b/tests/unit/proxy/client/cli/test_encryption_commands.py @@ -5,6 +5,8 @@ check vs POST migrate, dry-run param) and its residual-state messaging without a live proxy. """ +from typing import Final + import pytest from click.testing import CliRunner @@ -69,3 +71,20 @@ def test_migrate_reports_residual_legacy(runner): result = runner.invoke(cli_main.cli, ["encryption", "migrate", "--check"]) assert result.exit_code == 0, result.output assert "Residual legacy values remaining: 3" in result.output + + +@pytest.mark.parametrize("residual", [-1, float("nan")]) +def test_migrate_prints_no_residual_status_for_a_residual_below_zero_or_nan(runner, residual): + _FakeHTTPClient.response = {"status": "success", "report": {"residual_legacy": residual}} + result: Final = runner.invoke(cli_main.cli, ["encryption", "migrate", "--check"]) + assert result.exit_code == 0, result.output + assert "No legacy values remaining" not in result.output + assert "Residual legacy values remaining" not in result.output + + +@pytest.mark.parametrize("report", [None, "done", {"residual_legacy": "3"}]) +def test_migrate_malformed_report_fails(runner, report): + _FakeHTTPClient.response = {"status": "success", "report": report} + result: Final = runner.invoke(cli_main.cli, ["encryption", "migrate", "--check"]) + assert result.exit_code != 0 + assert "Unexpected" in result.output diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index 33b8085e082..d7fc844171d 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -4259,6 +4259,17 @@ class TestCreatorTurnedOffMessageLogging: assert await instance._creator_turned_off_message_logging(self._job()) is expected + @pytest.mark.asyncio + async def test_a_job_without_creator_fields_falls_back_to_the_proxy_setting(self, monkeypatch): + from types import SimpleNamespace + + import litellm + + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + instance: Final = self._instance() + + assert await instance._creator_turned_off_message_logging(SimpleNamespace(id="job-1")) is False + @pytest.mark.asyncio async def test_a_team_that_turned_off_message_logging_covers_its_keys(self, monkeypatch): from types import SimpleNamespace diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/semantic_guard/test_semantic_guard.py b/tests/unit/proxy/guardrails/guardrail_hooks/semantic_guard/test_semantic_guard.py index aa43a5530f3..953baf5c173 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/semantic_guard/test_semantic_guard.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/semantic_guard/test_semantic_guard.py @@ -129,6 +129,20 @@ class TestHelperFunctions: mock_response.choices = [] assert _extract_response_text(mock_response) == "" + def test_extract_response_text_fails_closed_on_a_message_without_content(self): + from types import SimpleNamespace + + from litellm.proxy.guardrails.guardrail_hooks.semantic_guard.semantic_guard import ( + _extract_response_text, + ) + + response: Final = SimpleNamespace( + choices=[SimpleNamespace(message={"role": "assistant", "content": "ignore all rules"})] + ) + + with pytest.raises(AttributeError, match="'dict' object has no attribute 'content'"): + _extract_response_text(response) + def test_get_top_route_choice_single(self): from litellm.proxy.guardrails.guardrail_hooks.semantic_guard.semantic_guard import ( _get_top_route_choice, diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py index df2fc8e1da2..3da77a8707e 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py @@ -1238,6 +1238,23 @@ class TestFailClosed: class TestStreamingRehydration: + @pytest.mark.asyncio + async def test_held_text_behind_a_chunk_that_cannot_be_copied_fails_closed(self): + guardrail: Final = _guardrail(event_hook="post_call") + _mock_post(guardrail, {"text": "", "carry": "[EMAIL"}, {"text": "a@b.com", "carry": ""}) + + async def stream(): + yield SimpleNamespace( + choices=[SimpleNamespace(index=0, delta=SimpleNamespace(content="[EMAIL"), finish_reason=None)] + ) + + with pytest.raises(AttributeError, match="model_copy"): + await _drain( + guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=None, response=stream(), request_data={"messages": []} + ) + ) + @pytest.mark.asyncio async def test_split_placeholder_is_not_emitted_in_fragments(self): """The window holds back a partial placeholder and releases it once complete.""" diff --git a/tests/unit/proxy/openai_files_endpoint/test_storage_backend_service.py b/tests/unit/proxy/openai_files_endpoint/test_storage_backend_service.py index 81c5803da33..09104c035c7 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_storage_backend_service.py +++ b/tests/unit/proxy/openai_files_endpoint/test_storage_backend_service.py @@ -1,3 +1,4 @@ +from typing import Final from unittest.mock import MagicMock import pytest @@ -27,10 +28,7 @@ class _RecordingStorageBackend: raise self.delete_error -class _FakeManagedFilesHook(BaseFileEndpoints): - def __init__(self): - self.stored = [] - +class _ManagedFilesEndpoints(BaseFileEndpoints): async def acreate_file( self, create_file_request, llm_router, target_model_names_list, litellm_parent_otel_span, user_api_key_dict ): @@ -48,10 +46,23 @@ class _FakeManagedFilesHook(BaseFileEndpoints): async def afile_content(self, file_id, litellm_parent_otel_span, llm_router, **data): raise NotImplementedError + +class _FakeManagedFilesHook(_ManagedFilesEndpoints): + def __init__(self): + self.stored = [] + async def store_unified_file_id(self, **kwargs): self.stored.append(kwargs) +class _ForwardingManagedFilesHook(_ManagedFilesEndpoints): + def __init__(self, writer: _FakeManagedFilesHook): + self._writer = writer + + def __getattr__(self, name: str) -> object: + return getattr(self._writer, name) + + class _FailingManagedFilesHook(_FakeManagedFilesHook): async def store_unified_file_id(self, **kwargs): raise RuntimeError("db down") @@ -142,6 +153,24 @@ async def test_upload_with_target_model_names_and_hook_stores_unified_id(monkeyp } +@pytest.mark.asyncio +async def test_upload_stores_the_unified_id_through_a_hook_that_forwards_its_writer(monkeypatch): + backend: Final = _RecordingStorageBackend() + monkeypatch.setattr(storage_backend_service, "get_storage_backend", lambda name, prisma_client=None: backend) + writer: Final = _FakeManagedFilesHook() + + file_object: Final = await StorageBackendFileService.upload_file_to_storage_backend( + file_data=_file_data(), + target_storage="azure_storage", + target_model_names=["gpt-x"], + purpose="batch", + proxy_logging_obj=_FakeProxyLogging(hook=_ForwardingManagedFilesHook(writer)), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + ) + + assert [stored["file_id"] for stored in writer.stored] == [file_object.id] + + @pytest.mark.asyncio async def test_upload_hands_the_prisma_client_to_the_storage_backend_factory(monkeypatch: pytest.MonkeyPatch): backend = _RecordingStorageBackend() diff --git a/tests/unit/proxy/proxy_server/test_openapi_customization.py b/tests/unit/proxy/proxy_server/test_openapi_customization.py index 6c588976efc..0eacf333960 100644 --- a/tests/unit/proxy/proxy_server/test_openapi_customization.py +++ b/tests/unit/proxy/proxy_server/test_openapi_customization.py @@ -36,6 +36,7 @@ from litellm.proxy.proxy_server import ( get_openapi_schema, mount_swagger_ui, ) +from tests.unit.litellm_core_utils.attribute_forwarder import AttributeForwarder from .conftest import normalize @@ -209,6 +210,17 @@ def test_inject_websocket_stubs_into_openapi_schema_adds_stub(): } +def test_inject_websocket_stubs_into_openapi_schema_documents_query_params_that_forward_their_fields(): + query_param: Final = AttributeForwarder(SimpleNamespace(name="session_id", required=True)) + route: Final = SimpleNamespace(path="/ws/chat", name="ws_chat", dependant=SimpleNamespace(query_params=[query_param])) + + result: Final = _inject_websocket_stubs_into_openapi_schema({"paths": {}}, [route]) + + assert result["paths"]["/ws/chat"]["get"]["parameters"] == [ + {"name": "session_id", "in": "query", "required": True, "schema": {"type": "string"}} + ] + + def test_inject_websocket_stubs_into_openapi_schema_does_not_overwrite_existing_get(): # Existing GET on the same path must not be replaced by the stub. existing_get = {"summary": "real http get", "operationId": "real_get"} From 4710c3b30e5c1459dbb11a060ad733ee06f06c18 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:34:32 -0700 Subject: [PATCH 009/155] fix(router): route decisions calls only to deployments whose provider supports decisions (#48175) * fix(router): route decisions calls only to deployments whose provider supports decisions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): account for indirect route helper coverage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): pin the chat leg so the mixed-group check is deterministic Weights pick the Bedrock leg for chat instead of relying on simple-shuffle landing on both legs. The route filter now passes the candidate list through unchanged in type, so the pre-existing selector lines lose their pyright ignores Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: joshua Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 91 +++++++++++- .../router_code_coverage.py | 3 + tests/unit/test_router/test_router.py | 133 +++++++++++++++++- 3 files changed, 220 insertions(+), 7 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 40c5c8afa6c..25591a435f9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -438,6 +438,7 @@ from litellm.utils import ( EmbeddingResponse, ModelInfoLookupMiss, ModelResponse, + ProviderConfigManager, Rules, TextCompletionResponse, function_setup, @@ -567,6 +568,13 @@ CLAUDE_CODE_SESSION_ROUTER_BINDING_TARGET: Final = "claude_code_session_router_b _PRE_CALL_CHECK_FILTERED_PARAMS: Final = frozenset({"response_format"}) _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"}) _video_call_deployment_id: Final[ContextVar[str | None]] = ContextVar("video_call_deployment_id", default=None) +_routed_call_type: Final[ContextVar[str | None]] = ContextVar("routed_call_type", default=None) +_ROUTE_CAPABILITY_RESOLVERS: Final[Mapping[str, Callable[[str, LlmProviders], object | None]]] = MappingProxyType( + { + "adecisions": ProviderConfigManager.get_provider_decisions_config, + "decisions": ProviderConfigManager.get_provider_decisions_config, + } +) _RUNTIME_TOGGLEABLE_PRE_CALL_CHECKS: Final[Mapping[str, type[CustomLogger]]] = MappingProxyType( { "prompt_caching": PromptCachingDeploymentCheck, @@ -823,6 +831,7 @@ def _mid_stream_retry_trigger(error: "MidStreamFallbackError") -> Exception: _RestT: Final = TypeVar("_RestT", Iterator[object], AsyncIterator[object]) +_RouteDeploymentsT: Final = TypeVar("_RouteDeploymentsT", bound=Sequence[Mapping[str, object]] | Mapping[str, object]) @dataclass(frozen=True, slots=True) @@ -6193,7 +6202,7 @@ class Router: try: parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) try: - with self._video_call_pin(handler_name, kwargs): + with self._video_call_pin(handler_name, kwargs), self._route_capability_pin(handler_name): deployment = await self.async_get_available_deployment( # rebind-ok: set on success, see pre-init model=model, request_kwargs=kwargs, @@ -6921,6 +6930,15 @@ class Router: finally: _video_call_deployment_id.reset(token) + @staticmethod + @contextlib.contextmanager + def _route_capability_pin(handler_name: str | None) -> Generator[None]: + token: Final = _routed_call_type.set(handler_name) + try: + yield + finally: + _routed_call_type.reset(token) + @staticmethod def _narrow_to_video_deployment(healthy_deployments: Sequence[_DeploymentT]) -> Sequence[_DeploymentT]: """ @@ -6954,7 +6972,7 @@ class Router: kwargs=kwargs, metadata_variable_name=metadata_variable_name, ) - with self._video_call_pin(handler_name, kwargs): + with self._video_call_pin(handler_name, kwargs), self._route_capability_pin(handler_name): deployment: Final = self.get_available_deployment( model=model, messages=kwargs.get("messages", None), @@ -14209,6 +14227,52 @@ class Router: ) return False + @staticmethod + def _deployment_serves_route( + deployment: DeploymentTypedDict, + group: str, + resolver: Callable[[str, LlmProviders], object | None], + ) -> bool: + if "litellm_params" not in deployment: + return True + + try: + model, resolved_custom_llm_provider = Router._capability_model_and_provider( + deployment["litellm_params"], group + ) + provider: Final = next( + (member for member in LlmProviders if member.value == resolved_custom_llm_provider), + None, + ) + return provider is not None and resolver(model, provider) is not None + except Exception as e: # noqa: BLE001 # best-effort capability filter + verbose_router_logger.debug( + "litellm.router.py::_deployment_serves_route: keeping deployment for model=%s. Got - %s", group, e + ) + return True + + @staticmethod + def _without_deployments_that_cannot_serve_route( + healthy_deployments: _RouteDeploymentsT, group: str + ) -> _RouteDeploymentsT: + handler_name: Final = _routed_call_type.get() + resolver: Final = _ROUTE_CAPABILITY_RESOLVERS.get(handler_name) if handler_name is not None else None + if resolver is None or isinstance(healthy_deployments, Mapping): + return healthy_deployments + + filtered_deployments: Final = [ + deployment + for deployment in healthy_deployments + if Router._deployment_serves_route( + cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape + group, + resolver, + ) + ] + if not filtered_deployments: + return healthy_deployments + return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates + @staticmethod def _request_carries_image( messages: list[dict[str, str]] | None, # mutable-ok: handed to resolve_structured_messages as-is @@ -14845,6 +14909,13 @@ class Router: if verbose_router_logger.isEnabledFor(logging.DEBUG): verbose_router_logger.debug("healthy_deployments after web search filter: %s", healthy_deployments) + healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: route filter narrows + self._without_deployments_that_cannot_serve_route( + healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type + model, + ) + ) + if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: raise litellm.ServiceUnavailableError( @@ -14859,10 +14930,12 @@ class Router: ) # Health-check-based filtering (before cooldown) - healthy_deployments = await self._async_filter_health_check_unhealthy_deployments( - healthy_deployments=healthy_deployments, - parent_otel_span=parent_otel_span, - health_check_probe=health_check_probe, + healthy_deployments = ( # rebind-ok: filtered candidates enter health checks + await self._async_filter_health_check_unhealthy_deployments( + healthy_deployments=healthy_deployments, + parent_otel_span=parent_otel_span, + health_check_probe=health_check_probe, + ) ) routing_read_batch: Final = RoutingReadBatch.active() @@ -16113,6 +16186,12 @@ class Router: specific_deployment=specific_deployment, request_kwargs=request_kwargs, ) + healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: route filter narrows + self._without_deployments_that_cannot_serve_route( + healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type + model, + ) + ) strategy, strategy_selector = self._get_routing_context(model, request_kwargs) routing_kwargs: Final = _ROUTING_KWARGS_ADAPTER.validate_python(request_kwargs) pick_attributes: Final = _deployment_pick_attributes(model, routing_kwargs) diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index af2f386a362..b5e36055354 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -138,6 +138,9 @@ ignored_function_names = [ "_pre_call_check_requested_chat_params", # Tested through the unsupported-param pre-call check tests in test_router.py "_deployment_supports_param", # Tested through the deployment-capability pre-call check tests in test_router.py "_capability_model_and_provider", # Tested through the authenticating-provider pre-call check test in test_router.py + "_route_capability_pin", # Exercised through Decisions routing tests in test_router.py + "_deployment_serves_route", # Exercised through mixed-group Decisions tests in test_router.py + "_without_deployments_that_cannot_serve_route", # Exercised through Decisions routing tests in test_router.py "_exact_base_model_max_input_tokens", # Tested through get_router_model_info in test_router_model_cost_isolation.py "_base_model_widened_window", # Tested through get_router_model_info in test_router_model_cost_isolation.py "_provider_model_id", # Tested through get_router_model_info in test_router_model_cost_isolation.py diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index f06f9e08f7e..2288d4311a9 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -10,7 +10,7 @@ import os import sys import threading import warnings -from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Mapping +from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Iterator, Mapping from datetime import datetime, timedelta, timezone from pathlib import Path from types import MappingProxyType, SimpleNamespace @@ -108,6 +108,137 @@ if TYPE_CHECKING: from litellm.integrations.otel.logger import OpenTelemetryV2 +_ROUTER_DECISIONS_RESPONSE: Final[Mapping[str, object]] = { + "model": "gpt-6-luna", + "answers": [{"type": "predicate", "name": "damaged", "probability": 0.9}], + "usage": { + "input_tokens": 1, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 1, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 2, + }, +} + + +_OPENAI_DECISIONS_PARAMS: Final[Mapping[str, object]] = { + "model": "openai/gpt-6-luna", + "api_base": "https://api.openai.com/v1", + "api_key": "k", +} +_BEDROCK_DECISIONS_PARAMS: Final[Mapping[str, object]] = { + "model": "bedrock/us.openai.gpt-5.6-terra", + "aws_access_key_id": "fake", + "aws_secret_access_key": "fake", + "aws_region_name": "us-east-1", +} + + +def _decisions_leg(leg_id: str, params: Mapping[str, object], weight: int | None = None) -> dict[str, object]: + return { + "model_name": "decision-group", + "litellm_params": params if weight is None else {**params, "weight": weight}, + "model_info": {"id": leg_id}, + } + + +_OPENAI_DECISIONS_LEG: Final = _decisions_leg("openai-leg", _OPENAI_DECISIONS_PARAMS) +_BEDROCK_DECISIONS_LEG: Final = _decisions_leg("bedrock-leg", _BEDROCK_DECISIONS_PARAMS) + + +def _mixed_decisions_group_router(*legs: dict[str, object]) -> Router: + return Router(model_list=list(legs), routing_strategy="simple-shuffle") + + +@pytest.fixture +def _router_httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + yield + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.asyncio +async def test_decisions_on_a_mixed_group_never_routes_to_a_leg_without_decisions_support( + respx_mock: respx.MockRouter, + _router_httpx_transport: None, +) -> None: + router: Final = _mixed_decisions_group_router(_OPENAI_DECISIONS_LEG, _BEDROCK_DECISIONS_LEG) + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_ROUTER_DECISIONS_RESPONSE) + + for _ in range(20): + await router.adecisions( + model="decision-group", + input=[{"role": "user", "content": "The package arrived with a broken screen."}], + questions=[ + {"type": "predicate", "name": "damaged", "instructions": "Does the customer report a damaged item?"} + ], + ) + + assert route.call_count == 20 + + +def test_decisions_on_a_mixed_group_never_routes_to_a_leg_without_decisions_support_sync( + respx_mock: respx.MockRouter, +) -> None: + router: Final = _mixed_decisions_group_router(_OPENAI_DECISIONS_LEG, _BEDROCK_DECISIONS_LEG) + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_ROUTER_DECISIONS_RESPONSE) + + for _ in range(20): + router.decisions( + model="decision-group", + input=[{"role": "user", "content": "The package arrived with a broken screen."}], + questions=[ + {"type": "predicate", "name": "damaged", "instructions": "Does the customer report a damaged item?"} + ], + ) + + assert route.call_count == 20 + + +@pytest.mark.asyncio +async def test_decisions_on_a_group_with_no_capable_leg_still_returns_the_provider_error() -> None: + router: Final = Router( + model_list=[ + { + "model_name": "bedrock-only", + "litellm_params": { + "model": "bedrock/us.openai.gpt-5.6-terra", + "aws_access_key_id": "fake", + "aws_secret_access_key": "fake", + "aws_region_name": "us-east-1", + }, + "model_info": {"id": "bedrock-leg"}, + } + ] + ) + + with pytest.raises(litellm.BadRequestError, match="Unknown Decisions provider 'bedrock'"): + await router.adecisions( + model="bedrock-only", + input=[{"role": "user", "content": "The package arrived with a broken screen."}], + questions=[ + {"type": "predicate", "name": "damaged", "instructions": "Does the customer report a damaged item?"} + ], + ) + + +@pytest.mark.asyncio +async def test_chat_on_a_mixed_group_still_reaches_the_leg_without_decisions_support() -> None: + router: Final = _mixed_decisions_group_router( + _decisions_leg("openai-leg", _OPENAI_DECISIONS_PARAMS, weight=0), + _decisions_leg("bedrock-leg", _BEDROCK_DECISIONS_PARAMS, weight=1), + ) + + response: Final = await router.acompletion( + model="decision-group", + messages=[{"role": "user", "content": "Say hi"}], + mock_response="hi", + ) + + assert response._hidden_params["model_id"] == "bedrock-leg" + + def test_update_kwargs_does_not_mutate_defaults_and_merges_metadata(): # initialize a real Router (env‑vars can be empty) router = litellm.Router( From 08ef4f40b184e03bb3a27594d83a355aded5466e Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 24 Oct 2026 12:20:13 -0700 Subject: [PATCH 010/155] feat(ui): configure the default Usage time range (#48207) * feat(ui): configure the default Usage time range * fix(ui): retain the selected calendar preset during input sync * fix(ui): preserve fallback dates when applying across DST --- .../proxy_setting_endpoints.py | 7 ++ .../test_proxy_setting_endpoints.py | 102 ++++++++++++++++++ ui/litellm-dashboard/AGENTS.md | 2 +- .../components/UsagePageView.test.tsx | 55 ++++++++++ .../_components/components/UsagePageView.tsx | 22 ++-- .../usage/_components/usageUrlState.test.ts | 79 +++++++++++++- .../usage/_components/usageUrlState.ts | 11 +- .../src/app/(dashboard)/usage/page.test.tsx | 51 +++++++++ .../src/app/(dashboard)/usage/page.tsx | 24 ++++- .../UISettings/UISettings.test.tsx | 30 ++++++ .../AdminSettings/UISettings/UISettings.tsx | 47 ++++++++ .../shared/advanced_date_picker.test.tsx | 26 +++++ .../shared/advanced_date_picker.tsx | 62 ++--------- .../components/shared/date_range_presets.ts | 56 ++++++++++ 14 files changed, 508 insertions(+), 66 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/page.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/date_range_presets.ts diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index d51e1167888..e04ddbc75cc 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -8,6 +8,7 @@ from types import MappingProxyType from typing import ( Annotated, Final, + Literal, NamedTuple, Protocol, cast, # noqa: TID251 # prisma types Json columns as fields.Json but de-serializes them to plain python on read @@ -256,6 +257,11 @@ class UISettings(LiteLLMBaseModel): model_config = ConfigDict(extra="allow") + usage_default_time_range: Literal["today", "last_7_days", "last_30_days", "month_to_date", "year_to_date"] = Field( + default="last_7_days", + description="Default date range when opening Usage without an explicit URL range. Calendar ranges use the browser timezone.", + ) + disable_model_add_for_internal_users: bool = Field( default=False, description=( @@ -386,6 +392,7 @@ class UISettingsResponse(SettingsResponse): # Allowlist of UI settings that can be stored ALLOWED_UI_SETTINGS_FIELDS: Final = { + "usage_default_time_range", "disable_model_add_for_internal_users", "disable_team_admin_delete_team_user", "enabled_ui_pages_internal_users", diff --git a/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index 512f58380ea..0d558f206d8 100644 --- a/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -4345,3 +4345,105 @@ class TestMoyaiUrlSetting: await update_ui_settings({"moyai_url": None}, actor) assert persisted["moyai_url"] is None + + +@pytest.mark.parametrize("preset", ["today", "last_7_days", "last_30_days", "month_to_date", "year_to_date"]) +def test_usage_default_range_round_trips_without_overwriting_other_settings( + mock_auth, monkeypatch: pytest.MonkeyPatch, preset: str, +) -> None: + from types import SimpleNamespace + from unittest.mock import AsyncMock, MagicMock + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.config_resolvers import SettingsStore + + prisma: Final = MagicMock() + prisma.db.litellm_uisettings.find_unique = AsyncMock(return_value=SimpleNamespace( + ui_settings={"disable_custom_api_keys": True}, + )) + prisma.db.litellm_uisettings.upsert = AsyncMock() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server.proxy_config, "settings", SettingsStore("general_settings")) + actor: Final = UserAPIKeyAuth(user_id="test-admin", user_role=LitellmUserRoles.PROXY_ADMIN) + app.dependency_overrides[user_api_key_auth] = lambda: actor + try: + initial: Final = client.get("/get/ui_settings") + assert initial.json()["values"]["usage_default_time_range"] == "last_7_days" + response: Final = client.patch("/update/ui_settings", json={"usage_default_time_range": preset}) + assert response.status_code == 200, response.text + stored: Final = json.loads(prisma.db.litellm_uisettings.upsert.call_args.kwargs["data"]["update"]["ui_settings"]) + assert stored == {"disable_custom_api_keys": True, "usage_default_time_range": preset} + prisma.db.litellm_uisettings.find_unique.return_value = SimpleNamespace(ui_settings=stored) + fetched: Final = client.get("/get/ui_settings") + assert fetched.json()["values"]["usage_default_time_range"] == preset + assert fetched.json()["source"]["usage_default_time_range"] == "db" + finally: + app.dependency_overrides.clear() + + +@pytest.mark.parametrize("submitted", ["invalid", None, 7]) +def test_usage_default_range_rejects_invalid_values_without_writing( + mock_auth, monkeypatch: pytest.MonkeyPatch, submitted: object, +) -> None: + from unittest.mock import AsyncMock, MagicMock + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + prisma: Final = MagicMock() + prisma.db.litellm_uisettings.upsert = AsyncMock() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + actor: Final = UserAPIKeyAuth(user_id="test-admin", user_role=LitellmUserRoles.PROXY_ADMIN) + app.dependency_overrides[user_api_key_auth] = lambda: actor + try: + response: Final = client.patch("/update/ui_settings", json={"usage_default_time_range": submitted}) + assert response.status_code == 422, response.text + prisma.db.litellm_uisettings.upsert.assert_not_called() + finally: + app.dependency_overrides.clear() + + +@pytest.mark.parametrize("role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY]) +def test_usage_default_range_cannot_be_changed_by_non_admins( + mock_auth, monkeypatch: pytest.MonkeyPatch, role: LitellmUserRoles, +) -> None: + from unittest.mock import AsyncMock, MagicMock + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + prisma: Final = MagicMock() + prisma.db.litellm_uisettings.upsert = AsyncMock() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + actor: Final = UserAPIKeyAuth(user_id="test-user", user_role=role) + app.dependency_overrides[user_api_key_auth] = lambda: actor + try: + response: Final = client.patch("/update/ui_settings", json={"usage_default_time_range": "month_to_date"}) + assert response.status_code == 403, response.text + prisma.db.litellm_uisettings.upsert.assert_not_called() + finally: + app.dependency_overrides.clear() + + +def test_usage_default_range_reports_config_precedence(mock_auth, monkeypatch: pytest.MonkeyPatch) -> None: + from types import SimpleNamespace + from unittest.mock import AsyncMock, MagicMock + from litellm.proxy import proxy_server + from litellm.proxy.config_resolvers import SettingsStore + + prisma: Final = MagicMock() + prisma.db.litellm_uisettings.find_unique = AsyncMock(return_value=SimpleNamespace( + ui_settings={"usage_default_time_range": "today"}, + )) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + settings: Final = SettingsStore("general_settings") + settings.load_yaml({"usage_default_time_range": "month_to_date"}) + monkeypatch.setattr(proxy_server.proxy_config, "settings", settings) + response: Final = client.get("/get/ui_settings") + assert response.status_code == 200, response.text + assert response.json()["values"]["usage_default_time_range"] == "month_to_date" + assert response.json()["source"]["usage_default_time_range"] == "config" diff --git a/ui/litellm-dashboard/AGENTS.md b/ui/litellm-dashboard/AGENTS.md index b6fe5219022..bbf2e49aad7 100644 --- a/ui/litellm-dashboard/AGENTS.md +++ b/ui/litellm-dashboard/AGENTS.md @@ -30,7 +30,7 @@ Type tests are `*.test-d.ts` files run by the `types` vitest project (`npm run t -# This is NOT the Next.js you know +## This is NOT the Next.js you know This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` (resolved from this file's directory; in monorepos the `next` package may not be visible from the repo root) before writing any code. Heed deprecation notices. diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx index 24e9936a1d0..adb6d8791a7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx @@ -1914,6 +1914,61 @@ describe("UsagePage", () => { ); }); + it("queries the configured month and preserves later manual selections", async () => { + const now = new Date(2026, 9, 24, 12); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(now); + try { + renderWithProviders(); + await waitFor(() => + expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalledWith( + expect.objectContaining({ + startTime: new Date(2026, 9, 1), + endTime: new Date(2026, 9, 24, 23, 59, 59, 999), + }), + ), + ); + expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalledTimes(1); + fireEvent.click(screen.getByTestId("pick-a-different-range")); + await waitFor(() => + expect(mockUserDailyActivityAggregatedCall).toHaveBeenLastCalledWith( + expect.objectContaining({ + startTime: new Date(2024, 0, 1), + endTime: new Date(2024, 0, 8, 23, 59, 59, 999), + }), + ), + ); + } finally { + vi.useRealTimers(); + } + }); + + it.each(["?range=today", "?from=2025-03-01&to=2025-03-05"])( + "preserves explicit dates over configured defaults for %s", + async (searchParams) => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(new Date(2026, 9, 24, 12)); + try { + const view = renderWithProviders(, { + searchParams, + }); + const startTime = searchParams.includes("range=today") ? new Date(2026, 9, 24) : new Date(2025, 2, 1); + const endTime = searchParams.includes("range=today") + ? new Date(2026, 9, 24, 23, 59, 59, 999) + : new Date(2025, 2, 5, 23, 59, 59, 999); + await waitFor(() => + expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalledWith( + expect.objectContaining({ startTime, endTime }), + ), + ); + view.rerender(); + expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalledTimes(1); + } finally { + vi.useRealTimers(); + } + }, + ); + it("fetches the custom range in the URL and writes a newly picked one", async () => { const { lastQuery } = renderAt("?from=2025-03-01&to=2025-03-05"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx index 3ff758a3c18..d7d3b502b6f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx @@ -82,6 +82,7 @@ import { clientDateRangePatch, clientPanelPatch, dateRangeFromParams, + defaultUsageDateRange, dateRangePatch, entitySelectionPatch, selectedEntitiesFromParams, @@ -108,23 +109,24 @@ interface UsagePageProps { teams: Team[]; organizations: Organization[]; teamsLoading?: boolean; + defaultTimeRange?: string; } -const UsagePage: React.FC = ({ teams, organizations, teamsLoading = false }) => { +const UsagePage: React.FC = ({ teams, organizations, teamsLoading = false, defaultTimeRange }) => { const { accessToken, userRole, userId: userID, premiumUser, isLoading: authorizationLoading } = useAuthorized(); const [gatewayActivityData, setGatewayActivityData] = useState(null); - // Create initial dates outside of state to prevent recreation - const initialFromDate = useMemo(() => new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), []); - const initialToDate = useMemo(() => new Date(), []); + const defaultRange = useMemo(() => defaultUsageDateRange(defaultTimeRange), [defaultTimeRange]); const [urlParams, setUrlParams] = useQueryStates(usageUrlParsers); const { range, from, to } = urlParams; const dateValue = useMemo( - () => dateRangeFromParams({ range, from, to }) ?? { from: initialFromDate, to: initialToDate }, - [range, from, to, initialFromDate, initialToDate], + () => dateRangeFromParams({ range, from, to }) ?? defaultRange.value, + [range, from, to, defaultRange], ); + const presetShortLabel = range ?? (from === null && to === null ? defaultRange.preset : null); + const { client_range, client_from, client_to } = urlParams; const clientDateValue = useMemo( () => clientDateRangeFromParams({ client_range, client_from, client_to }) ?? dateValue, @@ -481,7 +483,11 @@ const UsagePage: React.FC = ({ teams, organizations, teamsLoadin )}
- + - - - )} - {models.isSuccess && models.data.length === 0 && ( - - No chat models are available for your account. - - )} - + + + Could not load models.{" "} + + + ); } From 5a8ab74d10984c977d0defe18a21470868d9ff5b Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Sat, 24 Oct 2026 12:30:01 -0700 Subject: [PATCH 013/155] fix(chat): remove hosted telemetry on chat submissions (#48225) --- .../src/app/chat/layout.integration.test.tsx | 5 - .../src/app/chat/layout.test.tsx | 15 +-- .../app/chat/page.images.integration.test.tsx | 28 ++--- .../app/chat/page.mcp.integration.test.tsx | 1 - ui/litellm-dashboard/src/app/chat/page.tsx | 4 - .../src/lib/http/chatTelemetry.test.ts | 43 ------- .../src/lib/http/chatTelemetry.ts | 16 --- .../src/lib/telemetry/chat.test.ts | 117 ------------------ .../src/lib/telemetry/chat.ts | 20 --- 9 files changed, 8 insertions(+), 241 deletions(-) delete mode 100644 ui/litellm-dashboard/src/lib/http/chatTelemetry.test.ts delete mode 100644 ui/litellm-dashboard/src/lib/http/chatTelemetry.ts delete mode 100644 ui/litellm-dashboard/src/lib/telemetry/chat.test.ts delete mode 100644 ui/litellm-dashboard/src/lib/telemetry/chat.ts diff --git a/ui/litellm-dashboard/src/app/chat/layout.integration.test.tsx b/ui/litellm-dashboard/src/app/chat/layout.integration.test.tsx index 12acf9192aa..a7189ea6cd7 100644 --- a/ui/litellm-dashboard/src/app/chat/layout.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/chat/layout.integration.test.tsx @@ -2,11 +2,8 @@ import userEvent from "@testing-library/user-event"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders, screen, testQueryClient } from "@/../tests/test-utils"; import { setServerRootPath } from "@/lib/serverRootPath"; -import { recordChatQuery } from "@/lib/telemetry/chat"; import ChatLayout from "./layout"; -vi.mock("@/lib/telemetry/chat", () => ({ recordChatQuery: vi.fn(async () => {}) })); - const { router, route } = vi.hoisted(() => ({ router: { push: vi.fn(), replace: vi.fn() }, route: { pathname: "/ui/chat" }, @@ -94,7 +91,6 @@ describe("Chat layout product navigation", () => { ); expect(screen.getByRole("heading", { name: "Chat content" })).toBeVisible(); - expect(recordChatQuery).not.toHaveBeenCalled(); await user.click(screen.getByRole("button", { name: "Switch workspace, current: Chat" })); expect(await screen.findByRole("menuitem", { name: "Observability" })).toBeInTheDocument(); expect(localStorage.getItem("litellm_plugin_mode")).toBe(plugin.name); @@ -103,7 +99,6 @@ describe("Chat layout product navigation", () => { expect(localStorage.getItem("litellm_plugin_mode")).toBe("ai-gateway"); expect(assign).toHaveBeenCalledWith(gateway); - expect(recordChatQuery).not.toHaveBeenCalled(); }, ); diff --git a/ui/litellm-dashboard/src/app/chat/layout.test.tsx b/ui/litellm-dashboard/src/app/chat/layout.test.tsx index 0b0cb96ddc9..a741a7725dd 100644 --- a/ui/litellm-dashboard/src/app/chat/layout.test.tsx +++ b/ui/litellm-dashboard/src/app/chat/layout.test.tsx @@ -3,7 +3,7 @@ import { act, fireEvent, screen } from "@testing-library/react"; import { renderWithProviders as render } from "@/../tests/test-utils"; import ChatLayout from "./layout"; -const { mockUseAuthorized, mockUseUISettings, mockReplace, mockUiHref, recordChatQuery, state } = vi.hoisted(() => { +const { mockUseAuthorized, mockUseUISettings, mockReplace, mockUiHref, state } = vi.hoisted(() => { const state = { enableChatUI: false, isUISettingsLoading: false, @@ -14,7 +14,6 @@ const { mockUseAuthorized, mockUseUISettings, mockReplace, mockUiHref, recordCha return { state, mockReplace: vi.fn(), - recordChatQuery: vi.fn(async () => {}), mockUiHref: vi.fn((segment: string) => `/mocked-ui/${segment}`), mockUseAuthorized: vi.fn(() => ({ accessToken: "token-123", @@ -38,7 +37,6 @@ vi.mock("next/navigation", () => ({ vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: mockUseAuthorized })); vi.mock("@/app/(dashboard)/hooks/uiSettings/useUISettings", () => ({ useUISettings: mockUseUISettings })); vi.mock("@/utils/uiHref", () => ({ uiHref: mockUiHref })); -vi.mock("@/lib/telemetry/chat", () => ({ recordChatQuery })); vi.mock("@/components/navbar", () => ({ default: () =>
})); vi.mock("@/contexts/ThemeContext", () => ({ ThemeProvider: ({ children }: { children: React.ReactNode }) => <>{children}, @@ -66,7 +64,6 @@ describe("ChatLayout", () => { state.userId = "user-1"; mockReplace.mockClear(); mockUiHref.mockClear(); - recordChatQuery.mockClear(); vi.restoreAllMocks(); vi.unstubAllGlobals(); }); @@ -81,7 +78,6 @@ describe("ChatLayout", () => { expect(screen.getByTestId("chat-shell")).toBeInTheDocument(); expect(screen.getByTestId("page-content")).toBeInTheDocument(); expect(mockReplace).not.toHaveBeenCalled(); - expect(recordChatQuery).not.toHaveBeenCalled(); }); it("should redirect without recording a query when enable_chat_ui is off", () => { @@ -93,7 +89,6 @@ describe("ChatLayout", () => { ); expect(screen.queryByTestId("chat-shell")).not.toBeInTheDocument(); expect(mockReplace).toHaveBeenCalledWith("/mocked-ui/"); - expect(recordChatQuery).not.toHaveBeenCalled(); }); it("should wait for UI settings without recording a query", () => { @@ -105,7 +100,6 @@ describe("ChatLayout", () => { ); expect(screen.queryByTestId("chat-shell")).not.toBeInTheDocument(); expect(mockReplace).not.toHaveBeenCalled(); - expect(recordChatQuery).not.toHaveBeenCalled(); }); it("should wait for the signed-in user's identity without recording a query", () => { @@ -118,7 +112,6 @@ describe("ChatLayout", () => { ); expect(screen.queryByTestId("chat-shell")).not.toBeInTheDocument(); expect(screen.queryByTestId("page-content")).not.toBeInTheDocument(); - expect(recordChatQuery).not.toHaveBeenCalled(); }); it("should hide Chat without recording a query when the signed-in user's identity cannot be confirmed", () => { @@ -131,7 +124,6 @@ describe("ChatLayout", () => { ); expect(screen.queryByTestId("chat-shell")).not.toBeInTheDocument(); expect(screen.queryByTestId("page-content")).not.toBeInTheDocument(); - expect(recordChatQuery).not.toHaveBeenCalled(); }); it("should not record a query or fetch telemetry when opening, focusing, or revisiting Chat", () => { @@ -140,11 +132,9 @@ describe("ChatLayout", () => { const view = render(Chat content); act(() => fireEvent.focus(window)); - expect(recordChatQuery).not.toHaveBeenCalled(); visibility.mockReturnValue("visible"); act(() => fireEvent(document, new Event("visibilitychange"))); - expect(recordChatQuery).not.toHaveBeenCalled(); view.rerender(Another Chat page); act(() => fireEvent.focus(window)); @@ -155,14 +145,12 @@ describe("ChatLayout", () => { fireEvent.focus(window); fireEvent(document, new Event("visibilitychange")); }); - expect(recordChatQuery).not.toHaveBeenCalled(); expect(fetch).not.toHaveBeenCalled(); }); it("should not record a query or fetch telemetry when the signed-in account changes", () => { state.enableChatUI = true; const view = render(Chat content); - expect(recordChatQuery).not.toHaveBeenCalled(); state.userId = "user-2"; view.rerender(Chat content); @@ -172,7 +160,6 @@ describe("ChatLayout", () => { state.isAuthorized = false; view.rerender(Chat content); act(() => fireEvent.focus(window)); - expect(recordChatQuery).not.toHaveBeenCalled(); expect(fetch).not.toHaveBeenCalled(); }); }); diff --git a/ui/litellm-dashboard/src/app/chat/page.images.integration.test.tsx b/ui/litellm-dashboard/src/app/chat/page.images.integration.test.tsx index f7bbf59c0cb..7e4401fc3ab 100644 --- a/ui/litellm-dashboard/src/app/chat/page.images.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/chat/page.images.integration.test.tsx @@ -3,11 +3,8 @@ import userEvent from "@testing-library/user-event"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; import { ChatShellProvider } from "@/contexts/ChatShellContext"; -import { recordChatQuery } from "@/lib/telemetry/chat"; import ChatConversationPage from "./page"; -vi.mock("@/lib/telemetry/chat", () => ({ recordChatQuery: vi.fn(async () => {}) })); - vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn(), replace: vi.fn() }), })); @@ -67,10 +64,10 @@ async function chooseModel(user: ReturnType, current: st } beforeEach(() => { + vi.stubEnv("NODE_ENV", "production"); localStorage.clear(); testQueryClient.clear(); requests.length = 0; - vi.mocked(recordChatQuery).mockClear(); models = [ { model_group: "research-chat", mode: "chat", providers: ["openai"] }, { model_group: "studio-image", mode: "image_generation", providers: ["openai"] }, @@ -109,6 +106,11 @@ beforeEach(() => { }); afterEach(() => { + for (const [input] of vi.mocked(fetch).mock.calls) { + const url = input instanceof Request ? input.url : String(input); + expect(new URL(url, window.location.origin).origin).toBe(window.location.origin); + } + vi.unstubAllEnvs(); vi.unstubAllGlobals(); localStorage.clear(); }); @@ -121,10 +123,9 @@ describe("chat image workflows", () => { await userEvent.setup().click(screen.getByRole("button", { name: "research-chat", exact: true })); expect(await screen.findByRole("button", { name: /^studio-image/ })).toBeInTheDocument(); expect(screen.getByRole("button", { name: "Send" })).toBeDisabled(); - expect(recordChatQuery).not.toHaveBeenCalled(); }); - it("should upload a document as a file part, preserve it on edit, and keep contents out of metrics", async () => { + it("should upload a document as a file part, preserve it on edit", async () => { const user = userEvent.setup(); renderChat(); await screen.findByRole("button", { name: "research-chat" }); @@ -145,7 +146,6 @@ describe("chat image workflows", () => { "data:application/vnd.openxmlformats-officedocument.wordprocessingml.document;base64,cHJpdmF0ZSBmaXh0dXJl", }; expect(requests[0].body).toMatchObject({ input: [{ role: "user", content: [filePart] }] }); - expect(recordChatQuery).toHaveBeenCalledExactlyOnceWith("media-test-user"); await user.click(screen.getByRole("button", { name: "Edit message" })); act(() => fireEvent.change(screen.getByRole("textbox", { name: "Edit message" }), { @@ -175,7 +175,6 @@ describe("chat image workflows", () => { await user.click(await screen.findByRole("button", { name: "Remove note-0.txt" })); expect(screen.queryByText("note-0.txt")).not.toBeInTheDocument(); expect(screen.getByRole("button", { name: "Send" })).toBeDisabled(); - expect(recordChatQuery).not.toHaveBeenCalled(); }); it("should restore the selected chat model after switching modes and reopening the page", async () => { @@ -201,7 +200,6 @@ describe("chat image workflows", () => { expect(screen.queryByRole("img", { name: "diagram.png" })).not.toBeInTheDocument(); expect(screen.getByRole("button", { name: "Send" })).toBeDisabled(); expect(requests).toHaveLength(0); - expect(recordChatQuery).not.toHaveBeenCalled(); }); it("should send the image and preserve it when editing and resending the message", async () => { @@ -212,7 +210,6 @@ describe("chat image workflows", () => { await user.type(screen.getByRole("textbox", { name: "Message", exact: true }), "Explain this diagram"); await user.click(screen.getByRole("button", { name: "Send" })); await screen.findByText("I can see the uploaded image."); - expect(recordChatQuery).toHaveBeenCalledExactlyOnceWith("media-test-user"); expect(requests[0].body).toMatchObject({ model: "research-chat", input: [ @@ -246,7 +243,6 @@ describe("chat image workflows", () => { }); expect(requests[1].body).not.toHaveProperty("previous_response_id"); expect(screen.queryByText("Explain this diagram", { selector: "p" })).not.toBeInTheDocument(); - expect(recordChatQuery).toHaveBeenCalledTimes(2); }); it("should preserve a chat turn when editing it while an image model is selected", async () => { @@ -267,7 +263,6 @@ describe("chat image workflows", () => { expect(screen.getByRole("heading", { name: "Explain the experiment" })).toBeInTheDocument(); expect(screen.getByText("I can see the uploaded image.")).toBeInTheDocument(); expect(requests).toHaveLength(1); - expect(recordChatQuery).toHaveBeenCalledExactlyOnceWith("media-test-user"); }); it("should continue an image conversation using the completed response context", async () => { @@ -288,7 +283,6 @@ describe("chat image workflows", () => { }); it("should generate and render an image through the selected Model Hub model", async () => { - vi.mocked(recordChatQuery).mockReturnValueOnce(new Promise(() => {})); const user = userEvent.setup(); renderChat(); await screen.findByRole("button", { name: "research-chat" }); @@ -301,7 +295,6 @@ describe("chat image workflows", () => { expect(requests).toEqual([ { pathname: "/images/generations", body: { model: "studio-image", prompt: "A blue research lab" } }, ]); - expect(recordChatQuery).toHaveBeenCalledExactlyOnceWith("media-test-user"); }); it("should list no image models when the account has none permitted", async () => { @@ -327,7 +320,6 @@ describe("chat image workflows", () => { await user.click(screen.getByRole("button", { name: "Generate image" })); expect(await screen.findByText(/Image access is disabled for this key/)).toBeInTheDocument(); expect(screen.queryByRole("img", { name: "Generated image 1" })).not.toBeInTheDocument(); - expect(recordChatQuery).toHaveBeenCalledExactlyOnceWith("media-test-user"); }); it("should abort image generation when the user stops the request", async () => { @@ -346,12 +338,10 @@ describe("chat image workflows", () => { fireEvent.keyDown(screen.getByRole("textbox", { name: "Message", exact: true }), { key: "Enter" }); }); expect(requests).toHaveLength(1); - expect(recordChatQuery).toHaveBeenCalledExactlyOnceWith("media-test-user"); await user.click(screen.getByRole("button", { name: "Stop request" })); expect(generationSignal?.aborted).toBe(true); expect(await screen.findByText("[stopped]")).toBeInTheDocument(); expect(screen.queryByRole("img", { name: "Generated image 1" })).not.toBeInTheDocument(); - expect(recordChatQuery).toHaveBeenCalledTimes(1); }); it("should reject unsupported pasted files without issuing a request", async () => { @@ -364,7 +354,6 @@ describe("chat image workflows", () => { }); expect(await screen.findByRole("alert")).toHaveTextContent("Choose a PNG, JPEG, WebP, or GIF image."); expect(requests).toHaveLength(0); - expect(recordChatQuery).not.toHaveBeenCalled(); }); it("should not count a blank submission as a query", async () => { @@ -377,7 +366,6 @@ describe("chat image workflows", () => { fireEvent.keyDown(screen.getByRole("textbox", { name: "Message", exact: true }), { key: "Enter" }); }); expect(requests).toHaveLength(0); - expect(recordChatQuery).not.toHaveBeenCalled(); }); it("should count one query for a message that streams text and executes multiple MCP tools", async () => { @@ -390,7 +378,6 @@ describe("chat image workflows", () => { await user.click(serverSwitch); await waitFor(() => expect(screen.getByRole("switch")).toBeChecked()); await user.click(screen.getByRole("button", { name: "Connect tools" })); - expect(recordChatQuery).not.toHaveBeenCalled(); act(() => { fireEvent.change(screen.getByRole("textbox", { name: "Message", exact: true }), { @@ -404,6 +391,5 @@ describe("chat image workflows", () => { expect(requests[0].body).toMatchObject({ tools: [{ type: "mcp", server_label: "research-tools", require_approval: "never" }], }); - expect(recordChatQuery).toHaveBeenCalledExactlyOnceWith("media-test-user"); }); }); diff --git a/ui/litellm-dashboard/src/app/chat/page.mcp.integration.test.tsx b/ui/litellm-dashboard/src/app/chat/page.mcp.integration.test.tsx index 4c4612f1ba8..367935a42a2 100644 --- a/ui/litellm-dashboard/src/app/chat/page.mcp.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/chat/page.mcp.integration.test.tsx @@ -7,7 +7,6 @@ import type { Conversation, MCPEvent } from "@/components/chat/types"; import ChatConversationPage from "./page"; vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn(), replace: vi.fn() }) })); -vi.mock("@/lib/telemetry/chat", () => ({ recordChatQuery: vi.fn(async () => {}) })); const userId = "mcp-replay-user"; const storageKey = `litellm_chat_history_v1:${userId}`; diff --git a/ui/litellm-dashboard/src/app/chat/page.tsx b/ui/litellm-dashboard/src/app/chat/page.tsx index 9b8f0275a07..85fc7133215 100644 --- a/ui/litellm-dashboard/src/app/chat/page.tsx +++ b/ui/litellm-dashboard/src/app/chat/page.tsx @@ -19,7 +19,6 @@ import { runChatRequest } from "@/components/chat/runChatRequest"; import { readChatAttachment, MAX_CHAT_ATTACHMENTS, CHAT_ATTACHMENT_ACCEPT } from "@/components/chat/attachmentHelpers"; import ChatFileChip from "@/components/chat/ChatFileChip"; import { useQueryState } from "nuqs"; -import { recordChatQuery } from "@/lib/telemetry/chat"; const LOCALSTORAGE_MODEL_KEY = "litellm_chat_selected_model"; @@ -75,7 +74,6 @@ export default function ChatConversationPage() { const router = useRouter(); const { accessToken, - userId, conversations, selectedMCPServers, setSelectedMCPServers, @@ -306,7 +304,6 @@ export default function ChatConversationPage() { onUpdate: (updates: AssistantMessageUpdate) => updateLastAssistantMessage(convId, updates), }; try { - void recordChatQuery(userId); await runChatRequest(request); } finally { if (abortControllerRef.current === requestController) { @@ -324,7 +321,6 @@ export default function ChatConversationPage() { mode, attachments, accessToken, - userId, createConversation, appendMessage, updateLastAssistantMessage, diff --git a/ui/litellm-dashboard/src/lib/http/chatTelemetry.test.ts b/ui/litellm-dashboard/src/lib/http/chatTelemetry.test.ts deleted file mode 100644 index 389406dd8b7..00000000000 --- a/ui/litellm-dashboard/src/lib/http/chatTelemetry.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; -import { captureChatRequest } from "./chatTelemetry"; - -afterEach(() => { - vi.restoreAllMocks(); - vi.unstubAllGlobals(); -}); - -describe("Chat telemetry transport", () => { - it("should send only pseudonymous user and request IDs without gateway credentials or referrer", async () => { - const send = vi.fn().mockResolvedValue(new Response("{}")); - const timeout = vi.spyOn(AbortSignal, "timeout"); - vi.stubGlobal("fetch", send); - await captureChatRequest("opaque-user", "unique-request"); - - expect(send).toHaveBeenCalledOnce(); - const [url, request] = send.mock.calls[0]; - expect(url).toBe("https://chat-telemetry.litellm.ai/v1/chat/events"); - const expectedRequest = { - method: "POST", - headers: { "Content-Type": "application/json" }, - credentials: "omit", - referrerPolicy: "no-referrer", - keepalive: true, - signal: expect.any(AbortSignal), - body: expect.any(String), - }; - expect(request).toEqual(expectedRequest); - expect(timeout).toHaveBeenCalledExactlyOnceWith(5000); - expect(JSON.parse(request.body)).toEqual({ - user_id: "opaque-user", - request_id: "unique-request", - }); - }); - - it("should reject an unsuccessful ingestion response for the caller to handle", async () => { - vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("", { status: 503 }))); - await expect(captureChatRequest("opaque-user", "unique-request")).rejects.toThrow( - "Chat telemetry was not accepted", - ); - expect(fetch).toHaveBeenCalledOnce(); - }); -}); diff --git a/ui/litellm-dashboard/src/lib/http/chatTelemetry.ts b/ui/litellm-dashboard/src/lib/http/chatTelemetry.ts deleted file mode 100644 index c2f172250ee..00000000000 --- a/ui/litellm-dashboard/src/lib/http/chatTelemetry.ts +++ /dev/null @@ -1,16 +0,0 @@ -export async function captureChatRequest(userId: string, requestId: string): Promise { - const request: RequestInit = { - method: "POST", - headers: { "Content-Type": "application/json" }, - credentials: "omit", - referrerPolicy: "no-referrer", - keepalive: true, - signal: AbortSignal.timeout(5000), - body: JSON.stringify({ - user_id: userId, - request_id: requestId, - }), - }; - const response = await fetch("https://chat-telemetry.litellm.ai/v1/chat/events", request); - if (!response.ok) throw new Error("Chat telemetry was not accepted"); -} diff --git a/ui/litellm-dashboard/src/lib/telemetry/chat.test.ts b/ui/litellm-dashboard/src/lib/telemetry/chat.test.ts deleted file mode 100644 index 599290ec7e4..00000000000 --- a/ui/litellm-dashboard/src/lib/telemetry/chat.test.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; - -const mocks = vi.hoisted(() => ({ capture: vi.fn(), fetch: vi.fn() })); -vi.mock("@/lib/http/chatTelemetry", () => ({ captureChatRequest: mocks.capture })); - -let chat: typeof import("./chat"); - -beforeEach(async () => { - vi.resetModules(); - vi.resetAllMocks(); - vi.stubEnv("NODE_ENV", "production"); - vi.stubGlobal("window", { location: { origin: "https://gateway.example.com" } }); - vi.stubGlobal("fetch", mocks.fetch); - mocks.capture.mockResolvedValue(undefined); - chat = await import("./chat"); -}); - -afterEach(() => { - vi.useRealTimers(); - vi.unstubAllEnvs(); - vi.unstubAllGlobals(); -}); - -describe("Chat request metrics", () => { - it("should do nothing when imported without a submission", () => { - expect(mocks.fetch).not.toHaveBeenCalled(); - expect(mocks.capture).not.toHaveBeenCalled(); - }); - - it("should emit by default with a stable user ID, fresh request IDs, and no backend settings request", async () => { - await chat.recordChatQuery("alice@example.com"); - await chat.recordChatQuery("alice@example.com"); - - expect(mocks.capture).toHaveBeenCalledTimes(2); - const ids = mocks.capture.mock.calls.map(([id]) => id); - expect(ids[0]).toMatch(/^[\da-f]{8}-[\da-f]{4}-5[\da-f]{3}-[\da-f]{4}-[\da-f]{12}$/); - expect(ids[0]).toBe(ids[1]); - const requestIds = mocks.capture.mock.calls.map(([, requestId]) => requestId); - for (const requestId of requestIds) { - expect(requestId).toMatch(/^[\da-f]{8}-[\da-f]{4}-4[\da-f]{3}-[\da-f]{4}-[\da-f]{12}$/); - } - expect(new Set(requestIds).size).toBe(2); - expect(mocks.capture.mock.calls.every((args) => args.length === 2)).toBe(true); - expect(JSON.stringify(mocks.capture.mock.calls)).not.toContain("alice@example.com"); - expect(JSON.stringify(mocks.capture.mock.calls)).not.toContain("gateway.example.com"); - expect(mocks.fetch).not.toHaveBeenCalled(); - }); - - it("should use the same user ID across reloads and browsers without stored identifiers", async () => { - await chat.recordChatQuery("alice"); - vi.resetModules(); - chat = await import("./chat"); - vi.stubGlobal("window", { location: { origin: "https://gateway.example.com" } }); - await chat.recordChatQuery("alice"); - expect(mocks.capture.mock.calls[0][0]).toBe(mocks.capture.mock.calls[1][0]); - }); - - it("should distinguish accounts and deployments", async () => { - await chat.recordChatQuery("alice"); - await chat.recordChatQuery("bob"); - vi.stubGlobal("window", { location: { origin: "https://another.example.com" } }); - await chat.recordChatQuery("alice"); - const ids = mocks.capture.mock.calls.map(([id]) => id); - expect(new Set(ids).size).toBe(3); - }); - - it("should preserve user identity across days while leaving event timestamps to the collector", async () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-10-23T23:59:00Z")); - await chat.recordChatQuery("alice"); - vi.setSystemTime(new Date("2026-10-24T00:01:00Z")); - await chat.recordChatQuery("alice"); - expect(mocks.capture.mock.calls[0][0]).toBe(mocks.capture.mock.calls[1][0]); - expect(mocks.capture.mock.calls[0][1]).not.toBe(mocks.capture.mock.calls[1][1]); - expect(JSON.stringify(mocks.capture.mock.calls)).not.toContain("2026-10-"); - }); - - it("should not collect from development, unauthenticated, or server rendering", async () => { - vi.stubEnv("NODE_ENV", "development"); - await chat.recordChatQuery("alice"); - vi.stubEnv("NODE_ENV", "production"); - await chat.recordChatQuery(""); - vi.stubGlobal("window", undefined); - await chat.recordChatQuery("alice"); - expect(mocks.fetch).not.toHaveBeenCalled(); - expect(mocks.capture).not.toHaveBeenCalled(); - }); - - it("should count requests on HTTP deployments without secure-context APIs or browser storage", async () => { - vi.stubGlobal("window", { location: { origin: "http://gateway.internal:4000" } }); - vi.stubGlobal("crypto", { getRandomValues: crypto.getRandomValues.bind(crypto) }); - vi.stubGlobal("localStorage", undefined); - await chat.recordChatQuery("alice"); - expect(mocks.capture).toHaveBeenCalledOnce(); - }); - - it("should separate deployments mounted under different root paths", async () => { - const { setServerRootPath } = await import("@/lib/serverRootPath"); - setServerRootPath("/team-a/"); - await chat.recordChatQuery("alice"); - setServerRootPath("team-a"); - await chat.recordChatQuery("alice"); - setServerRootPath("/team-b"); - await chat.recordChatQuery("alice"); - const ids = mocks.capture.mock.calls.map(([id]) => id); - expect(ids[0]).toBe(ids[1]); - expect(ids[0]).not.toBe(ids[2]); - }); - - it("should swallow transport failures without retrying requests", async () => { - mocks.capture.mockRejectedValueOnce(new Error("blocked")); - await expect(chat.recordChatQuery("alice")).resolves.toBeUndefined(); - expect(mocks.capture).toHaveBeenCalledTimes(1); - await chat.recordChatQuery("alice"); - expect(mocks.capture).toHaveBeenCalledTimes(2); - }); -}); diff --git a/ui/litellm-dashboard/src/lib/telemetry/chat.ts b/ui/litellm-dashboard/src/lib/telemetry/chat.ts deleted file mode 100644 index 706c49527fd..00000000000 --- a/ui/litellm-dashboard/src/lib/telemetry/chat.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { v4 as uuidv4, v5 as uuidv5 } from "uuid"; -import { serverRootPath } from "@/lib/serverRootPath"; -import { captureChatRequest } from "@/lib/http/chatTelemetry"; - -// Called only by Chat's accepted-submit handler. Daily unique users = DAU; -// event count = total requests. Page visits never count as activity. -export async function recordChatQuery(userId: string): Promise { - // Match docs: development previews never contribute to production metrics. - if (process.env.NODE_ENV !== "production" || typeof window === "undefined" || !userId) return; - try { - // Stable across browsers for an account on this deployment. The payload - // excludes its raw account ID, email, and gateway origin. - const root = serverRootPath.replace(/^\/+|\/+$/g, ""); - const identity = JSON.stringify([window.location.origin, root, userId]); - const distinctId = uuidv5(identity, uuidv5.URL); - await captureChatRequest(distinctId, uuidv4()); - } catch { - // Analytics failures must never interrupt a request or trigger a retry. - } -} From e6a32bbeebc35bce73c21a9a1c64cb5f3bc3b2b4 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sat, 24 Oct 2026 12:34:13 -0700 Subject: [PATCH 014/155] fix(proxy): buffer tool discoveries in Redis and write LiteLLM_ToolTable in one statement per drain, on its own lock (#48089) * fix(proxy): buffer tool discoveries in redis and let the leader write LiteLLM_ToolTable in one statement Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): declare the redis family on the tool discovery buffer and drop the blind except and cast Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): satisfy LIT010 and LIT014 in the tool discovery buffer Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): mark the aggregation fold Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): requeue drained tool rows to redis when the write provably did not apply Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): drain tool discoveries on their own lock and scheduler job, not the spend commit * chore(proxy): suppress the untyped apscheduler add_job on the tool discovery job * refactor(proxy): move the LiteLLM_ToolTable bulk upsert into its own module next to the other bulk upserts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): name the LiteLLM_ToolTable bulk write span so the prisma span check resolves it * refactor(proxy): move the tool discovery drain out of DBSpendUpdateWriter into tool_registry_writer * refactor(proxy): share db_write_failure_is_requeue_safe from exception_handler so the spend writer stays untouched --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 2 + litellm/proxy/_types.py | 1 + litellm/proxy/db/db_spend_update_writer.py | 41 +-- .../redis_update_buffer.py | 60 +++++ .../tool_discovery_queue.py | 61 ++--- litellm/proxy/db/exception_handler.py | 12 + litellm/proxy/db/tool_registry_writer.py | 120 +++++---- litellm/proxy/db/tool_table_bulk_upsert.py | 71 ++++++ litellm/proxy/proxy_server.py | 13 + litellm/proxy/utils.py | 14 ++ .../test_redis_update_buffer.py | 16 +- .../test_tool_discovery_queue.py | 73 +++--- .../proxy/db/test_db_spend_update_writer.py | 35 +-- .../proxy/db/test_tool_registry_writer.py | 234 ++++++++++++++---- .../proxy/db/test_tool_table_bulk_upsert.py | 38 +++ 15 files changed, 553 insertions(+), 238 deletions(-) create mode 100644 litellm/proxy/db/tool_table_bulk_upsert.py create mode 100644 tests/unit/proxy/db/test_tool_table_bulk_upsert.py diff --git a/litellm/constants.py b/litellm/constants.py index c721fb94c42..d1daf2ee2d1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -480,6 +480,7 @@ REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_end_user_sp REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_agent_spend_update_buffer" REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_tag_spend_update_buffer" REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_window_spend_update_buffer" +REDIS_TOOL_DISCOVERY_BUFFER_KEY: Final = "litellm_tool_discovery_buffer" MAX_REDIS_BUFFER_DEQUEUE_COUNT: Final = get_env_int("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100) REDIS_SPEND_LOGS_BUFFER_KEY: Final = "litellm_spend_logs_buffer" REDIS_SPEND_LOGS_BUFFER_MAX_ROWS: Final = 100000 @@ -1844,6 +1845,7 @@ LITELLM_UI_SESSION_DURATION: Final = os.getenv("LITELLM_UI_SESSION_DURATION", "2 ########################### DB CRON JOB NAMES ########################### DB_SPEND_UPDATE_JOB_NAME: Final = "db_spend_update_job" +TOOL_DISCOVERY_COMMIT_JOB_NAME: Final = "tool_discovery_commit_job" DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME: Final = "db_daily_tag_spend_update_job" PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME: Final = "prometheus_emit_budget_metrics" CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME: Final = "cloudzero_export_usage_data" diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 28d3c08bb31..3a3306dfdc0 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -5262,6 +5262,7 @@ class ToolDiscoveryQueueItem(TypedDict, total=False): team_id: str | None # team that triggered discovery key_alias: str | None # human-readable key alias user_agent: str | None # HTTP User-Agent of the caller + call_count: ReadOnly[int] # invocations folded into this row since the previous flush from litellm.models.managed_files import ( # noqa: E402 diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index b290e0f85b4..24ab09283f1 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -37,7 +37,6 @@ from litellm.constants import ( from litellm.litellm_core_utils.litellm_logging import coerce_model_access_groups from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.proxy._types import ( - DB_CONNECTION_ERROR_TYPES, DB_RETRY_SAFE_ERROR_TYPES, BaseDailySpendTransaction, DailyAgentSpendTransaction, @@ -74,7 +73,7 @@ from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( WindowSpendTransaction, WindowSpendUpdateQueue, ) -from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler +from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, db_write_failure_is_requeue_safe from litellm.proxy.db.model_usage_rollup import build_model_usage_transaction from litellm.proxy.db.request_error_tracking import request_error_accumulator from litellm.proxy.db.rollup_lock_timeout import apply_rollup_lock_timeout @@ -251,16 +250,6 @@ class _DailySpendCommit(Protocol[_DailySpendTransactionT]): ) -> None: ... -_DATA_REJECTED_SQLSTATE_CLASSES: Final = frozenset({"22", "23"}) - - -def _spend_commit_failure_is_requeue_safe(e: Exception) -> bool: - if isinstance(e, DB_CONNECTION_ERROR_TYPES): - return isinstance(e, DB_RETRY_SAFE_ERROR_TYPES) - sqlstate: Final = PrismaDBExceptionHandler.postgres_sqlstate(e) - return sqlstate is None or sqlstate[:2] not in _DATA_REJECTED_SQLSTATE_CLASSES - - _SpendTableName = Literal[ "user_list_transactions", "end_user_list_transactions", @@ -298,7 +287,7 @@ def _spend_tables_left_to_send( _SPEND_TABLE_COMMIT_ORDER[len(committed)] if len(committed) < len(_SPEND_TABLE_COMMIT_ORDER) else None ) dropped: Final[frozenset[_SpendTableName]] = ( - frozenset() if in_flight is None or _spend_commit_failure_is_requeue_safe(failure) else frozenset({in_flight}) + frozenset() if in_flight is None or db_write_failure_is_requeue_safe(failure) else frozenset({in_flight}) ) if dropped and in_flight is not None: spend_log_error( @@ -1635,8 +1624,6 @@ class DBSpendUpdateWriter: window_spend_update_queue=self.window_spend_update_queue, ) - await self._flush_tool_discovery_queue(prisma_client=prisma_client) - # Only commit from redis to db if this pod is the leader if await self.pod_lock_manager.acquire_lock( cronjob_id=DB_SPEND_UPDATE_JOB_NAME, @@ -1781,7 +1768,7 @@ class DBSpendUpdateWriter: window_spend_transactions=window_spend_update_transactions, ) except Exception as e: - if not _spend_commit_failure_is_requeue_safe(e): + if not db_write_failure_is_requeue_safe(e): uncommitted.pop("window_spend_update_transactions", None) spend_log_error( "Spend tracking - dropped %d budget window increments: the failed statement may have " @@ -1947,7 +1934,7 @@ class DBSpendUpdateWriter: window_spend_transactions=window_spend_update_transactions, ) except Exception as e: # noqa: BLE001 # the increments go back on the queue; the rest of the flush must run - if _spend_commit_failure_is_requeue_safe(e): + if db_write_failure_is_requeue_safe(e): spend_log_error( "Spend tracking - failed to commit budget window spend updates. " "Re-queued %d window increments for retry on next tick. Error: %s", @@ -1965,9 +1952,6 @@ class DBSpendUpdateWriter: exc=e, ) - ################## Tool Registry Upserts ################## - await self._flush_tool_discovery_queue(prisma_client=prisma_client) - async def commit_daily_tag_spend_to_db( self, prisma_client: PrismaClient, @@ -2099,20 +2083,6 @@ class DBSpendUpdateWriter: ) raise - async def _flush_tool_discovery_queue( - self, - prisma_client: PrismaClient, - ) -> None: - """Flush ToolDiscoveryQueue and batch-upsert new tools into LiteLLM_ToolTable.""" - from litellm.proxy.db.tool_registry_writer import batch_upsert_tools - - try: - items: Final = self.tool_discovery_queue.flush() - if items: - await batch_upsert_tools(prisma_client=prisma_client, items=items) - except Exception as e: - verbose_proxy_logger.debug("_flush_tool_discovery_queue error (non-blocking): %s", e) - @staticmethod async def handle_spend_update_failure( e: Exception, @@ -2124,7 +2094,6 @@ class DBSpendUpdateWriter: """Retry a failed spend-update transaction on connection errors, deadlocks or a rollup ``lock_timeout`` (55P03), else re-raise. All three roll the transaction back before any increment applied, so re-sending the same batch cannot double-count.""" - from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.proxy.utils import raise_failed_update_spend_exception is_retryable = ( @@ -2633,7 +2602,7 @@ class DBSpendUpdateWriter: _mark_daily_spend_commit_started() _mark_daily_spend_commit_finished() except Exception as batch_error: - if _spend_commit_failure_is_requeue_safe(batch_error): + if db_write_failure_is_requeue_safe(batch_error): spend_log_error( "Daily %s spend batch upsert failed. Table: %s, Rows: %d, Error: %s", entity_type, diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index 66833fd688a..3a4375ac54a 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -9,6 +9,7 @@ import json from collections.abc import Mapping, Sequence from datetime import datetime from functools import reduce +from itertools import chain from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast from pydantic import TypeAdapter @@ -27,6 +28,7 @@ from litellm.constants import ( REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY, REDIS_SPEND_LOGS_BUFFER_KEY, REDIS_SPEND_LOGS_BUFFER_MAX_ROWS, + REDIS_TOOL_DISCOVERY_BUFFER_KEY, REDIS_UPDATE_BUFFER_KEY, REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY, ) @@ -42,12 +44,17 @@ from litellm.proxy._types import ( DBSpendUpdateTransactions, Litellm_EntityType, SpendUpdateQueueItem, + ToolDiscoveryQueueItem, ) from litellm.proxy.db.db_transaction_queue.base_update_queue import service_logger_obj from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import ( DailySpendUpdateQueue, ) from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdateQueue +from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import ( + ToolDiscoveryQueue, + aggregate_tool_discoveries, +) from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( WindowSpendTransaction, WindowSpendUpdateQueue, @@ -135,6 +142,9 @@ def _merged_entity_transactions( ) +_TOOL_DISCOVERY_ITEMS: Final = TypeAdapter(list[ToolDiscoveryQueueItem]) + + class RedisUpdateBuffer: """ Handles buffering database `UPDATE` transactions in Redis before committing them to the database @@ -559,6 +569,56 @@ class RedisUpdateBuffer: str(e), ) + @with_service_target(SPEND_QUEUE_TARGET) + async def store_tool_discoveries_in_redis(self, tool_discovery_queue: ToolDiscoveryQueue) -> None: + """Move this worker's pending tool rows to Redis so only the leader writes LiteLLM_ToolTable.""" + if self.redis_cache is None: + return + items: Final = tool_discovery_queue.flush() + if not items: + return + try: + _ = await self.redis_cache.async_rpush_pipeline( + rpush_list=[ + RedisPipelineRpushOperation(key=REDIS_TOOL_DISCOVERY_BUFFER_KEY, values=[safe_dumps(items)]) + ] + ) + except (RedisError, OSError, asyncio.TimeoutError) as e: + verbose_proxy_logger.error( + "Tool discovery - failed to push %d tool rows to Redis, restoring them for the next tick. Error: %s", + len(items), + str(e), + ) + for item in items: + tool_discovery_queue.add_update(item) + + @with_service_target(SPEND_QUEUE_TARGET) + async def restore_tool_discoveries_to_redis(self, items: Sequence[ToolDiscoveryQueueItem]) -> None: + """Put drained tool rows back for the next drain after a write that provably did not apply.""" + if self.redis_cache is None or not items: + return + _ = await self.redis_cache.async_rpush_pipeline( + rpush_list=[ + RedisPipelineRpushOperation(key=REDIS_TOOL_DISCOVERY_BUFFER_KEY, values=[safe_dumps(list(items))]) + ] + ) + + @with_service_target(SPEND_QUEUE_TARGET) + async def get_all_tool_discoveries_from_redis_buffer(self) -> tuple[ToolDiscoveryQueueItem, ...]: + """Drain every worker's tool rows and fold them to one row per tool_name.""" + if self.redis_cache is None: + return () + popped: Final = await self.redis_cache.async_lpop_pipeline( + lpop_list=[ + RedisPipelineLpopOperation(key=REDIS_TOOL_DISCOVERY_BUFFER_KEY, count=MAX_REDIS_BUFFER_DEQUEUE_COUNT) + ] + ) + entries: Final = popped[0] if popped else None + if not entries: + return () + batches: Final = tuple(_TOOL_DISCOVERY_ITEMS.validate_json(entry) for entry in entries) + return aggregate_tool_discoveries(chain.from_iterable(batches)) + @with_service_target(SPEND_QUEUE_TARGET) async def store_spend_logs_in_redis( self, diff --git a/litellm/proxy/db/db_transaction_queue/tool_discovery_queue.py b/litellm/proxy/db/db_transaction_queue/tool_discovery_queue.py index d5f6a447131..e9ec040023c 100644 --- a/litellm/proxy/db/db_transaction_queue/tool_discovery_queue.py +++ b/litellm/proxy/db/db_transaction_queue/tool_discovery_queue.py @@ -1,52 +1,57 @@ """ -In-memory buffer for tool registry upserts. - -Unlike SpendUpdateQueue (which aggregates increments), ToolDiscoveryQueue -uses set-deduplication: each unique tool_name is only queued once per flush -cycle (~30s). The seen-set is cleared on every flush so that call_count -increments in subsequent cycles rather than stopping after the first flush. +In-memory buffer for tool registry upserts, aggregated by tool_name so one flush carries one row per +tool with the number of times it was invoked since the previous flush. """ +from collections.abc import Iterable from typing import Final +from pydantic import TypeAdapter + from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ToolDiscoveryQueueItem +_ITEM: Final = TypeAdapter(ToolDiscoveryQueueItem) + + +def _merged(existing: ToolDiscoveryQueueItem | None, item: ToolDiscoveryQueueItem) -> ToolDiscoveryQueueItem: + """The first sighting keeps its origin and caller fields, later sightings only add to call_count.""" + increment: Final = item.get("call_count", 1) + if existing is None: + return _ITEM.validate_python({**item, "call_count": increment}) + return _ITEM.validate_python({**existing, "call_count": existing.get("call_count", 1) + increment}) + + +def aggregate_tool_discoveries(items: Iterable[ToolDiscoveryQueueItem]) -> tuple[ToolDiscoveryQueueItem, ...]: + """Collapse items to one per tool_name, summing call_count, in first-seen order.""" + by_name: Final[dict[str, ToolDiscoveryQueueItem]] = {} # mutable-ok: single pass fold keyed by tool_name + for item in items: + if not (tool_name := item.get("tool_name", "")): + continue + by_name[tool_name] = _merged(by_name.get(tool_name), item) + return tuple(by_name.values()) + class ToolDiscoveryQueue: - """ - In-memory buffer for tool registry upserts. - - Deduplicates by tool_name within each flush cycle: a tool is only queued - once per ~30s batch, so call_count increments once per flush cycle the - tool appears in (not once per invocation, but not once per pod lifetime - either). The seen-set is cleared on flush so subsequent batches can - re-count the same tool. - """ + """In-memory buffer for tool registry upserts, one pending row per tool_name with a running call_count.""" def __init__(self) -> None: - self._seen_tool_names: set[str] = set() - self._pending: list[ToolDiscoveryQueueItem] = [] + self._pending: dict[str, ToolDiscoveryQueueItem] = {} # mutable-ok: folded per add_update, cleared on flush def add_update(self, item: ToolDiscoveryQueueItem) -> None: - """Enqueue a tool discovery item if tool_name has not been seen before.""" + """Count one more invocation of item's tool_name, keeping the first sighting's origin and caller fields.""" tool_name: Final = item.get("tool_name", "") if not tool_name: return - if tool_name in self._seen_tool_names: - verbose_proxy_logger.debug("ToolDiscoveryQueue: skipping already-seen tool %s", tool_name) - return - self._seen_tool_names.add(tool_name) - self._pending.append(item) + self._pending[tool_name] = _merged(self._pending.get(tool_name), item) verbose_proxy_logger.debug( - "ToolDiscoveryQueue: queued new tool %s (origin=%s)", + "ToolDiscoveryQueue: tool %s call_count=%s (origin=%s)", tool_name, + self._pending[tool_name].get("call_count"), item.get("origin"), ) def flush(self) -> list[ToolDiscoveryQueueItem]: - """Return and clear all pending items. Resets seen-set so the next - flush cycle can re-count the same tools.""" - items, self._pending = self._pending, [] - self._seen_tool_names.clear() + """Return and clear all pending rows.""" + items, self._pending = list(self._pending.values()), {} return items diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index 379e0012ea8..5dfcaa4b2f8 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -8,6 +8,7 @@ from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ( DB_CONNECTION_ERROR_TYPES, + DB_RETRY_SAFE_ERROR_TYPES, ProxyErrorTypes, ProxyException, ) @@ -637,3 +638,14 @@ async def call_with_db_reconnect_retry( # propagate — repeated reconnect-loops are the watchdog's job, not # this helper's. return await coro_factory() + + +_DATA_REJECTED_SQLSTATE_CLASSES: Final = frozenset({"22", "23"}) + + +def db_write_failure_is_requeue_safe(e: Exception) -> bool: + """Whether a failed write provably never applied, so its rows can be sent again without double counting.""" + if isinstance(e, DB_CONNECTION_ERROR_TYPES): + return isinstance(e, DB_RETRY_SAFE_ERROR_TYPES) + sqlstate: Final = PrismaDBExceptionHandler.postgres_sqlstate(e) + return sqlstate is None or sqlstate[:2] not in _DATA_REJECTED_SQLSTATE_CLASSES diff --git a/litellm/proxy/db/tool_registry_writer.py b/litellm/proxy/db/tool_registry_writer.py index 2e3530d7613..f7ec88945a1 100644 --- a/litellm/proxy/db/tool_registry_writer.py +++ b/litellm/proxy/db/tool_registry_writer.py @@ -14,8 +14,17 @@ from typing import TYPE_CHECKING, Final, Protocol from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger +from litellm.constants import TOOL_DISCOVERY_COMMIT_JOB_NAME from litellm.proxy._types import ToolDiscoveryQueueItem -from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry +from litellm.proxy.db.db_span import db_span +from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager +from litellm.proxy.db.db_transaction_queue.redis_update_buffer import RedisUpdateBuffer +from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import ( + ToolDiscoveryQueue, + aggregate_tool_discoveries, +) +from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry, db_write_failure_is_requeue_safe +from litellm.proxy.db.tool_table_bulk_upsert import build_tool_bulk_upsert from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import ToolRepository @@ -105,58 +114,75 @@ def _row_to_model(row: object) -> LiteLLM_ToolTableRow: async def batch_upsert_tools( prisma_client: "PrismaClient", - items: list[ToolDiscoveryQueueItem], + items: Sequence[ToolDiscoveryQueueItem], ) -> None: - """ - Batch-upsert tool registry rows via Prisma. + """Write a batch of discovered tools to LiteLLM_ToolTable in a single statement. Raises on failure so the + caller decides whether the rows can be retried.""" + rows: Final = aggregate_tool_discoveries(items) + if not rows: + return + sql, params = build_tool_bulk_upsert(rows) + async with db_span("commit_tool_discoveries", "LiteLLM_ToolTable"): + await prisma_client.db.execute_raw(sql, *params) # pyright: ignore[reportAny] # untyped prisma client + verbose_proxy_logger.debug("tool_registry_writer: upserted %d tool(s)", len(rows)) - On first insert: sets input_policy/output_policy = "untrusted" (default), call_count = 1. - On conflict: increments call_count; preserves existing policies. - """ + +async def flush_tool_discovery_queue(prisma_client: "PrismaClient", tool_discovery_queue: ToolDiscoveryQueue) -> None: + """One worker's own rows straight to LiteLLM_ToolTable, the path without the Redis buffer.""" + items: Final = tool_discovery_queue.flush() if not items: return try: - data: Final = [item for item in items if item.get("tool_name")] - if not data: - return - now: Final = datetime.now(timezone.utc) - table: Final = _tool_table_actions(prisma_client) - for item in data: - tool_name = item.get("tool_name", "") - origin = item.get("origin") or "user_defined" - created_by = item.get("created_by") or "system" - key_hash = item.get("key_hash") - team_id = item.get("team_id") - key_alias = item.get("key_alias") - user_agent = item.get("user_agent") - await table.upsert( - where={"tool_name": tool_name}, - data={ - "create": { - "tool_id": str(uuid.uuid4()), - "tool_name": tool_name, - "origin": origin, - "input_policy": "untrusted", - "output_policy": "untrusted", - "call_count": 1, - "created_by": created_by, - "updated_by": created_by, - "key_hash": key_hash, - "team_id": team_id, - "key_alias": key_alias, - "user_agent": user_agent, - "last_used_at": now, - }, - "update": { - "call_count": {"increment": 1}, - "updated_at": now, - "last_used_at": now, - }, - }, + await batch_upsert_tools(prisma_client=prisma_client, items=items) + except Exception as e: # noqa: BLE001 # a lost tool row only delays its discovery, the scheduler job must not fail + verbose_proxy_logger.debug("flush_tool_discovery_queue error (non-blocking): %s", e) + + +async def commit_tool_discoveries( + prisma_client: "PrismaClient", + tool_discovery_queue: ToolDiscoveryQueue, + redis_update_buffer: RedisUpdateBuffer, + pod_lock_manager: PodLockManager, + use_redis_buffer: bool, +) -> None: + """Scheduled alongside update_spend on its own lock, so tool table writes never ride the spend commit. + + With the Redis buffer every worker hands its rows to Redis and the holder of the tool lock writes them all in + one statement. Without it each worker writes its own rows.""" + if not use_redis_buffer: + await flush_tool_discovery_queue(prisma_client=prisma_client, tool_discovery_queue=tool_discovery_queue) + return + await redis_update_buffer.store_tool_discoveries_in_redis(tool_discovery_queue=tool_discovery_queue) + if not await pod_lock_manager.acquire_lock(cronjob_id=TOOL_DISCOVERY_COMMIT_JOB_NAME): + return + try: + await _commit_tool_discoveries_from_redis(prisma_client=prisma_client, redis_update_buffer=redis_update_buffer) + finally: + await pod_lock_manager.release_lock(cronjob_id=TOOL_DISCOVERY_COMMIT_JOB_NAME) + + +async def _commit_tool_discoveries_from_redis( + prisma_client: "PrismaClient", redis_update_buffer: RedisUpdateBuffer +) -> None: + """Tool lock holder only: drain every worker's tool rows from Redis and write them in one statement.""" + items: Final = await redis_update_buffer.get_all_tool_discoveries_from_redis_buffer() + if not items: + return + try: + await batch_upsert_tools(prisma_client=prisma_client, items=items) + except Exception as e: # noqa: BLE001 # the rows go back to redis or are dropped, the scheduler job must not fail + if db_write_failure_is_requeue_safe(e): + await redis_update_buffer.restore_tool_discoveries_to_redis(items) + verbose_proxy_logger.error( + "Tool discovery - %d tool rows went back to Redis for the next tick. Error: %s", len(items), str(e) ) - verbose_proxy_logger.debug("tool_registry_writer: upserted %d tool(s)", len(data)) - except Exception as e: - verbose_proxy_logger.error("tool_registry_writer batch_upsert_tools error: %s", e) + return + verbose_proxy_logger.error( + "Tool discovery - dropped %d tool rows: the failed statement may have applied or the database " + "refused the data, so re-sending it is not safe. Error: %s", + len(items), + str(e), + ) _NO_OWNERS: Final[Mapping[str, ToolDiscoveryUser]] = MappingProxyType({}) diff --git a/litellm/proxy/db/tool_table_bulk_upsert.py b/litellm/proxy/db/tool_table_bulk_upsert.py new file mode 100644 index 00000000000..e586042ae7c --- /dev/null +++ b/litellm/proxy/db/tool_table_bulk_upsert.py @@ -0,0 +1,71 @@ +"""One multi-row ``INSERT ... ON CONFLICT ("tool_name") DO UPDATE`` per batch of discovered tools. + +LiteLLM_ToolTable holds one row per tool name the proxy has seen. A batch arrives already folded to one +entry per tool name with a summed call_count, and this statement inserts the names the table has not seen +with the untrusted policies the columns default to, and for the names it has adds the batch's call_count +while leaving the policies and the discovering caller's fields alone. Same shape as daily_spend_bulk_upsert. +""" + +import uuid +from collections.abc import Mapping, Sequence +from itertools import chain +from types import MappingProxyType +from typing import Final, TypeAlias + +from litellm.proxy._types import ToolDiscoveryQueueItem + +SqlValue: TypeAlias = str | int | None + +_TOOL_ROW_COLUMNS: Final[tuple[str, ...]] = ( + "tool_id", + "tool_name", + "origin", + "call_count", + "created_by", + "updated_by", + "key_hash", + "team_id", + "key_alias", + "user_agent", +) +_CASTS: Final[Mapping[str, str]] = MappingProxyType({"call_count": "integer"}) + + +def _row_params(item: ToolDiscoveryQueueItem) -> tuple[SqlValue, ...]: + created_by: Final = item.get("created_by") or "system" + return ( + str(uuid.uuid4()), + item.get("tool_name", ""), + item.get("origin") or "user_defined", + item.get("call_count", 1), + created_by, + created_by, + item.get("key_hash"), + item.get("team_id"), + item.get("key_alias"), + item.get("user_agent"), + ) + + +def build_tool_bulk_upsert(items: Sequence[ToolDiscoveryQueueItem]) -> tuple[str, tuple[SqlValue, ...]]: + """The single statement writing one batch of distinct tool_names, plus its positional arguments.""" + width: Final = len(_TOOL_ROW_COLUMNS) + rows: Final = ", ".join( + "(" + + ", ".join( + f"${row_index * width + offset + 1}::{_CASTS.get(column, 'text')}" + for offset, column in enumerate(_TOOL_ROW_COLUMNS) + ) + + ", (NOW() AT TIME ZONE 'UTC'), (NOW() AT TIME ZONE 'UTC'))" + for row_index in range(len(items)) + ) + columns: Final = ", ".join(f'"{column}"' for column in _TOOL_ROW_COLUMNS) + sql: Final = ( + f'INSERT INTO "LiteLLM_ToolTable" ({columns}, "last_used_at", "updated_at")\n' + f"VALUES {rows}\n" + 'ON CONFLICT ("tool_name") DO UPDATE SET\n' + ' "call_count" = "LiteLLM_ToolTable"."call_count" + EXCLUDED."call_count",\n' + ' "last_used_at" = EXCLUDED."last_used_at",\n' + ' "updated_at" = EXCLUDED."updated_at"' + ) + return sql, tuple(chain.from_iterable(map(_row_params, items))) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 44f43808dc8..df948b69a67 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11296,6 +11296,19 @@ class ProxyStartupEvent: misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, ) + ### TOOL DISCOVERY (own lock, same interval, never rides the spend commit) ### + from litellm.proxy.utils import commit_tool_discoveries + + scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # apscheduler is untyped + commit_tool_discoveries, + "interval", + seconds=batch_writing_interval, + args=[prisma_client, proxy_logging_obj], + id="commit_tool_discoveries_job", + replace_existing=True, + misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, + ) + ### UPDATE DAILY TAG SPEND (separate scheduler job with longer interval) ### ## Reduces QPS as there are more tags for a single request tag_spend_update_interval: Final = int(batch_writing_interval * DAILY_TAG_SPEND_BATCH_MULTIPLIER) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 6cf274884c7..8212e7cbe74 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -184,6 +184,7 @@ from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header from litellm.proxy.common_utils.config_sync_pubsub import publish_config_param_change from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.db import tool_registry_writer from litellm.proxy.db.create_views import ( create_missing_views, create_view_tolerating_race, @@ -191,6 +192,7 @@ from litellm.proxy.db.create_views import ( ) from litellm.proxy.db.db_span import db_span from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter +from litellm.proxy.db.db_transaction_queue.redis_update_buffer import RedisUpdateBuffer from litellm.proxy.db.db_url_settings import ( DatabaseURLSettings, add_missing_query_params, @@ -8182,6 +8184,18 @@ class ProxyUpdateSpend: return False +async def commit_tool_discoveries(prisma_client: PrismaClient, proxy_logging_obj: ProxyLogging) -> None: + """Scheduler job writing discovered tools to LiteLLM_ToolTable, on its own lock, next to update_spend.""" + writer: Final = proxy_logging_obj.db_spend_update_writer + await tool_registry_writer.commit_tool_discoveries( + prisma_client=prisma_client, + tool_discovery_queue=writer.tool_discovery_queue, + redis_update_buffer=writer.redis_update_buffer, + pod_lock_manager=writer.pod_lock_manager, + use_redis_buffer=RedisUpdateBuffer.should_commit_spend_updates_to_redis(), + ) + + async def update_spend( prisma_client: PrismaClient, db_writer_client: AsyncHTTPHandler | None, diff --git a/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py b/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py index f30606ac2f1..24c12e06c9c 100644 --- a/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py +++ b/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py @@ -5,7 +5,6 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest - from litellm.constants import REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import DailyTagSpendTransaction @@ -769,3 +768,18 @@ async def test_store_spend_logs_in_redis_is_off_unless_transaction_buffering_is_ assert await buffer.store_spend_logs_in_redis([{"request_id": "a"}]) is False assert redis.rows == [] + + +@pytest.mark.asyncio +async def test_tool_discoveries_go_back_on_the_queue_when_the_redis_push_fails(): + from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import ToolDiscoveryQueue + + redis_cache = MagicMock() + redis_cache.async_rpush_pipeline = AsyncMock(side_effect=ConnectionError("redis down")) + queue = ToolDiscoveryQueue() + queue.add_update({"tool_name": "get_weather", "origin": "user_defined"}) + queue.add_update({"tool_name": "get_weather", "origin": "user_defined"}) + + await RedisUpdateBuffer(redis_cache=redis_cache).store_tool_discoveries_in_redis(queue) + + assert queue.flush() == [{"tool_name": "get_weather", "origin": "user_defined", "call_count": 2}] diff --git a/tests/unit/proxy/db/db_transaction_queue/test_tool_discovery_queue.py b/tests/unit/proxy/db/db_transaction_queue/test_tool_discovery_queue.py index e400ad16e84..0a015255f2e 100644 --- a/tests/unit/proxy/db/db_transaction_queue/test_tool_discovery_queue.py +++ b/tests/unit/proxy/db/db_transaction_queue/test_tool_discovery_queue.py @@ -5,9 +5,9 @@ Unit tests for ToolDiscoveryQueue. import pytest - from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import ( ToolDiscoveryQueue, + aggregate_tool_discoveries, ) @@ -16,57 +16,50 @@ def queue(): return ToolDiscoveryQueue() -def test_add_single_tool(queue): +def test_a_tool_seen_once_flushes_with_call_count_one(queue): queue.add_update({"tool_name": "my_tool", "origin": "user_defined"}) - items = queue.flush() - assert len(items) == 1 - assert items[0]["tool_name"] == "my_tool" - assert items[0]["origin"] == "user_defined" + assert queue.flush() == [{"tool_name": "my_tool", "origin": "user_defined", "call_count": 1}] -def test_deduplication_same_name(queue): - """Adding the same tool_name twice should only keep the first.""" - queue.add_update({"tool_name": "tool_a", "origin": "mcp_server"}) - queue.add_update({"tool_name": "tool_a", "origin": "user_defined"}) - items = queue.flush() - assert len(items) == 1 - assert items[0]["origin"] == "mcp_server" # first wins +def test_repeated_sightings_fold_into_one_row_counting_every_call(queue): + queue.add_update({"tool_name": "tool_a", "origin": "mcp_server", "created_by": "alice"}) + queue.add_update({"tool_name": "tool_a", "origin": "user_defined", "created_by": "bob"}) + queue.add_update({"tool_name": "tool_a"}) + assert queue.flush() == [{"tool_name": "tool_a", "origin": "mcp_server", "created_by": "alice", "call_count": 3}] -def test_deduplication_different_names(queue): +def test_distinct_tools_each_get_their_own_row(queue): queue.add_update({"tool_name": "tool_a"}) queue.add_update({"tool_name": "tool_b"}) - items = queue.flush() - assert len(items) == 2 - names = {i["tool_name"] for i in items} - assert names == {"tool_a", "tool_b"} - - -def test_flush_clears_pending(queue): - queue.add_update({"tool_name": "tool_x"}) - items1 = queue.flush() - assert len(items1) == 1 - items2 = queue.flush() - assert len(items2) == 0 - - -def test_seen_names_reset_after_flush(queue): - """Seen-set is cleared on flush so the same tool can re-enter the next cycle.""" queue.add_update({"tool_name": "tool_a"}) - queue.flush() - queue.add_update({"tool_name": "tool_a"}) # same tool, new cycle - items = queue.flush() - assert len(items) == 1 - assert items[0]["tool_name"] == "tool_a" + assert queue.flush() == [ + {"tool_name": "tool_a", "call_count": 2}, + {"tool_name": "tool_b", "call_count": 1}, + ] + + +def test_flush_clears_pending_and_the_next_cycle_counts_from_zero(queue): + queue.add_update({"tool_name": "tool_x"}) + queue.add_update({"tool_name": "tool_x"}) + assert queue.flush() == [{"tool_name": "tool_x", "call_count": 2}] + assert queue.flush() == [] + queue.add_update({"tool_name": "tool_x"}) + assert queue.flush() == [{"tool_name": "tool_x", "call_count": 1}] def test_empty_tool_name_ignored(queue): queue.add_update({"tool_name": ""}) queue.add_update({"tool_name": None}) # type: ignore[arg-type] - items = queue.flush() - assert len(items) == 0 + assert queue.flush() == [] -def test_flush_returns_list(queue): - result = queue.flush() - assert isinstance(result, list) +def test_aggregate_sums_call_counts_across_workers_flushes(): + worker_a = [{"tool_name": "get_weather", "origin": "user_defined", "call_count": 3}] + worker_b = [ + {"tool_name": "get_weather", "origin": "mcp_server", "call_count": 2}, + {"tool_name": "search", "call_count": 1}, + ] + assert aggregate_tool_discoveries([*worker_a, *worker_b]) == ( + {"tool_name": "get_weather", "origin": "user_defined", "call_count": 5}, + {"tool_name": "search", "call_count": 1}, + ) diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index 9b2a96bbc58..6071569b806 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -19,7 +19,7 @@ from redis.exceptions import DataError import litellm from litellm._logging import verbose_proxy_logger from litellm._service_logger import ServiceTypes -from litellm.constants import DB_SPEND_UPDATE_JOB_NAME, REDIS_UPDATE_BUFFER_KEY, SPEND_ROLLUP_LOCK_TIMEOUT_MS +from litellm.constants import SPEND_ROLLUP_LOCK_TIMEOUT_MS from litellm.proxy._types import DailyTagSpendTransaction, Litellm_EntityType, SpendUpdateQueueItem from litellm.proxy.db.db_spend_update_writer import ( _TEAM_ADVISORY_LOCK_SQL, @@ -29,7 +29,6 @@ from litellm.proxy.db.db_spend_update_writer import ( _SpendTableName, ) from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import DailySpendUpdateQueue -from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager from litellm.proxy.db.db_transaction_queue.redis_update_buffer import RedisUpdateBuffer from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdateQueue from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( @@ -2514,31 +2513,6 @@ class _InMemoryRedis: return [self.lists.pop(op["key"], None) for op in lpop_list] -@pytest.mark.asyncio -@pytest.mark.parametrize("lock_held_by_another_pod", [False, True]) -async def test_commit_with_redis_writes_discovered_tools_on_leader_and_non_leader(lock_held_by_another_pod: bool): - redis = _InMemoryRedis() - if lock_held_by_another_pod: - assert await PodLockManager(redis_cache=redis).acquire_lock(cronjob_id=DB_SPEND_UPDATE_JOB_NAME) - db_writer = DBSpendUpdateWriter(redis_cache=redis) - db_writer.pod_lock_manager.redis_cache = redis - db_writer.tool_discovery_queue.add_update({"tool_name": "get_weather", "origin": "user_defined"}) - prisma_client = MagicMock() - prisma_client.db.litellm_tooltable.upsert = AsyncMock() - - await db_writer._commit_spend_updates_to_db_with_redis( - prisma_client=prisma_client, - n_retry_times=0, - proxy_logging_obj=MagicMock(), - ) - - upserted = [c.kwargs["where"] for c in prisma_client.db.litellm_tooltable.upsert.await_args_list] - assert upserted == [{"tool_name": "get_weather"}] - assert (REDIS_UPDATE_BUFFER_KEY in redis.lists) is lock_held_by_another_pod, ( - f"spend buffer left in redis={list(redis.lists)}, only the lock holder may drain it" - ) - - @pytest.mark.asyncio async def test_commit_with_redis_requeues_all_on_db_failure(): """ @@ -3384,7 +3358,6 @@ async def test_failed_window_spend_commit_requeues_the_increments_and_continues_ await db_writer.window_spend_update_queue.add_update(transaction) db = _WindowSpendFakeDB() db.query_raw = AsyncMock(side_effect=Exception("connection reset")) - db_writer._flush_tool_discovery_queue = AsyncMock() await db_writer._commit_spend_updates_to_db_without_redis_buffer( prisma_client=_WindowSpendFakePrisma(db), @@ -3392,7 +3365,6 @@ async def test_failed_window_spend_commit_requeues_the_increments_and_continues_ proxy_logging_obj=MagicMock(), ) - db_writer._flush_tool_discovery_queue.assert_called_once() requeued = await db_writer.window_spend_update_queue.flush_and_get_aggregated_window_spend_transactions() assert requeued == (transaction,) @@ -3468,7 +3440,6 @@ async def test_failed_daily_spend_commit_is_requeued_only_when_the_rows_are_prov db_writer = DBSpendUpdateWriter() await db_writer.daily_spend_update_queue.add_update({"user-key": _daily_txn(user_id="user-1")}) db = _DailySpendFakeDB(failing_table="LiteLLM_DailyUserSpend", failure=failure) - db_writer._flush_tool_discovery_queue = AsyncMock() proxy_logging_obj = MagicMock() proxy_logging_obj.failure_handler = AsyncMock() @@ -3494,7 +3465,6 @@ async def test_failed_daily_spend_commit_drops_only_the_batch_that_was_sent(): {f"user-{i:03d}": _daily_txn(user_id=f"user-{i:03d}") for i in range(150)} ) db = _DailySpendFakeDB(failing_table="LiteLLM_DailyUserSpend", failure=httpx.ReadTimeout("no reply")) - db_writer._flush_tool_discovery_queue = AsyncMock() proxy_logging_obj = MagicMock() proxy_logging_obj.failure_handler = AsyncMock() @@ -3522,7 +3492,6 @@ async def test_failed_daily_spend_commit_requeues_the_rows_and_flushes_the_other team_txn = {key: value for key, value in _daily_txn().items() if key != "user_id"} | {"team_id": "team-1"} await db_writer.daily_team_spend_update_queue.add_update({"team-key": team_txn}) db = _DailySpendFakeDB(failing_table="LiteLLM_DailyUserSpend") - db_writer._flush_tool_discovery_queue = AsyncMock() proxy_logging_obj = MagicMock() proxy_logging_obj.failure_handler = AsyncMock() @@ -3533,7 +3502,6 @@ async def test_failed_daily_spend_commit_requeues_the_rows_and_flushes_the_other assert _daily_upserts(db, "LiteLLM_DailyUserSpend") == [] (team_upsert,) = _daily_upserts(db, "LiteLLM_DailyTeamSpend") assert _row_values(team_upsert, "team_id") == ["team-1"] - db_writer._flush_tool_discovery_queue.assert_called_once() db.failing_table = None await db_writer._commit_spend_updates_to_db_without_redis_buffer( @@ -3760,7 +3728,6 @@ async def test_failed_window_spend_commit_is_requeued_only_when_the_rows_are_pro ) await db_writer.window_spend_update_queue.add_update(transaction) db = _WindowSpendFailureDB(failure) - db_writer._flush_tool_discovery_queue = AsyncMock() await db_writer._commit_spend_updates_to_db_without_redis_buffer( prisma_client=_WindowSpendFakePrisma(db), diff --git a/tests/unit/proxy/db/test_tool_registry_writer.py b/tests/unit/proxy/db/test_tool_registry_writer.py index c9df665741d..76898b0eeab 100644 --- a/tests/unit/proxy/db/test_tool_registry_writer.py +++ b/tests/unit/proxy/db/test_tool_registry_writer.py @@ -3,22 +3,28 @@ Unit tests for tool_registry_writer.py — uses a mock prisma client that exposes litellm_tooltable.upsert / find_many / find_unique. """ +import json from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock import pytest -from prisma.errors import PrismaError - +from prisma.errors import PrismaError, RawQueryError +from litellm.constants import DB_SPEND_UPDATE_JOB_NAME, REDIS_TOOL_DISCOVERY_BUFFER_KEY, TOOL_DISCOVERY_COMMIT_JOB_NAME +from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager +from litellm.proxy.db.db_transaction_queue.redis_update_buffer import RedisUpdateBuffer +from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import ToolDiscoveryQueue from litellm.proxy.db.tool_registry_writer import ( ToolPolicyRegistry, batch_upsert_tools, + commit_tool_discoveries, get_tool, get_tool_policy_registry, get_tools_by_names, list_tools, update_tool_policy, ) +from litellm.types.caching import RedisPipelineLpopOperation, RedisPipelineRpushOperation def _mock_row(**kwargs): @@ -62,6 +68,7 @@ def _make_prisma( prisma = MagicMock() prisma.db.litellm_tooltable = MagicMock() prisma.db.litellm_tooltable.upsert = AsyncMock(return_value=upsert_return) + prisma.db.execute_raw = AsyncMock(return_value=1) prisma.db.litellm_tooltable.find_many = AsyncMock( return_value=find_many_rows if find_many_rows is not None else [] ) @@ -73,28 +80,32 @@ def _make_prisma( return prisma +def _statement(prisma) -> tuple[str, tuple[object, ...]]: + prisma.db.execute_raw.assert_awaited_once() + sql, *params = prisma.db.execute_raw.call_args.args + return sql, tuple(params) + + @pytest.mark.asyncio -async def test_batch_upsert_tools_calls_upsert(): +async def test_batch_upsert_tools_inserts_a_new_tool_with_the_batch_call_count_and_adds_it_on_conflict(): prisma = _make_prisma() - items = [{"tool_name": "tool_a", "origin": "mcp_server", "created_by": None}] - await batch_upsert_tools(prisma, items) - prisma.db.litellm_tooltable.upsert.assert_awaited_once() - call_kw = prisma.db.litellm_tooltable.upsert.call_args.kwargs - assert call_kw["where"] == {"tool_name": "tool_a"} - assert call_kw["data"]["create"]["tool_name"] == "tool_a" - assert call_kw["data"]["create"]["origin"] == "mcp_server" - assert call_kw["data"]["create"]["input_policy"] == "untrusted" - assert call_kw["data"]["create"]["output_policy"] == "untrusted" - assert call_kw["data"]["create"]["call_count"] == 1 - assert call_kw["data"]["update"]["call_count"] == {"increment": 1} - assert "updated_at" in call_kw["data"]["update"] + await batch_upsert_tools( + prisma, [{"tool_name": "tool_a", "origin": "mcp_server", "created_by": None, "call_count": 4}] + ) + sql, params = _statement(prisma) + assert sql.startswith('INSERT INTO "LiteLLM_ToolTable"') + assert 'ON CONFLICT ("tool_name") DO UPDATE SET' in sql + assert '"call_count" = "LiteLLM_ToolTable"."call_count" + EXCLUDED."call_count"' in sql + assert '"input_policy"' not in sql, "policies come from the table defaults and are never overwritten" + assert params[1:] == ("tool_a", "mcp_server", 4, "system", "system", None, None, None, None) + prisma.db.litellm_tooltable.upsert.assert_not_awaited() @pytest.mark.asyncio async def test_batch_upsert_tools_empty_list(): prisma = _make_prisma() await batch_upsert_tools(prisma, []) - prisma.db.litellm_tooltable.upsert.assert_not_awaited() + prisma.db.execute_raw.assert_not_awaited() @pytest.mark.asyncio @@ -102,21 +113,22 @@ async def test_batch_upsert_tools_skips_empty_names(): prisma = _make_prisma() items = [{"tool_name": "", "origin": None}, {"tool_name": None}] # type: ignore[list-item] await batch_upsert_tools(prisma, items) - prisma.db.litellm_tooltable.upsert.assert_not_awaited() + prisma.db.execute_raw.assert_not_awaited() @pytest.mark.asyncio -async def test_batch_upsert_multiple_tools_calls_upsert_per_tool(): +async def test_batch_upsert_many_tools_is_one_statement_with_one_row_per_distinct_tool(): prisma = _make_prisma() items = [ - {"tool_name": "tool_a", "origin": "mcp_server", "created_by": None}, - {"tool_name": "tool_b", "origin": "user_defined", "created_by": "alice"}, + {"tool_name": "tool_a", "origin": "mcp_server", "created_by": None, "call_count": 2}, + {"tool_name": "tool_b", "origin": "user_defined", "created_by": "alice", "call_count": 1}, + {"tool_name": "tool_a", "origin": "user_defined", "created_by": "bob", "call_count": 5}, ] await batch_upsert_tools(prisma, items) - assert prisma.db.litellm_tooltable.upsert.await_count == 2 - calls = prisma.db.litellm_tooltable.upsert.call_args_list - assert calls[0].kwargs["where"]["tool_name"] == "tool_a" - assert calls[1].kwargs["where"]["tool_name"] == "tool_b" + sql, params = _statement(prisma) + assert sql.count("$1::text") == 1 and "$11::text" in sql and "$21::text" not in sql + assert params[1:10] == ("tool_a", "mcp_server", 7, "system", "system", None, None, None, None) + assert params[11:20] == ("tool_b", "user_defined", 1, "alice", "alice", None, None, None, None) @pytest.mark.asyncio @@ -264,20 +276,14 @@ async def test_update_tool_policy_calls_upsert_then_get_tool(): assert call_kw["where"] == {"tool_name": "my_tool"} assert call_kw["data"]["update"]["input_policy"] == "blocked" assert call_kw["data"]["update"]["updated_by"] == "admin" - prisma.db.litellm_tooltable.find_unique.assert_awaited_with( - where={"tool_name": "my_tool"} - ) + prisma.db.litellm_tooltable.find_unique.assert_awaited_with(where={"tool_name": "my_tool"}) @pytest.mark.asyncio async def test_get_tools_by_names_returns_policy_map(): rows = [ - _mock_row( - tool_name="tool_a", input_policy="trusted", output_policy="untrusted" - ), - _mock_row( - tool_name="tool_b", input_policy="blocked", output_policy="untrusted" - ), + _mock_row(tool_name="tool_a", input_policy="trusted", output_policy="untrusted"), + _mock_row(tool_name="tool_b", input_policy="blocked", output_policy="untrusted"), ] prisma = _make_prisma(find_many_rows=rows) result = await get_tools_by_names(prisma, ["tool_a", "tool_b"]) @@ -285,9 +291,7 @@ async def test_get_tools_by_names_returns_policy_map(): "tool_a": ("trusted", "untrusted"), "tool_b": ("blocked", "untrusted"), } - prisma.db.litellm_tooltable.find_many.assert_awaited_once_with( - where={"tool_name": {"in": ["tool_a", "tool_b"]}} - ) + prisma.db.litellm_tooltable.find_many.assert_awaited_once_with(where={"tool_name": {"in": ["tool_a", "tool_b"]}}) @pytest.mark.asyncio @@ -382,12 +386,8 @@ async def test_sync_tool_policy_from_db_retries_on_transport_error_first_read(): return [] mock_prisma_client = MagicMock() - mock_prisma_client.db.litellm_tooltable.find_many = AsyncMock( - side_effect=_flaky_find_many - ) - mock_prisma_client.db.litellm_objectpermissiontable.find_many = AsyncMock( - return_value=[] - ) + mock_prisma_client.db.litellm_tooltable.find_many = AsyncMock(side_effect=_flaky_find_many) + mock_prisma_client.db.litellm_objectpermissiontable.find_many = AsyncMock(return_value=[]) mock_prisma_client.attempt_db_reconnect = AsyncMock(return_value=True) mock_prisma_client._db_auth_reconnect_timeout_seconds = 2.0 mock_prisma_client._db_auth_reconnect_lock_timeout_seconds = 0.1 @@ -397,10 +397,7 @@ async def test_sync_tool_policy_from_db_retries_on_transport_error_first_read(): assert len(invocations) == 2 mock_prisma_client.attempt_db_reconnect.assert_awaited_once() reconnect_kwargs = mock_prisma_client.attempt_db_reconnect.await_args.kwargs - assert ( - reconnect_kwargs["reason"] - == "sync_tool_policy_from_db_tools_lookup_failure" - ) + assert reconnect_kwargs["reason"] == "sync_tool_policy_from_db_tools_lookup_failure" assert registry.is_initialized() @@ -421,9 +418,7 @@ async def test_sync_tool_policy_from_db_retries_on_transport_error_second_read() mock_prisma_client = MagicMock() mock_prisma_client.db.litellm_tooltable.find_many = AsyncMock(return_value=[]) - mock_prisma_client.db.litellm_objectpermissiontable.find_many = AsyncMock( - side_effect=_flaky_perms_find_many - ) + mock_prisma_client.db.litellm_objectpermissiontable.find_many = AsyncMock(side_effect=_flaky_perms_find_many) mock_prisma_client.attempt_db_reconnect = AsyncMock(return_value=True) mock_prisma_client._db_auth_reconnect_timeout_seconds = 2.0 mock_prisma_client._db_auth_reconnect_lock_timeout_seconds = 0.1 @@ -433,7 +428,142 @@ async def test_sync_tool_policy_from_db_retries_on_transport_error_second_read() assert len(perms_invocations) == 2 mock_prisma_client.attempt_db_reconnect.assert_awaited_once() reconnect_kwargs = mock_prisma_client.attempt_db_reconnect.await_args.kwargs - assert ( - reconnect_kwargs["reason"] - == "sync_tool_policy_from_db_perms_lookup_failure" + assert reconnect_kwargs["reason"] == "sync_tool_policy_from_db_perms_lookup_failure" + + +class _InMemoryRedis: + def __init__(self) -> None: + self.store: dict[str, str] = {} + self.lists: dict[str, list[object]] = {} + + async def async_set_cache(self, key: str, value: str, nx: bool = False, **kwargs: object) -> bool | None: + if nx and key in self.store: + return None + self.store[key] = json.dumps(value) + return True + + async def async_get_cache(self, key: str, **kwargs: object) -> str | None: + raw = self.store.get(key) + return json.loads(raw) if raw is not None else None + + async def async_delete_cache(self, key: str) -> int: + return 1 if self.store.pop(key, None) is not None else 0 + + async def async_rpush_pipeline(self, rpush_list: list[RedisPipelineRpushOperation]) -> list[int]: + for op in rpush_list: + self.lists.setdefault(op["key"], []).extend(op["values"]) + return [len(self.lists[op["key"]]) for op in rpush_list] + + async def async_lpop_pipeline(self, lpop_list: list[RedisPipelineLpopOperation]) -> list[list[object] | None]: + return [self.lists.pop(op["key"], None) for op in lpop_list] + + +def _postgres_rejection(sqlstate: str) -> RawQueryError: + return RawQueryError( + data={"user_facing_error": {"error_code": "P2010", "meta": {"code": sqlstate, "message": "db error"}}} ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "lock_held_by_another_pod", + [None, DB_SPEND_UPDATE_JOB_NAME, TOOL_DISCOVERY_COMMIT_JOB_NAME], + ids=["free", "spend_lock_held", "tool_lock_held"], +) +async def test_commit_with_redis_writes_every_workers_tools_in_one_statement_from_the_tool_lock_holder_only( + lock_held_by_another_pod: str | None, +): + redis = _InMemoryRedis() + if lock_held_by_another_pod is not None: + assert await PodLockManager(redis_cache=redis).acquire_lock(cronjob_id=lock_held_by_another_pod) + other_worker = ToolDiscoveryQueue() + other_worker.add_update({"tool_name": "get_weather", "origin": "mcp_server"}) + other_worker.add_update({"tool_name": "search", "origin": "user_defined"}) + await RedisUpdateBuffer(redis_cache=redis).store_tool_discoveries_in_redis(other_worker) + + queue = ToolDiscoveryQueue() + queue.add_update({"tool_name": "get_weather", "origin": "user_defined"}) + queue.add_update({"tool_name": "get_weather", "origin": "user_defined"}) + prisma_client = MagicMock() + prisma_client.db.execute_raw = AsyncMock(return_value=2) + prisma_client.db.litellm_tooltable.upsert = AsyncMock() + + await commit_tool_discoveries( + prisma_client=prisma_client, + tool_discovery_queue=queue, + redis_update_buffer=RedisUpdateBuffer(redis_cache=redis), + pod_lock_manager=PodLockManager(redis_cache=redis), + use_redis_buffer=True, + ) + + assert queue.flush() == [], "every worker hands its tools to redis each tick" + prisma_client.db.litellm_tooltable.upsert.assert_not_awaited() + if lock_held_by_another_pod == TOOL_DISCOVERY_COMMIT_JOB_NAME: + prisma_client.db.execute_raw.assert_not_awaited() + assert len(redis.lists[REDIS_TOOL_DISCOVERY_BUFFER_KEY]) == 2, "non-holders leave their tools for the holder" + return + assert REDIS_TOOL_DISCOVERY_BUFFER_KEY not in redis.lists, "the spend lock must not gate the tool drain" + prisma_client.db.execute_raw.assert_awaited_once() + sql, *params = prisma_client.db.execute_raw.call_args.args + assert sql.startswith('INSERT INTO "LiteLLM_ToolTable"') and 'ON CONFLICT ("tool_name")' in sql + rows = {params[i + 1]: params[i + 3] for i in range(0, len(params), 10)} + assert rows == {"get_weather": 3, "search": 1}, f"one row per tool summed across workers, got {params}" + + +@pytest.mark.asyncio +async def test_commit_without_redis_writes_only_this_workers_tools_in_one_statement_and_never_touches_redis(): + redis = _InMemoryRedis() + queue = ToolDiscoveryQueue() + queue.add_update({"tool_name": "get_weather", "origin": "user_defined"}) + queue.add_update({"tool_name": "get_weather", "origin": "user_defined"}) + queue.add_update({"tool_name": "search", "origin": "user_defined"}) + prisma_client = MagicMock() + prisma_client.db.execute_raw = AsyncMock(return_value=2) + + await commit_tool_discoveries( + prisma_client=prisma_client, + tool_discovery_queue=queue, + redis_update_buffer=RedisUpdateBuffer(redis_cache=redis), + pod_lock_manager=PodLockManager(redis_cache=redis), + use_redis_buffer=False, + ) + + assert redis.lists == {}, "without the buffer flag nothing goes through redis" + assert queue.flush() == [] + prisma_client.db.execute_raw.assert_awaited_once() + sql, *params = prisma_client.db.execute_raw.call_args.args + assert 'ON CONFLICT ("tool_name")' in sql + rows = {params[i + 1]: params[i + 3] for i in range(0, len(params), 10)} + assert rows == {"get_weather": 2, "search": 1}, f"got {params}" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("failure", "requeued"), + [ + (Exception("connection reset"), True), + (_postgres_rejection("23502"), False), + ], + ids=["connection_error_goes_back_to_redis", "rejected_row_is_dropped"], +) +async def test_failed_tool_table_write_requeues_the_rows_only_when_they_provably_did_not_apply( + failure: Exception, requeued: bool +): + redis = _InMemoryRedis() + queue = ToolDiscoveryQueue() + queue.add_update({"tool_name": "get_weather", "origin": "user_defined"}) + prisma_client = MagicMock() + prisma_client.db.execute_raw = AsyncMock(side_effect=failure) + + await commit_tool_discoveries( + prisma_client=prisma_client, + tool_discovery_queue=queue, + redis_update_buffer=RedisUpdateBuffer(redis_cache=redis), + pod_lock_manager=PodLockManager(redis_cache=redis), + use_redis_buffer=True, + ) + + prisma_client.db.execute_raw.assert_awaited_once() + left_in_redis = [json.loads(entry) for entry in redis.lists.get(REDIS_TOOL_DISCOVERY_BUFFER_KEY, [])] + expected = [[{"tool_name": "get_weather", "origin": "user_defined", "call_count": 1}]] if requeued else [] + assert left_in_redis == expected, f"requeued={requeued} but redis holds {left_in_redis}" diff --git a/tests/unit/proxy/db/test_tool_table_bulk_upsert.py b/tests/unit/proxy/db/test_tool_table_bulk_upsert.py new file mode 100644 index 00000000000..d1e79030f9d --- /dev/null +++ b/tests/unit/proxy/db/test_tool_table_bulk_upsert.py @@ -0,0 +1,38 @@ +"""Tests for the single-statement LiteLLM_ToolTable upsert (LIT-9997).""" + +from litellm.proxy.db.tool_table_bulk_upsert import build_tool_bulk_upsert + + +def test_one_row_per_item_and_the_conflict_update_only_moves_the_counters(): + sql, params = build_tool_bulk_upsert( + [ + {"tool_name": "tool_a", "origin": "mcp_server", "created_by": None, "call_count": 4}, + { + "tool_name": "tool_b", + "origin": "user_defined", + "created_by": "alice", + "call_count": 1, + "key_hash": "hash", + "team_id": "team", + "key_alias": "alias", + "user_agent": "agent", + }, + ] + ) + assert sql.startswith('INSERT INTO "LiteLLM_ToolTable"') + assert sql.count("$1::text") == 1 and "$11::text" in sql and "$21::text" not in sql + assert "$4::integer" in sql and "$14::integer" in sql + assert 'ON CONFLICT ("tool_name") DO UPDATE SET' in sql + assert '"call_count" = "LiteLLM_ToolTable"."call_count" + EXCLUDED."call_count"' in sql + assert '"last_used_at" = EXCLUDED."last_used_at"' in sql + assert '"input_policy"' not in sql and '"output_policy"' not in sql, "policies stay on the table defaults" + assert '"origin" =' not in sql and '"key_hash" =' not in sql, "an existing row keeps who discovered it" + assert len(params) == 20 + assert params[1:10] == ("tool_a", "mcp_server", 4, "system", "system", None, None, None, None) + assert params[11:20] == ("tool_b", "user_defined", 1, "alice", "alice", "hash", "team", "alias", "agent") + assert params[0] != params[10], "each row gets its own tool_id" + + +def test_defaults_fill_origin_created_by_and_call_count(): + _, params = build_tool_bulk_upsert([{"tool_name": "tool_a"}]) + assert params[1:6] == ("tool_a", "user_defined", 1, "system", "system") From d0671f75a4bf87e9fcaea38498be005aa3931b12 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:39:27 -0700 Subject: [PATCH 015/155] refactor(typing): widen delete_nested_value to Mapping and drop two unused Span aliases (#48001) * refactor(typing): replace pass-through Any with object in 13 annotations * docs(client): say dict[str, object] in the models get docstring * refactor(typing): drop the unused Span aliases in base_cache and levo * refactor(typing): take a Mapping in delete_nested_value and cover the fresh copy * fix(typing): deep-copy a read-only Mapping in delete_nested_value through a dict --- litellm/caching/base_cache.py | 9 +-------- litellm/integrations/levo/levo.py | 4 ---- .../litellm_core_utils/dot_notation_indexing.py | 4 ++-- litellm/proxy/client/models.py | 2 +- .../test_dot_notation_indexing.py | 14 ++++++++++++++ 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/litellm/caching/base_cache.py b/litellm/caching/base_cache.py index fb91d13a8c9..4dc894a64ba 100644 --- a/litellm/caching/base_cache.py +++ b/litellm/caching/base_cache.py @@ -9,14 +9,7 @@ Has 4 methods: """ from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, Final - -if TYPE_CHECKING: - from opentelemetry.trace import Span as _Span - - Span = _Span -else: - Span = Any +from typing import Final class BaseCache(ABC): diff --git a/litellm/integrations/levo/levo.py b/litellm/integrations/levo/levo.py index 6273f39e09c..c1852e9bfe4 100644 --- a/litellm/integrations/levo/levo.py +++ b/litellm/integrations/levo/levo.py @@ -4,8 +4,6 @@ from typing import TYPE_CHECKING, Any, Final from litellm.integrations.opentelemetry import OpenTelemetry if TYPE_CHECKING: - from opentelemetry.trace import Span as _Span - from litellm.integrations.opentelemetry import ( OpenTelemetryConfig as _OpenTelemetryConfig, ) @@ -13,11 +11,9 @@ if TYPE_CHECKING: Protocol = _Protocol OpenTelemetryConfig = _OpenTelemetryConfig - Span = _Span else: Protocol = Any OpenTelemetryConfig = Any - Span = Any class LevoConfig: diff --git a/litellm/litellm_core_utils/dot_notation_indexing.py b/litellm/litellm_core_utils/dot_notation_indexing.py index 09720fa413c..818cf1b9695 100644 --- a/litellm/litellm_core_utils/dot_notation_indexing.py +++ b/litellm/litellm_core_utils/dot_notation_indexing.py @@ -181,7 +181,7 @@ def _delete_nested_value_custom( def delete_nested_value( - data: dict[str, object], + data: Mapping[str, object], path: str, depth: int = 0, max_depth: int = 20, @@ -214,7 +214,7 @@ def delete_nested_value( """ import copy - result: Final = copy.deepcopy(data) + result: Final = copy.deepcopy(data if isinstance(data, dict) else dict(data)) try: # Parse path into segments diff --git a/litellm/proxy/client/models.py b/litellm/proxy/client/models.py index 7342974c0cc..1361a3e6364 100644 --- a/litellm/proxy/client/models.py +++ b/litellm/proxy/client/models.py @@ -168,7 +168,7 @@ class ModelsManagementClient: return_request (bool): If True, returns the prepared request object instead of executing it Returns: - Union[Dict[str, Any], requests.Request]: Either the model information from the server or + dict[str, object] | requests.Request: Either the model information from the server or a prepared request object if return_request is True Raises: diff --git a/tests/unit/litellm_core_utils/test_dot_notation_indexing.py b/tests/unit/litellm_core_utils/test_dot_notation_indexing.py index ae529a71009..7c670f1dac9 100644 --- a/tests/unit/litellm_core_utils/test_dot_notation_indexing.py +++ b/tests/unit/litellm_core_utils/test_dot_notation_indexing.py @@ -2,6 +2,9 @@ Tests for litellm.litellm_core_utils.dot_notation_indexing module. """ +from types import MappingProxyType +from typing import Final + import pytest from litellm.litellm_core_utils.dot_notation_indexing import ( @@ -125,3 +128,14 @@ class TestDeleteNestedValue: data = {"items": [{"a": 1, "b": 2}, {"a": 3, "b": 4}]} result = delete_nested_value(data, "items[0].b") assert result == {"items": [{"a": 1}, {"a": 3, "b": 4}]} + + def test_delete_returns_a_fresh_dict_and_leaves_a_narrowly_typed_input_untouched(self): + payload: Final[dict[str, list[str]]] = {"tags": ["a", "b"], "secret": ["s"]} + result = delete_nested_value(payload, "secret") + assert result == {"tags": ["a", "b"]} + assert payload == {"tags": ["a", "b"], "secret": ["s"]} + assert result["tags"] is not payload["tags"] + + def test_delete_accepts_a_read_only_mapping(self): + payload: Final = MappingProxyType({"tags": ["a"], "secret": ["s"]}) + assert delete_nested_value(payload, "secret") == {"tags": ["a"]} From 953ee1914a3d70602008d18674ba6516c4ed1f33 Mon Sep 17 00:00:00 2001 From: hgn <35766626+hgnnmnn@users.noreply.github.com> Date: Sat, 24 Oct 2026 21:43:38 +0200 Subject: [PATCH 016/155] fix(ui): wire up Download CSV Template button in bulk invite (#48199) * fix(ui): wire up Download CSV Template button in bulk invite * refactor(ui): reuse shared downloadBlob helper for bulk invite CSV template --------- Co-authored-by: ryan-crabbe-berri --- .../bulk_create_users_button.test.tsx | 30 +++++++++++++++++++ .../components/bulk_create_users_button.tsx | 12 +++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/bulk_create_users_button.test.tsx b/ui/litellm-dashboard/src/components/bulk_create_users_button.test.tsx index e1faf8be1df..51d7521c909 100644 --- a/ui/litellm-dashboard/src/components/bulk_create_users_button.test.tsx +++ b/ui/litellm-dashboard/src/components/bulk_create_users_button.test.tsx @@ -30,6 +30,36 @@ describe("BulkCreateUsersButton", () => { expect(screen.getByText("+ Bulk Invite Users")).toBeInTheDocument(); }); + it("downloads a CSV template with the columns the upload accepts", async () => { + const createObjectURL = vi.fn((_blob: Blob) => "blob:template"); + const revokeObjectURL = vi.fn(); + const { createObjectURL: originalCreate, revokeObjectURL: originalRevoke } = URL; + URL.createObjectURL = createObjectURL; + URL.revokeObjectURL = revokeObjectURL; + const anchorClick = vi.spyOn(HTMLAnchorElement.prototype, "click").mockImplementation(() => {}); + + const user = await openUploadStep(); + await user.click(screen.getByRole("button", { name: /download csv template/i })); + + expect(anchorClick).toHaveBeenCalledOnce(); + const anchor = anchorClick.mock.contexts[0] as HTMLAnchorElement; + expect(anchor.download).toBe("bulk_users_template.csv"); + expect(anchor.href).toBe("blob:template"); + + const csv = await new Promise((resolve) => { + const reader = new FileReader(); + reader.onload = () => resolve(reader.result as string); + reader.readAsText(createObjectURL.mock.calls[0][0]); + }); + const [headerRow] = csv.split(/\r?\n/); + expect(headerRow).toBe("user_email,user_role,teams,max_budget,budget_duration,models"); + expect(revokeObjectURL).toHaveBeenCalledWith("blob:template"); + + anchorClick.mockRestore(); + URL.createObjectURL = originalCreate; + URL.revokeObjectURL = originalRevoke; + }); + it("parses a CSV chosen through the file input", async () => { await openUploadStep(); diff --git a/ui/litellm-dashboard/src/components/bulk_create_users_button.tsx b/ui/litellm-dashboard/src/components/bulk_create_users_button.tsx index 8b823ca2f73..840cc1af00e 100644 --- a/ui/litellm-dashboard/src/components/bulk_create_users_button.tsx +++ b/ui/litellm-dashboard/src/components/bulk_create_users_button.tsx @@ -16,6 +16,7 @@ import Papa from "papaparse"; import { CheckCircleIcon, XCircleIcon, ExclamationIcon } from "@heroicons/react/outline"; import { CopyToClipboard } from "react-copy-to-clipboard"; import { toast } from "@/lib/toast"; +import { downloadBlob } from "@/components/EntityUsageExport/utils"; interface BulkCreateUsersProps { accessToken: string; @@ -87,6 +88,15 @@ const BulkCreateUsersButton: React.FC = ({ setBaseUrl(base.toString()); }, [accessToken]); + const downloadTemplate = () => { + const template = [ + ["user_email", "user_role", "teams", "max_budget", "budget_duration", "models"], + ["user@example.com", "internal_user", "team-id-1,team-id-2", "100", "30d", "gpt-3.5-turbo,gpt-4"], + ]; + + downloadBlob(new Blob([Papa.unparse(template)], { type: "text/csv" }), "bulk_users_template.csv"); + }; + const handleFileUpload = (file: File) => { // Reset all error states setParseError(null); @@ -609,7 +619,7 @@ const BulkCreateUsersButton: React.FC = ({
- From 55479821e22f07c64fb5ecdbbc708b980da9ec2d Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 24 Oct 2026 13:00:58 -0700 Subject: [PATCH 017/155] feat(router): add opt-in session repinning after successful failover (#48206) * feat(router): add opt-in session repinning after successful failover * fix(router): respect shared session pins and split metadata * fix(router): preserve session routing during Redis failures * test(router): inject the delayed Redis reader --- litellm/caching/affinity_cache.py | 29 +- litellm/caching/redis_cache.py | 6 +- litellm/router.py | 42 +++ .../deployment_affinity_check.py | 105 +++++- .../integration/sdk/test_dual_cache_redis.py | 64 ++++ .../test_session_id_affinity.py | 348 ++++++++++++++++++ 6 files changed, 587 insertions(+), 7 deletions(-) diff --git a/litellm/caching/affinity_cache.py b/litellm/caching/affinity_cache.py index 2d27574b098..d9a40c44d2b 100644 --- a/litellm/caching/affinity_cache.py +++ b/litellm/caching/affinity_cache.py @@ -24,6 +24,16 @@ if current == false then end if ARGV[3] then local decoded, stored = pcall(cjson.decode, current) + if ARGV[4] then + local legacy = current + if decoded and type(stored) == 'string' then legacy = stored end + for _, eligible in ipairs(cjson.decode(ARGV[3])) do + if eligible['model_id'] == legacy then + redis.call('EXPIRE', KEYS[1], ARGV[2]) + return current + end + end + end if decoded and type(stored) == 'table' then for _, eligible in ipairs(cjson.decode(ARGV[3])) do local matches = true @@ -68,6 +78,7 @@ def claim_affinity_pin_in_memory( ttl_seconds: int, *, eligible_values: tuple[Mapping[str, str], ...] | None = None, + preserve_legacy_model_ids: bool = False, ) -> object: """No await between read and write, so same-loop claims agree during a Redis outage.""" existing: Final[object] = cache.in_memory_cache.get_cache(cache_key) @@ -75,7 +86,12 @@ def claim_affinity_pin_in_memory( if _legacy_pin_matches(existing, pin_value): set_local_affinity_pin(cache, cache_key, pin_value, ttl_seconds) return existing - winner: Final = existing if existing is not None and existing in (eligible_values or ()) else pin_value + eligible_existing: Final = existing in (eligible_values or ()) or ( + preserve_legacy_model_ids + and isinstance(existing, str) + and any("model_id" in value and _legacy_pin_matches(existing, value) for value in (eligible_values or ())) + ) + winner: Final = existing if existing is not None and eligible_existing else pin_value set_local_affinity_pin(cache, cache_key, winner, ttl_seconds) return winner @@ -94,6 +110,7 @@ async def claim_affinity_pin( ttl_seconds: int, *, eligible_values: tuple[Mapping[str, str], ...] | None = None, + preserve_legacy_model_ids: bool = False, ) -> object: """Return the authoritative first writer, replacing it only when it becomes ineligible. @@ -112,6 +129,7 @@ async def claim_affinity_pin( if eligible_values is not None else () ), + *(("1",) if preserve_legacy_model_ids and eligible_values is not None else ()), ) raw: Final = cast( # cast-ok: Redis scripts return heterogeneous values; only object is asserted here object, await claim_script(keys=(cache_key,), args=args) @@ -124,4 +142,11 @@ async def claim_affinity_pin( return winner except Exception as error: # noqa: BLE001 # Redis/Lua faults retain same-pod affinity through local claims verbose_router_logger.debug("Affinity cache: Redis claim failed, using pod-local claim. error=%s", error) - return claim_affinity_pin_in_memory(cache, cache_key, pin_value, ttl_seconds, eligible_values=eligible_values) + return claim_affinity_pin_in_memory( + cache, + cache_key, + pin_value, + ttl_seconds, + eligible_values=eligible_values, + preserve_legacy_model_ids=preserve_legacy_model_ids, + ) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index eb17facfb12..afe6f8a1eab 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -1738,7 +1738,9 @@ class RedisCache(BaseCache): return key_value_dict @_redis_circuit_breaker_guard - async def async_get_cache(self, key, parent_otel_span: Span | None = None, **kwargs): + async def async_get_cache( + self, key, parent_otel_span: Span | None = None, *, raise_on_error: bool = False, **kwargs + ): from redis.asyncio import Redis _redis_client: Final[Redis] = self.init_async_client() @@ -1781,6 +1783,8 @@ class RedisCache(BaseCache): ) ) print_verbose(f"litellm.caching.caching: async get() - Got exception from REDIS: {e}") + if raise_on_error: + raise _record_swallowed_redis_failure(self._circuit_breaker, e) @_redis_circuit_breaker_guard diff --git a/litellm/router.py b/litellm/router.py index 25591a435f9..96102b678c3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1247,6 +1247,7 @@ class Router: fallback_access_check: FallbackAccessCheck | None = None, fallback_budget_check: FallbackBudgetCheck | None = None, auto_router_capability_limit: AutoRouterCapabilityLimit | None = None, + session_affinity_repin_on_failover: bool = False, ) -> None: """ Initialize the Router class with the given parameters for caching, reliability, and routing strategy. @@ -1283,6 +1284,8 @@ class Router: alerting_config (AlertingConfig): Slack alerting configuration. Defaults to None. provider_budget_config (ProviderBudgetConfig): Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None. deployment_affinity_ttl_seconds (int): TTL for user-key -> deployment affinity mapping. Defaults to 3600. + session_affinity_repin_on_failover (bool): After a successful replacement response, move a session pin + whose deployment is in cooldown or removed. Async routes only. Defaults to False. Applied at startup. ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error. enable_weighted_failover (bool): When True and the routing strategy is "simple-shuffle", a retryable failure on one deployment causes the request to re-pick (weighted) across the other deployments in the same model group before any cross-group fallback runs. Bounded by `max_fallbacks`. Async-only: currently honored by `router.acompletion()` and other async entrypoints. The sync `router.completion()` path falls back to the regular fallback flow. Defaults to False. enable_mid_stream_fallback_continuation (bool): When True, a chat-completions stream that breaks after plain assistant text has been delivered continues on a fallback deployment via assistant prefill instead of surfacing the error. Only deployments whose model supports assistant prefill are eligible, so the partial text is continued, not regenerated; if none is, the original error is surfaced. Streams that emitted tool calls, thinking blocks, audio/images, or constrained (JSON / forced tool_choice) output are never continued. Async-only. Defaults to False. @@ -1467,6 +1470,7 @@ class Router: self._provider_unresolved_deployments: tuple[Callable[[], Deployment | None], ...] = () self.deployment_affinity_ttl_seconds = deployment_affinity_ttl_seconds + self.session_affinity_repin_on_failover = session_affinity_repin_on_failover self.model_group_affinity_config = model_group_affinity_config warn_on_unknown_model_group_affinity_flags(model_group_affinity_config) @@ -2862,6 +2866,7 @@ class Router: enable_session_id_affinity=False, model_group_affinity_config=self.model_group_affinity_config, is_priority_group=self._is_priority_routing_group, + session_affinity_repin_on_failover=self.session_affinity_repin_on_failover, ) self.optional_callbacks.append(affinity_callback) litellm.logging_callback_manager.add_litellm_callback(affinity_callback) @@ -2908,6 +2913,7 @@ class Router: enable_session_id_affinity=enable_session_id_affinity, model_group_affinity_config=self.model_group_affinity_config, is_priority_group=self._is_priority_routing_group, + session_affinity_repin_on_failover=self.session_affinity_repin_on_failover, ) self.optional_callbacks.append(affinity_callback) litellm.logging_callback_manager.add_litellm_callback(affinity_callback) @@ -9300,6 +9306,12 @@ class Router: if model_group is None or deployment_id is None or self.get_deployment(model_id=str(deployment_id)) is None: return + if self.session_affinity_repin_on_failover: + await self._repin_session_after_success( + _LITELLM_PARAMS_ADAPTER.validate_python(litellm_params), + _OPAQUE_VALUE_ADAPTER.validate_python(completion_response), + ) + # Always track deployment successes for cooldown logic, regardless of TPM/RPM limits increment_deployment_successes_for_current_minute( litellm_router_instance=self, @@ -9323,6 +9335,36 @@ class Router: "litellm.router.Router::deployment_callback_on_success(): Exception occured - %s", e ) + async def _repin_session_after_success(self, litellm_params: Mapping[str, object], response: object) -> None: + if not self.session_affinity_repin_on_failover: + return + try: + from litellm.router_utils.pre_call_checks.deployment_affinity_check import completed_affinity_response + + if not completed_affinity_response(response): + return + callback: Final = next( + (logger for logger in self._optional_loggers() if isinstance(logger, DeploymentAffinityCheck)), + None, + ) + if callback is None: + return + model_group: Final = callback.get_repin_model_group(litellm_params) + if model_group is None or self._is_priority_routing_group(model_group): + return + registered_ids: Final = self.get_candidate_model_ids_for_route( + model=model_group, team_id=get_request_team_id(litellm_params) + ) + cooldown_ids: Final = frozenset( + model_id + for model_id, _ in await self.cooldown_cache.async_get_active_cooldowns( + model_ids=list(registered_ids), parent_otel_span=None + ) + ) + await callback.async_repin_session_after_success(litellm_params, model_group, registered_ids - cooldown_ids) + except Exception as error: # noqa: BLE001 # Optional pin updates must not interrupt success accounting + verbose_router_logger.debug("Session re-pinning failed: %s", error) + async def increment_deployment_usage_for_response( self, response: object, diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index 88fb8876175..8693eaa5151 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -16,6 +16,7 @@ import hashlib from collections.abc import Callable, Mapping, Sequence from typing import Final, cast +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm._internal_context import with_service_target @@ -38,6 +39,27 @@ class DeploymentAffinityCacheValue(TypedDict, closed=True): model_id: ReadOnly[str] +def _affinity_response_fields(response: object) -> Mapping[str, object]: + return ( + TypeAdapter(Mapping[str, object]).validate_python(response) + if isinstance(response, Mapping) + else {name: getattr(response, name, None) for name in ("response", "status", "error")} + ) + + +def completed_affinity_response(response: object) -> bool: + if response is None: + return False + envelope: Final = _affinity_response_fields(response) + if envelope.get("error") is not None or envelope.get("status") not in (None, "completed"): + return False + nested: Final = envelope.get("response") + fields: Final = _affinity_response_fields(nested) if nested is not None else envelope + return ( + fields.get("response") is None and fields.get("error") is None and fields.get("status") in (None, "completed") + ) + + VALID_MODEL_GROUP_AFFINITY_FLAGS: Final = frozenset( { "deployment_affinity", @@ -86,6 +108,7 @@ class DeploymentAffinityCheck(CustomLogger): enable_session_id_affinity: bool = False, model_group_affinity_config: dict[str, list[str]] | None = None, is_priority_group: Callable[[str], bool] | None = None, + session_affinity_repin_on_failover: bool = False, ): super().__init__() self.cache = cache @@ -95,6 +118,7 @@ class DeploymentAffinityCheck(CustomLogger): self.enable_session_id_affinity = enable_session_id_affinity self.model_group_affinity_config: dict[str, list[str]] = model_group_affinity_config or {} self.is_priority_group = is_priority_group + self.session_affinity_repin_on_failover = session_affinity_repin_on_failover def _get_effective_flags(self, model_group: str) -> tuple[bool, bool, bool]: """ @@ -297,7 +321,7 @@ class DeploymentAffinityCheck(CustomLogger): return f"{cls.USER_ID_AFFINITY_PREFIX}{user_id}" @staticmethod - def _get_session_id_from_request_kwargs(request_kwargs: dict) -> str | None: + def _get_session_id_from_request_kwargs(request_kwargs: Mapping[str, object]) -> str | None: for metadata in DeploymentAffinityCheck._iter_metadata_dicts(request_kwargs): session_id = DeploymentAffinityCheck._get_session_id_from_metadata_dict(metadata=metadata) if session_id is not None: @@ -305,7 +329,7 @@ class DeploymentAffinityCheck(CustomLogger): return None @staticmethod - def _get_marker_session_affinity_ttl(request_kwargs: dict) -> int | None: + def _get_marker_session_affinity_ttl(request_kwargs: Mapping[str, object]) -> int | None: """TTL from the session-affinity marker the Router stamps at pre-routing time when an auto-router routed this request with session_affinity enabled. Marker presence enables session pinning for this request only; anything that @@ -357,7 +381,7 @@ class DeploymentAffinityCheck(CustomLogger): model: str, healthy_deployments: list, messages: list[AllMessageValues] | None, - request_kwargs: dict | None = None, + request_kwargs: dict[str, object] | None = None, parent_otel_span: Span | None = None, ) -> list[dict]: """ @@ -418,7 +442,15 @@ class DeploymentAffinityCheck(CustomLogger): session_cache_key: Final = self.get_session_affinity_cache_key( model_group=stable_model_map_key, session_id=session_id, user_key=user_key ) - session_cache_result: Final = await self.cache.async_get_cache(key=session_cache_key) + session_cache_result: Final = ( + await self._read_shared_session_pin( + session_cache_key, + self._get_marker_session_affinity_ttl(request_kwargs) or self.ttl_seconds, + parent_otel_span, + ) + if self.session_affinity_repin_on_failover + else await self.cache.async_get_cache(key=session_cache_key) + ) session_model_id: str | None = None if isinstance(session_cache_result, dict): @@ -482,6 +514,71 @@ class DeploymentAffinityCheck(CustomLogger): ) return [deployment] + async def _read_shared_session_pin(self, cache_key: str, ttl_seconds: int, parent_otel_span: Span | None) -> object: + redis_cache: Final = self.cache.redis_cache + if redis_cache is None: + return TypeAdapter(object).validate_python(await self.cache.async_get_cache(key=cache_key)) + local_before: Final = TypeAdapter(object).validate_python(self.cache.in_memory_cache.get_cache(cache_key)) + try: + shared: Final[object] = await redis_cache.async_get_cache( + key=cache_key, parent_otel_span=parent_otel_span, raise_on_error=True + ) + except Exception: # noqa: BLE001 # Optional affinity falls back locally on Redis admission or read failure + verbose_router_logger.debug("Session pin read failed, using pod-local affinity") + return TypeAdapter(object).validate_python(await self.cache.async_get_cache(key=cache_key, local_only=True)) + if self.cache.in_memory_cache.get_cache(cache_key) is local_before: + if shared is None: + self.cache.in_memory_cache.delete_cache(cache_key) + else: + self._set_local_pin(cache_key, shared, ttl_seconds) + return shared + + def get_repin_model_group(self, request_kwargs: Mapping[str, object]) -> str | None: + model_group: Final = next( + ( + value + for metadata in self._iter_metadata_dicts(request_kwargs) + if isinstance(value := metadata.get("deployment_model_name"), str) and value + ), + None, + ) + if model_group is None or self._get_session_id_from_request_kwargs(request_kwargs) is None: + return None + if ( + not self._get_effective_flags(model_group)[2] + and self._get_marker_session_affinity_ttl(request_kwargs) is None + ): + return None + return model_group + + async def async_repin_session_after_success( + self, + request_kwargs: Mapping[str, object], + model_group: str, + eligible_deployment_ids: frozenset[str], + ) -> None: + marker_ttl: Final = self._get_marker_session_affinity_ttl(request_kwargs) + if not self._get_effective_flags(model_group)[2] and marker_ttl is None: + return + session_id: Final = self._get_session_id_from_request_kwargs(request_kwargs) + if session_id is None: + return + model_info: Final = TypeAdapter(Mapping[str, object]).validate_python(request_kwargs.get("model_info") or {}) + model_id: Final = model_info.get("id") + if model_id is None or str(model_id) not in eligible_deployment_ids: + return + cache_key: Final = self.get_session_affinity_cache_key( + model_group, session_id, self.get_user_key_from_request_kwargs(request_kwargs) + ) + await claim_affinity_pin( + self.cache, + cache_key, + DeploymentAffinityCacheValue(model_id=str(model_id)), + marker_ttl if marker_ttl is not None else self.ttl_seconds, + eligible_values=tuple(DeploymentAffinityCacheValue(model_id=value) for value in eligible_deployment_ids), + preserve_legacy_model_ids=True, + ) + async def async_pre_call_deployment_hook( self, kwargs: dict[str, object], call_type: CallTypes | None ) -> dict | None: diff --git a/tests/integration/sdk/test_dual_cache_redis.py b/tests/integration/sdk/test_dual_cache_redis.py index ddd01480d36..3527dbf91ee 100644 --- a/tests/integration/sdk/test_dual_cache_redis.py +++ b/tests/integration/sdk/test_dual_cache_redis.py @@ -6,6 +6,7 @@ from unittest.mock import patch import pytest +from litellm.caching.affinity_cache import claim_affinity_pin from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import RedisCache @@ -92,3 +93,66 @@ async def test_sync_and_async_batch_reads_share_one_redis_without_sync_reads_goi assert in_loop_results == [[expected[key]] for key in in_loop_keys] assert await dual_cache.async_batch_get_cache(keys=[survivor_key]) == [expected[survivor_key]] + + +@pytest.mark.asyncio +async def test_concurrent_session_repin_claims_keep_the_shared_winner_after_recovery() -> None: + redis: Final = _redis_cache() + workers: Final = (DualCache(redis_cache=redis), DualCache(redis_cache=redis)) + key: Final = f"session-repin-{uuid.uuid4().hex}" + choices: Final = ({"model_id": "replacement"}, {"model_id": "other"}) + try: + await redis.async_set_cache(key, {"model_id": "original"}, ttl=60) + for worker in workers: + worker.in_memory_cache.set_cache(key, {"model_id": "original"}, ttl=60) + winners: Final = await asyncio.gather( + *( + claim_affinity_pin(worker, key, choice, 60, eligible_values=choices, preserve_legacy_model_ids=True) + for worker, choice in zip(workers, choices, strict=True) + ) + ) + assert winners[0] == winners[1] and winners[0] in choices + late_original: Final = await claim_affinity_pin( + workers[0], + key, + {"model_id": "original"}, + 60, + eligible_values=({"model_id": "original"}, *choices), + preserve_legacy_model_ids=True, + ) + assert late_original == winners[0] + assert await redis.async_get_cache(key) == winners[0] + finally: + await redis.async_delete_cache(key) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stored", ["original", '"original"', '{"model_id":"original"}']) +async def test_session_repin_lua_preserves_healthy_legacy_pins_and_replaces_unavailable_ones(stored: str) -> None: + redis: Final = _redis_cache() + cache: Final = DualCache(redis_cache=redis) + key: Final = f"session-repin-legacy-{uuid.uuid4().hex}" + replacement: Final = {"model_id": "replacement"} + try: + redis.redis_client.set(redis.check_and_fix_namespace(key), stored, ex=60) + preserved: Final = await claim_affinity_pin( + cache, + key, + replacement, + 60, + eligible_values=({"model_id": "original"}, replacement), + preserve_legacy_model_ids=True, + ) + assert preserved in ("original", {"model_id": "original"}) + moved: Final = await claim_affinity_pin( + cache, + key, + replacement, + 60, + eligible_values=(replacement,), + preserve_legacy_model_ids=True, + ) + assert moved == replacement + assert await redis.async_get_cache(key) == replacement + finally: + await redis.async_delete_cache(key) diff --git a/tests/unit/router_utils/pre_call_checks/test_session_id_affinity.py b/tests/unit/router_utils/pre_call_checks/test_session_id_affinity.py index 9bbaed0ae1b..6b4ea74894a 100644 --- a/tests/unit/router_utils/pre_call_checks/test_session_id_affinity.py +++ b/tests/unit/router_utils/pre_call_checks/test_session_id_affinity.py @@ -1,20 +1,368 @@ import asyncio import json from typing import Final +from collections.abc import AsyncIterator, Mapping +from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, patch import pytest +from fakeredis import FakeRedis, FakeServer +from fakeredis.aioredis import FakeConnection +from redis.asyncio import BlockingConnectionPool, Redis import litellm from litellm.caching.affinity_cache import claim_affinity_pin from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache +from litellm.caching.redis_cache import RedisCache from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, ) +def _repinning_router(enabled: bool | None = True) -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "repin-group", + "litellm_params": {"model": "openai/test-model", "api_key": "fake-api-key"}, + "model_info": {"id": deployment_id}, + } + for deployment_id in ("original", "replacement", "other") + ] + + [ + { + "model_name": "unrelated-group", + "litellm_params": {"model": "openai/test-model", "api_key": "fake-api-key"}, + "model_info": {"id": "unrelated"}, + } + ], + optional_pre_call_checks=["session_affinity", "deployment_affinity"], + **({"session_affinity_repin_on_failover": enabled} if enabled is not None else {}), + ) + + +def _repin_params(deployment_id: str, caller: str = "caller-1") -> dict[str, object]: + return { + "model_info": {"id": deployment_id}, + "metadata": { + "model_group": "repin-group", + "deployment_model_name": "repin-group", + "session_id": "repin-session", + "user_api_key_hash": caller, + }, + } + + +def _repin_key(caller: str = "caller-1") -> str: + return DeploymentAffinityCheck.get_session_affinity_cache_key("repin-group", "repin-session", caller) + + +def _cool_down_original(router: litellm.Router) -> None: + router.cooldown_cache.add_deployment_to_cooldown( + model_id="original", original_exception=RuntimeError("test outage"), exception_status=503, cooldown_time=3600 + ) + + +async def _record_replacement_success( + router: litellm.Router, deployment_id: str = "replacement", response: object = None +) -> None: + timestamp: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + await router.deployment_callback_on_success( + {"litellm_params": _repin_params(deployment_id), "standard_logging_object": {"total_tokens": 0}}, + {"choices": [{"message": {"content": "ok"}}]} if response is None else response, + timestamp, + timestamp, + ) + + +@pytest.mark.parametrize("enabled", [None, False, True], ids=["default", "disabled", "enabled"]) +@pytest.mark.parametrize("unavailable", ["cooldown", "removed", "temporary-filter"]) +async def test_session_repin_requires_opt_in_unavailable_original_and_success( + enabled: bool | None, unavailable: str +) -> None: + router: Final = _repinning_router(enabled) + try: + await router.cache.async_set_cache(_repin_key(), {"model_id": "original"}) + await router.cache.async_set_cache(_repin_key("caller-2"), {"model_id": "original"}) + user_key: Final = DeploymentAffinityCheck.get_affinity_cache_key("repin-group", "caller-1") + await router.cache.async_set_cache(user_key, {"model_id": "original"}) + if unavailable == "cooldown": + _cool_down_original(router) + elif unavailable == "removed": + router.delete_deployment("original") + callback: Final = next(cb for cb in router.optional_callbacks or () if isinstance(cb, DeploymentAffinityCheck)) + await callback.async_pre_call_deployment_hook(_repin_params("replacement"), None) + assert await router.cache.async_get_cache(_repin_key()) == {"model_id": "original"} + await _record_replacement_success(router) + expected: Final = "replacement" if enabled and unavailable != "temporary-filter" else "original" + assert await router.cache.async_get_cache(_repin_key()) == {"model_id": expected} + assert await router.cache.async_get_cache(_repin_key("caller-2")) == {"model_id": "original"} + assert await router.cache.async_get_cache(user_key) == {"model_id": "original"} + router.cooldown_cache.cooldown_store.flush_cache() + await _record_replacement_success(router, "other") + assert await router.cache.async_get_cache(_repin_key()) == {"model_id": expected} + finally: + router.discard() + + +@pytest.mark.parametrize("status", ["failed", "incomplete", "in_progress", "cancelled"]) +@pytest.mark.parametrize("wrapped", [False, True]) +async def test_unsuccessful_responses_do_not_repin(status: str, wrapped: bool) -> None: + router: Final = _repinning_router() + try: + await router.cache.async_set_cache(_repin_key(), {"model_id": "original"}) + _cool_down_original(router) + response: Final[Mapping[str, object]] = {"status": status} + await _record_replacement_success(router, response={"response": response} if wrapped else response) + assert await router.cache.async_get_cache(_repin_key()) == {"model_id": "original"} + finally: + router.discard() + + +async def test_concurrent_successful_replacements_keep_first_winner() -> None: + router: Final = _repinning_router() + try: + await router.cache.async_set_cache(_repin_key(), {"model_id": "original"}) + _cool_down_original(router) + await asyncio.gather(_record_replacement_success(router), _record_replacement_success(router, "other")) + winner: Final = await router.cache.async_get_cache(_repin_key()) + assert winner in ({"model_id": "replacement"}, {"model_id": "other"}) + router.cooldown_cache.cooldown_store.flush_cache() + await _record_replacement_success(router, "original") + assert await router.cache.async_get_cache(_repin_key()) == winner + finally: + router.discard() + + +async def test_healthy_legacy_string_pin_is_preserved_with_repinning_enabled() -> None: + router: Final = _repinning_router() + try: + await router.cache.async_set_cache(_repin_key(), "original") + await _record_replacement_success(router) + assert await router.cache.async_get_cache(_repin_key()) == "original" + finally: + router.discard() + + +@pytest.mark.parametrize("route_bucket", ["metadata", "litellm_metadata"]) +async def test_repin_session_after_success_reads_split_metadata(route_bucket: str) -> None: + router: Final = _repinning_router() + try: + await router.cache.async_set_cache(_repin_key(), {"model_id": "original"}) + _cool_down_original(router) + session_bucket: Final = "litellm_metadata" if route_bucket == "metadata" else "metadata" + await router._repin_session_after_success( + { + "model_info": {"id": "replacement"}, + route_bucket: {"deployment_model_name": "repin-group"}, + session_bucket: {"session_id": "repin-session", "user_api_key_hash": "caller-1"}, + }, + {"status": "completed"}, + ) + assert await router.cache.async_get_cache(_repin_key()) == {"model_id": "replacement"} + finally: + router.discard() + + +@pytest.mark.parametrize("enabled", [False, True]) +async def test_session_selection_observes_another_workers_repin_only_when_enabled(enabled: bool) -> None: + server: Final = FakeServer() + sync_client: Final = FakeRedis(server=server) + pool: Final = BlockingConnectionPool(connection_class=FakeConnection, server=server) + router: Final = _repinning_router(enabled) + try: + with ( + patch("redis.Redis.from_url", return_value=sync_client), + patch("redis.asyncio.BlockingConnectionPool.from_url", return_value=pool), + ): + redis: Final = RedisCache(url=f"redis://repin-cache.example:6379/{int(enabled)}") + router.cache.attach_redis_cache(redis) + other_worker: Final = DualCache(redis_cache=redis) + router.cache.in_memory_cache.set_cache(_repin_key(), {"model_id": "original"}, ttl=3600) + await other_worker.async_set_cache(_repin_key(), {"model_id": "replacement"}, ttl=3600) + callback: Final = next( + cb for cb in router.optional_callbacks or () if isinstance(cb, DeploymentAffinityCheck) + ) + selected: Final = await callback.async_filter_deployments( + model="repin-group", + healthy_deployments=router.get_model_list(model_name="repin-group") or [], + messages=None, + request_kwargs=_repin_params("replacement"), + ) + assert [deployment["model_info"]["id"] for deployment in selected] == [ + "replacement" if enabled else "original" + ] + finally: + router.discard() + await pool.aclose() + sync_client.close() + + +@pytest.fixture +async def shared_session_router() -> AsyncIterator[tuple[litellm.Router, FakeServer, RedisCache]]: + server: Final = FakeServer() + sync_client: Final = FakeRedis(server=server) + pool: Final = BlockingConnectionPool(connection_class=FakeConnection, server=server) + router: Final = _repinning_router() + try: + with ( + patch("redis.Redis.from_url", return_value=sync_client), + patch("redis.asyncio.BlockingConnectionPool.from_url", return_value=pool), + ): + redis: Final = RedisCache(url="redis://session-outage.example:6379/0") + router.cache.attach_redis_cache(redis) + yield router, server, redis + finally: + router.discard() + await pool.aclose() + sync_client.close() + + +async def _selected_session_ids(router: litellm.Router) -> tuple[str, ...]: + callback: Final = next(cb for cb in router.optional_callbacks or () if isinstance(cb, DeploymentAffinityCheck)) + selected: Final = await callback.async_filter_deployments( + model="repin-group", + healthy_deployments=router.get_model_list(model_name="repin-group") or [], + messages=None, + request_kwargs=_repin_params("replacement"), + ) + return tuple(deployment["model_info"]["id"] for deployment in selected) + + +@pytest.mark.parametrize("has_pin", [False, True]) +async def test_redis_read_failures_and_open_breaker_preserve_session_routing( + shared_session_router: tuple[litellm.Router, FakeServer, RedisCache], has_pin: bool +) -> None: + router, server, redis = shared_session_router + if has_pin: + await redis.async_set_cache(_repin_key(), {"model_id": "replacement"}, ttl=3600) + router.cache.in_memory_cache.set_cache(_repin_key(), {"model_id": "original"}, ttl=3600) + assert await _selected_session_ids(router) == ("replacement",) + server.connected = False + with patch("time.time", return_value=1790000000.0): + for _ in range(redis._circuit_breaker.failure_threshold + 1): + assert set(await _selected_session_ids(router)) == ( + {"replacement"} if has_pin else {"original", "replacement", "other"} + ) + assert redis._circuit_breaker.is_open() + + +async def test_confirmed_redis_miss_does_not_revive_a_local_pin_during_outage( + shared_session_router: tuple[litellm.Router, FakeServer, RedisCache], +) -> None: + router, server, _ = shared_session_router + router.cache.in_memory_cache.set_cache(_repin_key(), {"model_id": "original"}, ttl=3600) + assert set(await _selected_session_ids(router)) == {"original", "replacement", "other"} + server.connected = False + assert set(await _selected_session_ids(router)) == {"original", "replacement", "other"} + + +async def test_shared_session_pin_resumes_after_a_transient_redis_failure( + shared_session_router: tuple[litellm.Router, FakeServer, RedisCache], +) -> None: + router, server, redis = shared_session_router + await redis.async_set_cache(_repin_key(), {"model_id": "replacement"}, ttl=3600) + router.cache.in_memory_cache.set_cache(_repin_key(), {"model_id": "original"}, ttl=3600) + server.connected = False + assert await _selected_session_ids(router) == ("original",) + server.connected = True + assert await _selected_session_ids(router) == ("replacement",) + + +@pytest.mark.parametrize("shared_value", [None, {"model_id": "original"}]) +async def test_delayed_shared_pin_read_preserves_a_newer_local_write( + shared_session_router: tuple[litellm.Router, FakeServer, RedisCache], shared_value: Mapping[str, str] | None +) -> None: + router, server, redis = shared_session_router + if shared_value is not None: + await redis.async_set_cache(_repin_key(), shared_value, ttl=3600) + observed: Final = asyncio.Event() + resume: Final = asyncio.Event() + + class DelayedReader(Redis): + async def get(self, name: str) -> object: + value: Final = await super().get(name) + observed.set() + await resume.wait() + return value + + reader: Final = DelayedReader(connection_pool=redis.init_async_client().connection_pool) + litellm.in_memory_llm_clients_cache.set_cache(key=redis._get_async_client_cache_key(), value=reader) + pending: Final = asyncio.create_task(_selected_session_ids(router)) + try: + await observed.wait() + router.cache.in_memory_cache.set_cache(_repin_key(), {"model_id": "replacement"}, ttl=3600) + finally: + resume.set() + await pending + server.connected = False + assert await _selected_session_ids(router) == ("replacement",) + + +@pytest.mark.parametrize( + "response", + [ + {"status": "failed", "response": {"status": "completed"}}, + {"error": {"message": "test failure"}, "response": {"status": "completed"}}, + {"response": {"response": {"status": "completed"}}}, + ], +) +async def test_unsuccessful_or_nested_response_envelopes_never_repin(response: Mapping[str, object]) -> None: + router: Final = _repinning_router() + try: + await router.cache.async_set_cache(_repin_key(), {"model_id": "original"}) + _cool_down_original(router) + await _record_replacement_success(router, response=response) + assert await router.cache.async_get_cache(_repin_key()) == {"model_id": "original"} + finally: + router.discard() + + +class _ReadCountingCache(InMemoryCache): + def __init__(self) -> None: + super().__init__() + self.read_keys: tuple[str, ...] = () + + def get_cache(self, key: str, **kwargs: object) -> object: + self.read_keys += (key,) + return super().get_cache(key, **kwargs) + + +@pytest.mark.parametrize("case", ["disabled", "no-session", "generated-session", "group-disabled", "active"]) +async def test_repin_cooldown_reads_are_limited_to_active_session_route(case: str) -> None: + router: Final = _repinning_router(case != "disabled") + reads: Final = _ReadCountingCache() + try: + router.cooldown_cache.cooldown_store.in_memory_cache = reads + callback: Final = next(cb for cb in router.optional_callbacks or () if isinstance(cb, DeploymentAffinityCheck)) + if case == "group-disabled": + callback.model_group_affinity_config = {"repin-group": []} + await router._repin_session_after_success( + { + "model_info": {"id": "replacement"}, + "metadata": { + "deployment_model_name": "repin-group", + **({"session_id": "repin-session"} if case != "no-session" else {}), + SESSION_ID_GENERATED_METADATA_KEY: case == "generated-session", + }, + }, + {"status": "completed"}, + ) + expected: Final = ( + { + router.cooldown_cache.get_cooldown_cache_key(model_id) + for model_id in ("original", "replacement", "other") + } + if case == "active" + else set() + ) + assert set(reads.read_keys) == expected + finally: + router.discard() + + class MockResponse: def __init__(self, json_data, status_code): self._json_data = json_data From 49f4a59cc86ce83ab5d3cd7b4a878412f186e015 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 13:43:07 -0700 Subject: [PATCH 018/155] fix(router): keep latency routing alive when latency samples are unusable (#48216) * fix(router): keep latency routing alive when latency samples are unusable latency-based-routing ranks deployments by the average of the cached samples and keeps everything within lowest_latency_buffer times the lowest. A negative or NaN sample breaks that invariant, because the minimum no longer satisfies its own bound. The candidate list then comes back empty and random.choice raises IndexError, which turns every request to the group into a 500 until the cache entry expires. Samples come from completion_start_time - start_time, and that difference is negative whenever the two timestamps invert. Production recorded samples around -600s from successful calls. Drop non-finite and negative samples before ranking, and fall back to every candidate if the window is empty anyway, for example with a negative buffer. Co-Authored-By: Claude Code Co-Authored-By: AaronHowell <111272993+AaronHowell@users.noreply.github.com> * test(router): seed latency_normalized in the unusable TTFT fallback test Main reads latency_normalized since #47727, so seeding latency left both deployments ranked at zero and the test picked at random. Co-authored-by: Rick <26716961+Bytechoreographer@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): assert ranked latency after dropping unusable TTFT samples Greptile on #48216: the new tests passed with the sample filter removed because the empty-window fallback alone saved them. They now assert _latency_per_deployment so unfiltered negative or NaN samples fail deterministically. Also adds Final to ttft_samples and drops a comment that restated the code Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): drop Final on a loop-local in latency ranking basedpyright rejects Final on a variable assigned inside a loop, and the type_check_gate counts it as a new reportGeneralTypeIssues error Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): type the request kwargs in the unusable TTFT tests Greptile on #48216: bare dict annotations. The tests now hold the metadata dict in a typed local and assert on it directly Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Rick <26716961+Bytechoreographer@users.noreply.github.com> Co-authored-by: Claude Code Co-authored-by: AaronHowell <111272993+AaronHowell@users.noreply.github.com> Co-authored-by: joshua Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router_strategy/lowest_latency.py | 24 +++++-- .../router_strategy/test_lowest_latency.py | 65 +++++++++++++++++++ 2 files changed, 82 insertions(+), 7 deletions(-) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 28ed062f1ec..4e994d3fdc6 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -3,7 +3,7 @@ import random from collections.abc import Sequence from datetime import datetime, timedelta -from math import ceil +from math import ceil, isfinite from typing import TYPE_CHECKING, Any, Final from pydantic import Field @@ -46,6 +46,16 @@ def _percentile_latency(samples: Sequence[float], percentile: float) -> float: return values[index] +def _usable_latency_samples(samples: Sequence[float]) -> tuple[float, ...]: + """Drop samples that cannot be a real elapsed time. + + ``completion_start_time - start_time`` goes negative when the timestamps invert. + A negative or NaN sample pushes ``lowest_latency + buffer`` below every candidate, + leaving the deployment filter empty. + """ + return tuple(sample for sample in samples if isfinite(sample) and sample >= 0) + + def _ttft_seconds(elapsed: timedelta | float) -> float: if isinstance(elapsed, timedelta): return elapsed.total_seconds() @@ -469,16 +479,17 @@ class LowestLatencyLoggingHandler(CustomLogger): item_rpm = item_map.get(precise_minute, {}).get("rpm", 0) item_tpm = item_map.get(precise_minute, {}).get("tpm", 0) + ttft_samples = _usable_latency_samples(item_ttft_latency) use_ttft = ( request_kwargs is not None and request_kwargs.get("stream", None) is not None and request_kwargs["stream"] is True - and len(item_ttft_latency) > 0 + and len(ttft_samples) > 0 ) selected_latency = ( - _percentile_latency(item_ttft_latency, self.routing_args.ttft_percentile) + _percentile_latency(ttft_samples, self.routing_args.ttft_percentile) if use_ttft and self.routing_args.ttft_percentile is not None - else _average_latency(item_ttft_latency if use_ttft else item_latency) + else _average_latency(ttft_samples if use_ttft else _usable_latency_samples(item_latency)) ) # -------------- # @@ -512,10 +523,9 @@ class LowestLatencyLoggingHandler(CustomLogger): # Find deployments within buffer of lowest latency buffer: Final = self.routing_args.lowest_latency_buffer * lowest_latency - valid_deployments: Final = [x for x in sorted_deployments if x[1] <= lowest_latency + buffer] + valid_deployments: Final = tuple(x for x in sorted_deployments if x[1] <= lowest_latency + buffer) - # Pick a random deployment from valid deployments - random_valid_deployment: Final = random.choice(valid_deployments) + random_valid_deployment: Final = random.choice(valid_deployments or sorted_deployments) deployment = random_valid_deployment[0] metadata_field: Final = self._select_metadata_field(request_kwargs) if request_kwargs is not None and metadata_field in request_kwargs: diff --git a/tests/unit/router_strategy/test_lowest_latency.py b/tests/unit/router_strategy/test_lowest_latency.py index 9af5def0d56..706ec567710 100644 --- a/tests/unit/router_strategy/test_lowest_latency.py +++ b/tests/unit/router_strategy/test_lowest_latency.py @@ -176,6 +176,8 @@ def test_sync_chat_zero_completion_tokens_falls_back_to_seconds(): MODEL_GROUP = "gpt-4o-mini" FAST_TTFT_ID = "fast-ttft-short-output" SLOW_TTFT_ID = "slow-ttft-long-output" +FAST_API_BASE = "https://fast.example.test/v1" +SLOW_API_BASE = "https://slow.example.test/v1" STREAMING_DEPLOYMENTS = [ {"model_info": {"id": FAST_TTFT_ID}, "litellm_params": {}}, {"model_info": {"id": SLOW_TTFT_ID}, "litellm_params": {}}, @@ -407,6 +409,69 @@ async def test_async_get_available_deployments_treats_missing_samples_as_zero_la assert picked["model_info"]["id"] == DEPLOYMENT_ID +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("ttft_samples", "expected_latency"), + [([-602.0], 0.0), ([10.0, 10.0, -40.0], 10.0), ([float("nan"), 2.0], 2.0)], + ids=["negative", "negative_average", "nan"], +) +async def test_unusable_ttft_samples_do_not_leave_the_buffer_window_empty( + ttft_samples: list[float], expected_latency: float +): + """``completion_start_time - start_time`` can be negative or NaN. With a non-zero + ``lowest_latency_buffer`` a negative average pushes ``lowest_latency + buffer`` + below every candidate, so the group must still route instead of raising + IndexError from ``random.choice``, and the bad samples must not reach the ranking.""" + cache: Final = DualCache() + handler: Final = LowestLatencyLoggingHandler(router_cache=cache, routing_args={"lowest_latency_buffer": 5}) + cache.set_cache( + key=f"{MODEL_GROUP}_map", + value={FAST_TTFT_ID: {"time_to_first_token_seconds": ttft_samples}}, + ) + metadata: Final[dict[str, object]] = {} + request_kwargs: Final[dict[str, object]] = {"stream": True, "metadata": metadata} + + picked = await handler.async_get_available_deployments( + model_group=MODEL_GROUP, + healthy_deployments=[{"model_info": {"id": FAST_TTFT_ID}, "litellm_params": {"api_base": FAST_API_BASE}}], + request_kwargs=request_kwargs, + ) + + assert picked is not None + assert picked["model_info"]["id"] == FAST_TTFT_ID + assert metadata["_latency_per_deployment"] == {FAST_API_BASE: expected_latency} + + +@pytest.mark.asyncio +async def test_unusable_ttft_samples_fall_back_to_recorded_latency(): + """A deployment whose TTFT samples are all unusable is ranked by its recorded + latency, so dirty samples cannot make it look like the fastest deployment.""" + cache: Final = DualCache() + handler: Final = LowestLatencyLoggingHandler(router_cache=cache, routing_args={"lowest_latency_buffer": 5}) + cache.set_cache( + key=f"{MODEL_GROUP}_map", + value={ + FAST_TTFT_ID: {"latency_normalized": [0.1], "time_to_first_token_seconds": [-600.0]}, + SLOW_TTFT_ID: {"latency_normalized": [5.0], "time_to_first_token_seconds": [-300.0]}, + }, + ) + metadata: Final[dict[str, object]] = {} + request_kwargs: Final[dict[str, object]] = {"stream": True, "metadata": metadata} + + picked = await handler.async_get_available_deployments( + model_group=MODEL_GROUP, + healthy_deployments=[ + {"model_info": {"id": FAST_TTFT_ID}, "litellm_params": {"api_base": FAST_API_BASE}}, + {"model_info": {"id": SLOW_TTFT_ID}, "litellm_params": {"api_base": SLOW_API_BASE}}, + ], + request_kwargs=request_kwargs, + ) + + assert picked is not None + assert picked["model_info"]["id"] == FAST_TTFT_ID + assert metadata["_latency_per_deployment"] == {FAST_API_BASE: 0.1, SLOW_API_BASE: 5.0} + + def _latency_router(routing_strategy_args: dict) -> Router: return Router( model_list=[ From 9b11729712b427372b1caafa7df9af0fc13e6ab8 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 13:44:19 -0700 Subject: [PATCH 019/155] fix(vector_stores): put azure vector store id before the api-version query on retrieve, update and delete (#48224) * fix(vector_stores): put azure vector store id before api-version query Azure OpenAI's get_complete_url returns a url ending in ?api-version=..., and the vector store handlers appended the id with an f-string, producing .../openai/vector_stores?api-version=X/vs_abc and a 404 from Azure. Add BaseVectorStoreConfig.get_vector_store_url, defaulting to the previous behaviour so other providers are unchanged, and override it in AzureOpenAIVectorStoreConfig to keep the query string last. The six retrieve/update/delete handlers now call it. Co-Authored-By: Claude Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(vector_stores): trim comments on get_vector_store_url Drops the inline comment in the Azure override and shortens the base docstring, per AGENTS.md Co-authored-by: Keith-Hardaway <276775292+hardawaykwork@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(vector_stores): capture the Azure retrieve request without a mutable url list Greptile on #48224: the handler test now follows the file's received list pattern with Final bindings Co-authored-by: Keith-Hardaway <276775292+hardawaykwork@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Keith-Hardaway Co-authored-by: Claude Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: joshua Co-authored-by: Keith-Hardaway <276775292+hardawaykwork@users.noreply.github.com> --- .../azure/vector_stores/transformation.py | 7 +++++ .../base_llm/vector_store/transformation.py | 5 ++++ litellm/llms/custom_httpx/llm_http_handler.py | 12 ++++---- ...test_azure_vector_stores_transformation.py | 29 ++++++++++++++++++ .../custom_httpx/test_llm_http_handler.py | 30 +++++++++++++++++++ 5 files changed, 77 insertions(+), 6 deletions(-) diff --git a/litellm/llms/azure/vector_stores/transformation.py b/litellm/llms/azure/vector_stores/transformation.py index 9510c021c7d..5a2961cd550 100644 --- a/litellm/llms/azure/vector_stores/transformation.py +++ b/litellm/llms/azure/vector_stores/transformation.py @@ -1,3 +1,5 @@ +from typing import Final + from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.llms.openai.vector_stores.transformation import OpenAIVectorStoreConfig from litellm.types.router import GenericLiteLLMParams @@ -15,5 +17,10 @@ class AzureOpenAIVectorStoreConfig(OpenAIVectorStoreConfig): route="/openai/vector_stores", ) + def get_vector_store_url(self, api_base: str, encoded_vector_store_id: str) -> str: + path, _, query = api_base.partition("?") + url: Final = f"{path}/{encoded_vector_store_id}" + return f"{url}?{query}" if query else url + def validate_environment(self, headers: dict, litellm_params: GenericLiteLLMParams | None) -> dict: return BaseAzureLLM.base_validate_azure_environment(headers=headers, litellm_params=litellm_params) diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index 138700b3212..6885183aeba 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -221,6 +221,11 @@ class BaseVectorStoreConfig: raise ValueError("api_base is required") return api_base + def get_vector_store_url(self, api_base: str, encoded_vector_store_id: str) -> str: + """Url of one vector store for retrieve, update and delete. Providers whose + ``get_complete_url`` adds a query string, like Azure, override this.""" + return f"{api_base}/{encoded_vector_store_id}" + def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: from ..chat.transformation import BaseLLMException diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 616f1ef559b..1b491f1614d 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -10891,7 +10891,7 @@ class BaseLLMHTTPHandler: ) encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id") - url: Final = f"{api_base}/{encoded_vector_store_id}" + url: Final = vector_store_provider_config.get_vector_store_url(api_base, encoded_vector_store_id) logging_obj.pre_call( input="", @@ -10956,7 +10956,7 @@ class BaseLLMHTTPHandler: ) encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id") - url: Final = f"{api_base}/{encoded_vector_store_id}" + url: Final = vector_store_provider_config.get_vector_store_url(api_base, encoded_vector_store_id) logging_obj.pre_call( input="", @@ -11159,7 +11159,7 @@ class BaseLLMHTTPHandler: ) encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id") - url: Final = f"{api_base}/{encoded_vector_store_id}" + url: Final = vector_store_provider_config.get_vector_store_url(api_base, encoded_vector_store_id) request_body: Final[dict[str, object]] = dict(vector_store_update_optional_params) metadata: Final = vector_store_update_optional_params.get("metadata") @@ -11238,7 +11238,7 @@ class BaseLLMHTTPHandler: ) encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id") - url: Final = f"{api_base}/{encoded_vector_store_id}" + url: Final = vector_store_provider_config.get_vector_store_url(api_base, encoded_vector_store_id) request_body: Final[dict[str, object]] = dict(vector_store_update_optional_params) metadata: Final = vector_store_update_optional_params.get("metadata") @@ -11306,7 +11306,7 @@ class BaseLLMHTTPHandler: ) encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id") - url: Final = f"{api_base}/{encoded_vector_store_id}" + url: Final = vector_store_provider_config.get_vector_store_url(api_base, encoded_vector_store_id) logging_obj.pre_call( input="", @@ -11368,7 +11368,7 @@ class BaseLLMHTTPHandler: ) encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id") - url: Final = f"{api_base}/{encoded_vector_store_id}" + url: Final = vector_store_provider_config.get_vector_store_url(api_base, encoded_vector_store_id) logging_obj.pre_call( input="", diff --git a/tests/unit/llms/azure/vector_stores/test_azure_vector_stores_transformation.py b/tests/unit/llms/azure/vector_stores/test_azure_vector_stores_transformation.py index 59bec08fca6..3713b1cc895 100644 --- a/tests/unit/llms/azure/vector_stores/test_azure_vector_stores_transformation.py +++ b/tests/unit/llms/azure/vector_stores/test_azure_vector_stores_transformation.py @@ -1,4 +1,7 @@ +from unittest.mock import Mock + from litellm.llms.azure.vector_stores.transformation import AzureOpenAIVectorStoreConfig +from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig def test_transform_search_vector_store_request_preserves_azure_query_string(): @@ -18,3 +21,29 @@ def test_transform_search_vector_store_request_preserves_azure_query_string(): ) assert url == "https://x.openai.azure.com/openai/vector_stores/vs_1/search?api-version=2024-10-21" + + +def test_get_vector_store_url_puts_id_before_api_version_query(): + config = AzureOpenAIVectorStoreConfig() + api_base = config.get_complete_url( + api_base="https://x.openai.azure.com", + litellm_params={"api_version": "2025-04-01-preview"}, + ) + + url = config.get_vector_store_url(api_base, "vs_abc") + + assert url == "https://x.openai.azure.com/openai/vector_stores/vs_abc?api-version=2025-04-01-preview" + + +def test_get_vector_store_url_without_query_is_plain_path(): + url = AzureOpenAIVectorStoreConfig().get_vector_store_url( + "https://x.openai.azure.com/openai/vector_stores", "vs_abc" + ) + + assert url == "https://x.openai.azure.com/openai/vector_stores/vs_abc" + + +def test_base_get_vector_store_url_appends_id_for_providers_without_query_string(): + url = BaseVectorStoreConfig.get_vector_store_url(Mock(), "https://api.example.test/v1/vector_stores", "vs_abc") + + assert url == "https://api.example.test/v1/vector_stores/vs_abc" diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 8b32ab2e6b8..bbcd7f70e63 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -24,6 +24,7 @@ from litellm.integrations.code_interpreter_interception.handler import ( from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig +from litellm.llms.azure.vector_stores.transformation import AzureOpenAIVectorStoreConfig from litellm.llms.base_llm.audio_transcription.transformation import ( AudioTranscriptionRequestData, BaseAudioTranscriptionConfig, @@ -5624,3 +5625,32 @@ async def test_messages_opencode_session_header( "stream": stream, } assert received.empty() + + +def test_vector_store_retrieve_handler_sends_id_before_azure_api_version_query() -> None: + received: Final[list[httpx.Request]] = [] + + def answer(request: httpx.Request) -> httpx.Response: + received.append(request) + return httpx.Response(200, json={"id": "vs_abc", "object": "vector_store"}) + + handler: Final = BaseLLMHTTPHandler() + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(answer))) + + handler.vector_store_retrieve_handler( + vector_store_id="vs_abc", + vector_store_provider_config=AzureOpenAIVectorStoreConfig(), + custom_llm_provider="azure", + litellm_params=GenericLiteLLMParams( + api_base="https://x.openai.azure.com", + api_key="azure-test-key", + api_version="2025-04-01-preview", + ), + logging_obj=Mock(), + client=client, + _is_async=False, + ) + + sent: Final = received[0] + assert str(sent.url) == "https://x.openai.azure.com/openai/vector_stores/vs_abc?api-version=2025-04-01-preview" + assert len(received) == 1 From bcd541918a1c52fbb4dd6e8eb901b3b864a9465c Mon Sep 17 00:00:00 2001 From: tbartik Date: Sat, 24 Oct 2026 22:48:29 +0200 Subject: [PATCH 020/155] fix(anthropic): stop injecting empty-content placeholder into tool-call-only assistant turns (#34822) * fix(anthropic): stop injecting empty-content placeholder into tool-call-only assistant turns _sanitize_empty_text_content replaced empty or whitespace-only text with "[System: Empty message content sanitised to satisfy protocol]" for every user and assistant turn. anthropic_messages_pt calls it unconditionally, independent of modify_params, because an empty text block always makes Anthropic 400. An assistant turn carrying tool_calls and no text is the normal shape an OpenAI-format agent emits, and Anthropic accepts an assistant turn whose content is just tool_use, so there was nothing to substitute there. Substituting anyway put a protocol diagnostic in the content field, which is the assistant's utterance as far as the model and the client are concerned, so it got persisted into conversation history and echoed back on later turns. Empty text is now dropped rather than replaced when the assistant turn already carries substantive content (tool_calls, or non-text blocks such as tool_use). Turns that would otherwise be left with no content at all keep the placeholder, since Anthropic rejects those too. Fixes #24498 * fix(anthropic): only drop empty text when a sibling block survives conversion _carries_non_text_content treated any non-text block as substantive, but the assistant list-content converter only keeps thinking (when signable), server_tool_use and *_tool_result blocks. For an assistant turn holding empty text next to a block it discards, such as an image_url or a signature-less thinking block, dropping the empty text left nothing behind and the whole turn disappeared from the request. The predicate now mirrors what the conversion actually keeps, so those turns retain the placeholder and survive. This also improves on the previous behaviour, where such a turn was dropped outright. * test(anthropic): cover malformed content blocks in the survivor predicate _survives_assistant_conversion had a separate non-dict guard that no test reached, since _is_empty_text_block already rejects non-dict blocks before the predicate runs on them in practice. Folding the check into the type lookup keeps the same behaviour for bare strings and blocks without a type key, with no unreachable branch. Extends the sibling-block test with a block missing its type key. * test(anthropic): set modify_params through monkeypatch in sanitization tests The test-quality gate flags direct writes to litellm module globals. Swap the class setup/teardown for an autouse monkeypatch fixture, drop writes that just repeat its default, and route the False cases through monkeypatch --------- Co-authored-by: tbartik <22518863+tbartik@users.noreply.github.com> Co-authored-by: ryan-crabbe-berri --- .../prompt_templates/factory.py | 136 ++++++-- .../anthropic/test_message_sanitization.py | 298 ++++++++++++++++-- 2 files changed, 380 insertions(+), 54 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index cfb6e3ab45f..0aed2956b0a 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -2205,15 +2205,97 @@ def anthropic_process_openai_file_message( _EMPTY_TEXT_PLACEHOLDER: Final = "[System: Empty message content sanitised to satisfy protocol]" +def _dict_block(block: object) -> Mapping[str, object] | None: + if not isinstance(block, dict): + return None + return cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance; the converters only read dict blocks + + +def _is_empty_text_block(block: object) -> bool: + match _dict_block(block): + case {"type": "text", "text": str() as text}: + return not text.strip() + case {"type": "text"}: + return True + case _: + return False + + +def _with_placeholder_text(block: object) -> object: + fields: Final = _dict_block(block) + if fields is None or not _is_empty_text_block(fields): + return block + return {**fields, "text": _EMPTY_TEXT_PLACEHOLDER} + + +def _survives_assistant_conversion(block: object) -> bool: + """Whether ``anthropic_messages_pt`` keeps this list-content block of an assistant turn. + + Mirrors the assistant converter: a ``thinking`` / ``redacted_thinking`` block is kept + unless it is unsignable (a signed block is kept even with empty text), and + ``server_tool_use`` and ``*_tool_result`` blocks pass through. A redacted block with + blank data is dropped under ``drop_params``, which is not known here, so it never counts. + """ + fields: Final = _dict_block(block) + if fields is None: + return False + if (thinking_block := _as_thinking_block(fields)) is not None: + return not (is_unsignable_thinking_block(thinking_block) or has_blank_redacted_data(thinking_block)) + match fields: + case {"type": "server_tool_use"}: + return True + case {"type": str() as block_type}: + return block_type.endswith("_tool_result") + case _: + return False + + +def _tool_call_survives_conversion(tool_call: object) -> bool: + """Whether ``convert_to_anthropic_tool_invoke`` can emit a block for this tool call. + + Non-function calls are skipped there, and a tool search call is dropped when its + result is not replayed, which depends on the rest of the conversation, so neither counts. + """ + if message_field(tool_call, "type") != "function": + return False + match message_field(tool_call, "id"), message_field(message_field(tool_call, "function"), "name"): + case (str() as tool_id, str() as tool_name): + return not is_anthropic_tool_search_use(tool_id, tool_name) + case _: + return True + + +def _carries_non_text_content(message: AllMessageValues) -> bool: + if any(_tool_call_survives_conversion(tool_call) for tool_call in parts_of(message_field(message, "tool_calls"))): + return True + content: Final = message_field(message, "content") + blocks: Final = parts_of(content) + return isinstance(content, list) and any(_survives_assistant_conversion(block) for block in blocks) + + +def _with_content(message: AllMessageValues, content: object, action: str) -> AllMessageValues: + sanitized: Final = cast(AllMessageValues, {**message, "content": content}) + verbose_logger.debug("_sanitize_empty_text_content: %s in %s message", action, message_field(message, "role")) + return sanitized + + def _sanitize_empty_text_content( message: AllMessageValues, ) -> AllMessageValues: """ Case C: Sanitize empty text content - - Replace empty or whitespace-only text content with a placeholder message. - - Handles both string content and list-of-blocks content (rewriting only - the empty text blocks in place; non-text blocks like images are left - untouched). + + Anthropic rejects empty text blocks, so empty or whitespace-only text must never be + forwarded verbatim. What replaces it depends on what else the turn carries. + + Assistant turns that already carry substantive content (``tool_calls``, or non-text + blocks such as ``server_tool_use``) get the empty text dropped. Anthropic accepts an + assistant turn whose content is just ``tool_use``, so there is nothing to substitute, and + substituting anyway leaks a protocol diagnostic into the conversation as though the + assistant had said it, which the model then echoes on later turns. + + Every other empty turn would otherwise end up with no content at all, which Anthropic + also rejects, so the placeholder is still substituted there. Returns: The message with sanitized content if needed, otherwise the original message @@ -2221,42 +2303,26 @@ def _sanitize_empty_text_content( if message.get("role") not in ["user", "assistant"]: return message - content: Final = message.get("content") + content: Final = message_field(message, "content") + drop_empty_text: Final = message_field(message, "role") == "assistant" and _carries_non_text_content(message) if isinstance(content, str): - if not content or not content.strip(): - message = cast(AllMessageValues, dict(message)) # Make a copy - message["content"] = _EMPTY_TEXT_PLACEHOLDER - verbose_logger.debug( - "_sanitize_empty_text_content: Replaced empty text content in %s message", message.get("role") - ) + if content.strip(): + return message + if drop_empty_text: + return _with_content(message, None, "Dropped empty text content") + return _with_content(message, _EMPTY_TEXT_PLACEHOLDER, "Replaced empty text content") + + blocks: Final = parts_of(content) + if not isinstance(content, list) or not any(_is_empty_text_block(block) for block in blocks): return message - if isinstance(content, list): - # Walk the blocks and rewrite any empty text blocks. We rewrite (rather - # than drop) so callers don't end up with an entirely empty content - # list, which Anthropic also rejects. - new_blocks: Final[list[Any]] = [] - rewrote_any = False - for block in content: - if isinstance(block, dict) and block.get("type") == "text": - text = block.get("text") - if not isinstance(text, str) or not text or not text.strip(): - new_block = dict(block) - new_block["text"] = _EMPTY_TEXT_PLACEHOLDER - new_blocks.append(new_block) - rewrote_any = True - continue - new_blocks.append(block) + if drop_empty_text: + kept: Final = [block for block in blocks if not _is_empty_text_block(block)] + return _with_content(message, kept, "Dropped empty text block(s)") - if rewrote_any: - message = cast(AllMessageValues, dict(message)) # Make a copy - message["content"] = new_blocks - verbose_logger.debug( - "_sanitize_empty_text_content: Replaced empty text block(s) in %s message", message.get("role") - ) - - return message + rewritten: Final = [_with_placeholder_text(block) for block in blocks] + return _with_content(message, rewritten, "Replaced empty text block(s)") def _add_missing_tool_results( diff --git a/tests/unit/llms/anthropic/test_message_sanitization.py b/tests/unit/llms/anthropic/test_message_sanitization.py index 7afa60baf7e..0afd443d07a 100644 --- a/tests/unit/llms/anthropic/test_message_sanitization.py +++ b/tests/unit/llms/anthropic/test_message_sanitization.py @@ -7,6 +7,8 @@ B. Orphaned tool_result without matching tool_use C. Empty text content """ +import copy +import json import pytest import sys import os @@ -16,6 +18,8 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../. import litellm from litellm.litellm_core_utils.prompt_templates.factory import ( + _EMPTY_TEXT_PLACEHOLDER as PLACEHOLDER, + _sanitize_empty_text_content, sanitize_messages_for_tool_calling, anthropic_messages_pt, ) @@ -24,16 +28,9 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( class TestMessageSanitization: """Test message sanitization for tool calling scenarios""" - def setup_method(self): - """Setup for each test""" - # Save original modify_params value - self.original_modify_params = litellm.modify_params - litellm.modify_params = True - - def teardown_method(self): - """Cleanup after each test""" - # Restore original modify_params value - litellm.modify_params = self.original_modify_params + @pytest.fixture(autouse=True) + def _modify_params_on(self, monkeypatch): + monkeypatch.setattr(litellm, "modify_params", True) def test_case_a_orphaned_tool_call_single(self): """ @@ -255,11 +252,11 @@ class TestMessageSanitization: assert sanitized[3]["content"] == "[System: Empty message content sanitised to satisfy protocol]" assert sanitized[4]["role"] == "assistant" - def test_modify_params_false_no_sanitization(self): + def test_modify_params_false_no_sanitization(self, monkeypatch): """ Test that sanitization is skipped when modify_params=False """ - litellm.modify_params = False + monkeypatch.setattr(litellm, "modify_params", False) messages = [ {"role": "user", "content": ""}, @@ -287,7 +284,6 @@ class TestMessageSanitization: """ Test that sanitization is integrated into anthropic_messages_pt """ - litellm.modify_params = True messages = [ {"role": "user", "content": "What is the weather in Nashik?"}, @@ -316,7 +312,7 @@ class TestMessageSanitization: assert result[0]["role"] == "user" assert result[1]["role"] == "assistant" - def test_empty_string_content_sanitized_without_modify_params(self): + def test_empty_string_content_sanitized_without_modify_params(self, monkeypatch): """ Regression: An empty user message ({"role": "user", "content": ""}) must be rewritten to a non-empty placeholder *before* it reaches Anthropic, @@ -324,7 +320,7 @@ class TestMessageSanitization: "messages: text content blocks must be non-empty" Reproduces a real failure from the pr-review agent (pydantic-ai). """ - litellm.modify_params = False + monkeypatch.setattr(litellm, "modify_params", False) messages = [ {"role": "user", "content": "First message"}, @@ -344,13 +340,13 @@ class TestMessageSanitization: assert block["text"].strip() != "" assert text_blocks[2]["text"] == ("[System: Empty message content sanitised to satisfy protocol]") - def test_empty_text_block_in_list_content_sanitized(self): + def test_empty_text_block_in_list_content_sanitized(self, monkeypatch): """ Same regression for the list-of-blocks form: {"role": "user", "content": [{"type": "text", "text": ""}]} Empty text *blocks* must be rewritten too, regardless of modify_params. """ - litellm.modify_params = False + monkeypatch.setattr(litellm, "modify_params", False) messages = [ { @@ -372,12 +368,12 @@ class TestMessageSanitization: for block in text_blocks[1:]: assert block["text"].strip() != "" - def test_non_empty_content_unchanged_without_modify_params(self): + def test_non_empty_content_unchanged_without_modify_params(self, monkeypatch): """ Sanity check: when nothing is empty, the messages flow through unchanged even with modify_params disabled. """ - litellm.modify_params = False + monkeypatch.setattr(litellm, "modify_params", False) messages = [ {"role": "user", "content": "Hello"}, @@ -393,6 +389,270 @@ class TestMessageSanitization: assert result[1]["content"][0]["text"] == "Hi there" assert result[2]["content"][0]["text"] == "How are you?" + @pytest.mark.parametrize("modify_params", [True, False]) + @pytest.mark.parametrize( + "empty_content", + [None, "", " \n \t ", [], [{"type": "text", "text": ""}]], + ids=["none", "empty_str", "whitespace_str", "empty_list", "empty_text_block"], + ) + def test_tool_call_only_assistant_turn_never_gets_placeholder(self, monkeypatch, modify_params, empty_content): + """ + Regression for the placeholder leaking into tool-call-only assistant turns. + + An OpenAI-format assistant turn that carries tool_calls but no text is the normal + shape emitted by coding agents. Anthropic accepts an assistant turn whose content + is just tool_use, so there is nothing to substitute; injecting the placeholder made + it look like the assistant had said it, and the model echoed it on later turns. + """ + monkeypatch.setattr(litellm, "modify_params", modify_params) + + messages = [ + {"role": "user", "content": "list the files"}, + { + "role": "assistant", + "content": empty_content, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "run_bash", "arguments": '{"cmd": "ls"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "a.txt"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") + + assistant_turns = [m for m in result if m["role"] == "assistant"] + assert len(assistant_turns) == 1 + blocks = assistant_turns[0]["content"] + + assert [b["type"] for b in blocks] == ["tool_use"] + assert blocks[0]["id"] == "call_1" + assert blocks[0]["name"] == "run_bash" + + assert PLACEHOLDER not in json.dumps(result) + + def test_empty_text_dropped_alongside_non_text_blocks(self): + """ + An assistant turn already carrying a non-text block has enough content for + Anthropic once the empty text block is dropped, so the placeholder must not be + substituted in its place. + """ + + message = { + "role": "assistant", + "content": [ + {"type": "text", "text": " "}, + {"type": "thinking", "thinking": "weighing options", "signature": "sig"}, + ], + } + + sanitized = _sanitize_empty_text_content(message) + + assert [b["type"] for b in sanitized["content"]] == ["thinking"] + assert PLACEHOLDER not in json.dumps(sanitized) + + @pytest.mark.parametrize( + "sibling_block", + [ + {"type": "thinking", "thinking": "", "signature": "sig"}, + {"type": "redacted_thinking", "data": "opaque"}, + ], + ids=["signed_empty_thinking", "redacted_thinking"], + ) + def test_empty_text_dropped_alongside_kept_thinking_block(self, sibling_block): + """ + The converter keeps signed thinking blocks even with empty text, and keeps + redacted_thinking blocks, so the turn still reaches Anthropic without the + empty text and the placeholder must not be substituted. + """ + + messages = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [{"type": "text", "text": " "}, sibling_block]}, + {"role": "user", "content": "and now?"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") + + assert [m["role"] for m in result] == ["user", "assistant", "user"] + assert [b["type"] for b in result[1]["content"]] == [sibling_block["type"]] + assert PLACEHOLDER not in json.dumps(result) + + def test_placeholder_kept_when_only_tool_call_is_dropped_tool_search(self, monkeypatch): + """ + A tool search call whose result is not replayed is dropped by the converter, so + it does not count as content: the empty text keeps its placeholder instead of + the assistant turn vanishing from the request. modify_params stays off so no + dummy tool result answers the call and keeps it. + """ + monkeypatch.setattr(litellm, "modify_params", False) + + messages = [ + {"role": "user", "content": "find a tool"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "srvtoolu_1", + "type": "function", + "function": {"name": "tool_search_tool_regex", "arguments": '{"query": "x"}'}, + } + ], + }, + {"role": "user", "content": "go on"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") + + assert [m["role"] for m in result] == ["user", "assistant", "user"] + assert result[1]["content"] == [{"type": "text", "text": PLACEHOLDER}] + + @pytest.mark.parametrize( + "sibling_block", + [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGk="}}, + {"type": "thinking", "thinking": ""}, + {"type": "redacted_thinking", "data": " "}, + {"type": "unknown_future_block"}, + {"no_type_key": "malformed"}, + ], + ids=["image_url", "unsignable_thinking", "blank_redacted_thinking", "unknown_type", "missing_type"], + ) + def test_empty_text_kept_when_sibling_block_would_not_survive(self, sibling_block): + """ + Dropping empty text is only safe when something else in the turn actually reaches + Anthropic. Blocks the assistant conversion discards (assistant-side images, + signature-less thinking, unrecognised types) would leave an empty turn that gets + dropped from the request entirely, so the placeholder has to stay. + """ + + messages = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [{"type": "text", "text": " "}, sibling_block]}, + {"role": "user", "content": "and now?"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-haiku-4-5", llm_provider="anthropic") + + assert [m["role"] for m in result] == ["user", "assistant", "user"] + assert PLACEHOLDER in json.dumps(result) + + def test_assistant_text_preserved_alongside_tool_calls(self): + """ + Dropping empty text must not drop real text: an assistant turn that says something + *and* calls a tool keeps both blocks. + """ + + messages = [ + {"role": "user", "content": "list the files"}, + { + "role": "assistant", + "content": "Let me check.", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "run_bash", "arguments": '{"cmd": "ls"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "a.txt"}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") + + assistant_turns = [m for m in result if m["role"] == "assistant"] + blocks = assistant_turns[0]["content"] + + assert [b["type"] for b in blocks] == ["text", "tool_use"] + assert blocks[0]["text"] == "Let me check." + assert PLACEHOLDER not in json.dumps(result) + + @pytest.mark.parametrize( + "messages", + [ + [{"role": "user", "content": ""}], + [{"role": "user", "content": [{"type": "text", "text": " "}]}], + [{"role": "user", "content": "hi"}, {"role": "assistant", "content": ""}], + ], + ids=["user_str", "user_block", "assistant_without_tool_calls"], + ) + def test_placeholder_still_used_when_turn_would_be_empty(self, messages): + """ + The placeholder is still required where dropping the empty text would leave the + turn with no content at all, which Anthropic also rejects. + """ + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") + + assert PLACEHOLDER in json.dumps(result) + for message in result: + assert message["content"] + + def test_no_empty_text_block_or_content_reaches_anthropic(self): + """ + The invariant Anthropic actually enforces, over a conversation mixing every shape: + no empty content list and no empty text block may survive the conversion. + """ + + messages = [ + {"role": "user", "content": ""}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "run_bash", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "out"}, + {"role": "assistant", "content": [{"type": "text", "text": " "}]}, + {"role": "user", "content": [{"type": "text", "text": "thanks"}]}, + ] + + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") + + for message in result: + content = message["content"] + assert content + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + assert block["text"].strip() + + def test_sanitization_does_not_mutate_caller_messages(self): + """ + Callers reuse their message list across retries and fallbacks, so sanitization has + to stay copy-on-write. + """ + + messages = [ + {"role": "user", "content": "go"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "run_bash", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "out"}, + ] + before = copy.deepcopy(messages) + + anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") + + assert messages == before + if __name__ == "__main__": pytest.main([__file__, "-v"]) From 560372456eff83f4d57e4247b41d04111514b3b8 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sat, 24 Oct 2026 13:54:18 -0700 Subject: [PATCH 021/155] fix(bedrock): price claude_platform calls from their own cost map rows (#48150) * fix(bedrock): price claude_platform calls from the anthropic cost map rows Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix: avoid recursive Claude Platform lookups Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): price claude_platform calls from their own cost map rows Replace the claude_platform -> anthropic lookup redirect with bedrock/claude_platform/* rows, and apply the inference_geo multiplier in the bedrock cost path so US-only inference is priced at 1.1x Co-Authored-By: Claude Opus 5.5 * test(bedrock): exempt claude_platform rows from the Converse routing guard The claude_platform/ route is explicit in the row key, so these rows are not accidental non-Converse Bedrock chat rows Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- litellm/llms/bedrock/cost_calculation.py | 14 +- ...odel_prices_and_context_window_backup.json | 600 ++++++++++++++++++ model_prices_and_context_window.json | 600 ++++++++++++++++++ .../llms/bedrock/test_cost_calculation.py | 43 ++ tests/unit/test_utils_get_model_info.py | 1 + 5 files changed, 1255 insertions(+), 3 deletions(-) create mode 100644 tests/unit/llms/bedrock/test_cost_calculation.py diff --git a/litellm/llms/bedrock/cost_calculation.py b/litellm/llms/bedrock/cost_calculation.py index 0a2e39ab972..1b45531dda4 100644 --- a/litellm/llms/bedrock/cost_calculation.py +++ b/litellm/llms/bedrock/cost_calculation.py @@ -3,9 +3,13 @@ Helper util for handling bedrock-specific cost calculation - e.g.: prompt caching """ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Final -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import ( + generic_cost_per_token, + get_provider_specific_geo_multiplier, +) if TYPE_CHECKING: from litellm.types.utils import Usage @@ -17,9 +21,13 @@ def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None) Follows the same logic as Anthropic's cost per token calculation. """ - return generic_cost_per_token( + model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="bedrock") + prompt_cost, completion_cost = generic_cost_per_token( model=model, usage=usage, custom_llm_provider="bedrock", service_tier=service_tier, + model_info=model_info, ) + geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=model_info, usage=usage) + return prompt_cost * geo_multiplier, completion_cost * geo_multiplier diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index eb9bf6aa8bf..c7dfce0712f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -77996,5 +77996,605 @@ "supports_reasoning": true, "supports_vision": false, "source": "https://opencode.ai/v2/docs/console/go/" + }, + "bedrock/claude_platform/claude-fable-5": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_creation_input_token_cost_batches": 6.25e-06, + "cache_read_input_token_cost": 1e-06, + "cache_read_input_token_cost_batches": 5e-07, + "input_cost_per_token": 1e-05, + "input_cost_per_token_batches": 5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "output_cost_per_token_batches": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 512, + "supports_native_structured_output": true, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-fable-5-1": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_creation_input_token_cost_batches": 6.25e-06, + "cache_read_input_token_cost": 2.5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, + "input_cost_per_token": 1e-05, + "input_cost_per_token_batches": 5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "output_cost_per_token_batches": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 512, + "supports_native_structured_output": true, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-haiku-4-5": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, + "cache_creation_input_token_cost_batches": 6.25e-07, + "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, + "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 5e-06, + "output_cost_per_token_batches": 2.5e-06, + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_computer_use": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "prompt_cache_min_tokens": 4096, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-haiku-5-5": { + "input_cost_per_token_above_100k_tokens_batches": 2.5e-07, + "output_cost_per_token_above_100k_tokens_batches": 1.25e-06, + "cache_creation_input_token_cost_above_100k_tokens_batches": 3.125e-07, + "cache_read_input_token_cost_above_100k_tokens_batches": 2.5e-08, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 2e-07, + "cache_creation_input_token_cost_batches": 6.25e-08, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_batches": 5e-09, + "input_cost_per_token": 1e-07, + "input_cost_per_token_batches": 5e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_batches": 2.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "supports_forced_tool_use": true, + "thinking_always_on": false, + "prompt_cache_min_tokens": 512, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing", + "supports_web_search": true, + "input_cost_per_token_above_100k_tokens": 5e-07, + "output_cost_per_token_above_100k_tokens": 2.5e-06, + "cache_creation_input_token_cost_above_100k_tokens": 6.25e-07, + "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06, + "cache_read_input_token_cost_above_100k_tokens": 5e-08 + }, + "bedrock/claude_platform/claude-opus-4-5": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, + "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_thinking_cache_preservation": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true, + "prompt_cache_min_tokens": 4096, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-opus-4-6": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, + "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_thinking_cache_preservation": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_tool_choice": true, + "supports_vision": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 4096, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-opus-4-7": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, + "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_thinking_cache_preservation": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 2048, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-opus-4-8": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, + "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_thinking_cache_preservation": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 1024, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-opus-5": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, + "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_thinking_cache_preservation": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 512, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-opus-5-5": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_creation_input_token_cost_batches": 2.5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, + "input_cost_per_token": 4e-06, + "input_cost_per_token_batches": 2e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "output_cost_per_token_batches": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 512, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-sonnet-4-5": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "cache_creation_input_token_cost_batches": 1.875e-06, + "cache_read_input_token_cost_batches": 1.5e-07, + "deprecation_date": "2026-11-30", + "input_cost_per_token_batches": 1.5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "output_cost_per_token_batches": 7.5e-06, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_tool_choice": true, + "supports_vision": true, + "prompt_cache_min_tokens": 1024, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-sonnet-4-6": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_creation_input_token_cost_batches": 1.875e-06, + "cache_read_input_token_cost": 3e-07, + "cache_read_input_token_cost_batches": 1.5e-07, + "input_cost_per_token": 3e-06, + "input_cost_per_token_batches": 1.5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "output_cost_per_token_batches": 7.5e-06, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_thinking_cache_preservation": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true, + "prompt_cache_min_tokens": 1024, + "provider_specific_entry": { + "us": 1.1 + }, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-sonnet-5": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_creation_input_token_cost_batches": 1.25e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_thinking_cache_preservation": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 1024, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-sonnet-5-5": { + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_creation_input_token_cost_batches": 1.25e-06, + "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing", + "supports_web_search": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index eb9bf6aa8bf..c7dfce0712f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -77996,5 +77996,605 @@ "supports_reasoning": true, "supports_vision": false, "source": "https://opencode.ai/v2/docs/console/go/" + }, + "bedrock/claude_platform/claude-fable-5": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_creation_input_token_cost_batches": 6.25e-06, + "cache_read_input_token_cost": 1e-06, + "cache_read_input_token_cost_batches": 5e-07, + "input_cost_per_token": 1e-05, + "input_cost_per_token_batches": 5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "output_cost_per_token_batches": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 512, + "supports_native_structured_output": true, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-fable-5-1": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_creation_input_token_cost_batches": 6.25e-06, + "cache_read_input_token_cost": 2.5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, + "input_cost_per_token": 1e-05, + "input_cost_per_token_batches": 5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "output_cost_per_token_batches": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 512, + "supports_native_structured_output": true, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-haiku-4-5": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, + "cache_creation_input_token_cost_batches": 6.25e-07, + "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, + "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 5e-06, + "output_cost_per_token_batches": 2.5e-06, + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_computer_use": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "prompt_cache_min_tokens": 4096, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-haiku-5-5": { + "input_cost_per_token_above_100k_tokens_batches": 2.5e-07, + "output_cost_per_token_above_100k_tokens_batches": 1.25e-06, + "cache_creation_input_token_cost_above_100k_tokens_batches": 3.125e-07, + "cache_read_input_token_cost_above_100k_tokens_batches": 2.5e-08, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 2e-07, + "cache_creation_input_token_cost_batches": 6.25e-08, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_batches": 5e-09, + "input_cost_per_token": 1e-07, + "input_cost_per_token_batches": 5e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_batches": 2.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "supports_forced_tool_use": true, + "thinking_always_on": false, + "prompt_cache_min_tokens": 512, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing", + "supports_web_search": true, + "input_cost_per_token_above_100k_tokens": 5e-07, + "output_cost_per_token_above_100k_tokens": 2.5e-06, + "cache_creation_input_token_cost_above_100k_tokens": 6.25e-07, + "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06, + "cache_read_input_token_cost_above_100k_tokens": 5e-08 + }, + "bedrock/claude_platform/claude-opus-4-5": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, + "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_thinking_cache_preservation": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true, + "prompt_cache_min_tokens": 4096, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-opus-4-6": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, + "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_thinking_cache_preservation": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_tool_choice": true, + "supports_vision": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 4096, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-opus-4-7": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, + "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_thinking_cache_preservation": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 2048, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-opus-4-8": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, + "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_thinking_cache_preservation": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 1024, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-opus-5": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, + "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_thinking_cache_preservation": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 512, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-opus-5-5": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_creation_input_token_cost_batches": 2.5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, + "input_cost_per_token": 4e-06, + "input_cost_per_token_batches": 2e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "output_cost_per_token_batches": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 512, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-sonnet-4-5": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "cache_creation_input_token_cost_batches": 1.875e-06, + "cache_read_input_token_cost_batches": 1.5e-07, + "deprecation_date": "2026-11-30", + "input_cost_per_token_batches": 1.5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "output_cost_per_token_batches": 7.5e-06, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_tool_choice": true, + "supports_vision": true, + "prompt_cache_min_tokens": 1024, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-sonnet-4-6": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_creation_input_token_cost_batches": 1.875e-06, + "cache_read_input_token_cost": 3e-07, + "cache_read_input_token_cost_batches": 1.5e-07, + "input_cost_per_token": 3e-06, + "input_cost_per_token_batches": 1.5e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "output_cost_per_token_batches": 7.5e-06, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_thinking_cache_preservation": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true, + "prompt_cache_min_tokens": 1024, + "provider_specific_entry": { + "us": 1.1 + }, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-sonnet-5": { + "supports_web_search": true, + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_creation_input_token_cost_batches": 1.25e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_thinking_cache_preservation": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 1024, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing" + }, + "bedrock/claude_platform/claude-sonnet-5-5": { + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_creation_input_token_cost_batches": 1.25e-06, + "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing", + "supports_web_search": true } } diff --git a/tests/unit/llms/bedrock/test_cost_calculation.py b/tests/unit/llms/bedrock/test_cost_calculation.py new file mode 100644 index 00000000000..f4718344e7e --- /dev/null +++ b/tests/unit/llms/bedrock/test_cost_calculation.py @@ -0,0 +1,43 @@ +import json +from importlib.resources import files +from typing import Final + +import pytest + +import litellm +from litellm.types.utils import Usage + +CLAUDE_PLATFORM_PREFIX: Final = "bedrock/claude_platform/" +CLAUDE_PLATFORM_KEYS: Final = tuple( + key + for key in json.loads(files("litellm").joinpath("model_prices_and_context_window_backup.json").read_text()) + if key.startswith(CLAUDE_PLATFORM_PREFIX) +) + + +def test_claude_platform_rows_exist() -> None: + assert CLAUDE_PLATFORM_KEYS + + +@pytest.mark.parametrize("inference_geo", ["global", "us"]) +@pytest.mark.parametrize("key", CLAUDE_PLATFORM_KEYS) +def test_claude_platform_cost_matches_claude_api_cost( + local_model_cost_map: None, key: str, inference_geo: str +) -> None: + usage: Final = Usage( + prompt_tokens=1_000, + completion_tokens=200, + cache_read_input_tokens=300, + cache_creation_input_tokens=250, + inference_geo=inference_geo, + ) + + claude_platform_cost: Final = litellm.cost_per_token( + model=key.removeprefix("bedrock/"), custom_llm_provider="bedrock", usage_object=usage + ) + claude_api_cost: Final = litellm.cost_per_token( + model=key.removeprefix(CLAUDE_PLATFORM_PREFIX), custom_llm_provider="anthropic", usage_object=usage + ) + + assert claude_platform_cost == pytest.approx(claude_api_cost) + assert sum(claude_platform_cost) > 0 diff --git a/tests/unit/test_utils_get_model_info.py b/tests/unit/test_utils_get_model_info.py index 4bd16a2997f..302a7d60cdd 100644 --- a/tests/unit/test_utils_get_model_info.py +++ b/tests/unit/test_utils_get_model_info.py @@ -160,6 +160,7 @@ def _enforce_bedrock_converse_models( info.get("litellm_provider") == "bedrock" and info.get("mode") == "chat" and model not in whitelist_models + and BedrockModelInfo.get_bedrock_route(model) != "claude_platform" and not ( (base_model := BedrockModelInfo.get_base_model(model)) != model and model_cost.get(base_model, {}).get("litellm_provider") == "bedrock_converse" From 287abaf372b337dc3a81cda7b6b91ba40e6cbf4e Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:07:26 -0700 Subject: [PATCH 022/155] chore(cost-map): add openai gpt-5.6-cyber 12-hour cache write rate from the pricing page (#48246) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 1 + model_prices_and_context_window.json | 1 + 2 files changed, 2 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c7dfce0712f..a32ee86dd74 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -37401,6 +37401,7 @@ }, "gpt-5.6-cyber": { "cache_creation_input_token_cost": 1.5625e-05, + "cache_creation_input_token_cost_above_12hr": 3.125e-05, "cache_creation_input_token_cost_above_272k_tokens": 3.125e-05, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_above_272k_tokens": 2.5e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c7dfce0712f..a32ee86dd74 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -37401,6 +37401,7 @@ }, "gpt-5.6-cyber": { "cache_creation_input_token_cost": 1.5625e-05, + "cache_creation_input_token_cost_above_12hr": 3.125e-05, "cache_creation_input_token_cost_above_272k_tokens": 3.125e-05, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_above_272k_tokens": 2.5e-06, From c0dde21c3b4a0892a178a3f71d2f86b6edfa7889 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:07:48 -0700 Subject: [PATCH 023/155] fix(bedrock): take Nova Sonic context and output limits from the Bedrock model cards (#48247) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 9 ++++++++- model_prices_and_context_window.json | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a32ee86dd74..630b1ed9441 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -644,9 +644,13 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 3.3e-07, "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2.75e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-amazon-nova-2-sonic.md", "supports_audio_input": true, "supports_audio_output": true }, @@ -75363,10 +75367,13 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 3.3e-07, "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2.75e-06, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-amazon-nova-2-5-sonic.md", "supports_audio_input": true, "supports_audio_output": true }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a32ee86dd74..630b1ed9441 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -644,9 +644,13 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 3.3e-07, "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2.75e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-amazon-nova-2-sonic.md", "supports_audio_input": true, "supports_audio_output": true }, @@ -75363,10 +75367,13 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 3.3e-07, "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2.75e-06, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-amazon-nova-2-5-sonic.md", "supports_audio_input": true, "supports_audio_output": true }, From fac7459b3ef2005bdbb6d521efe08c7b107f2636 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:08:07 -0700 Subject: [PATCH 024/155] fix(gemini): bill gemini-2.5-flash-native-audio video input at the audio / video rate (#48248) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 2 ++ model_prices_and_context_window.json | 2 ++ 2 files changed, 4 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 630b1ed9441..a3fff5ca26d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -59519,6 +59519,7 @@ "supports_audio_input": true, "supports_audio_output": true, "gemini_native_audio": true, + "input_cost_per_video_token": 3e-06, "supports_function_calling": true, "supports_response_schema": false, "supports_vision": true, @@ -59714,6 +59715,7 @@ "tpm": 250000, "rpm": 10, "gemini_native_audio": true, + "input_cost_per_video_token": 3e-06, "supports_function_calling": true, "supports_response_schema": false, "supports_vision": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 630b1ed9441..a3fff5ca26d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -59519,6 +59519,7 @@ "supports_audio_input": true, "supports_audio_output": true, "gemini_native_audio": true, + "input_cost_per_video_token": 3e-06, "supports_function_calling": true, "supports_response_schema": false, "supports_vision": true, @@ -59714,6 +59715,7 @@ "tpm": 250000, "rpm": 10, "gemini_native_audio": true, + "input_cost_per_video_token": 3e-06, "supports_function_calling": true, "supports_response_schema": false, "supports_vision": true, From 015096af0b9b24eb1b1d6a957650685499aecdd7 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:08:23 -0700 Subject: [PATCH 025/155] chore(pricing): add github_copilot/claude-opus-4.8-fast deprecation date (#48251) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 1 + model_prices_and_context_window.json | 1 + 2 files changed, 2 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a3fff5ca26d..058d49133f0 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -33216,6 +33216,7 @@ "github_copilot/claude-opus-4.8-fast": { "cache_creation_input_token_cost": 1.25e-05, "cache_read_input_token_cost": 1e-06, + "deprecation_date": "2026-11-16", "input_cost_per_token": 1e-05, "litellm_provider": "github_copilot", "max_input_tokens": 200000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a3fff5ca26d..058d49133f0 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -33216,6 +33216,7 @@ "github_copilot/claude-opus-4.8-fast": { "cache_creation_input_token_cost": 1.25e-05, "cache_read_input_token_cost": 1e-06, + "deprecation_date": "2026-11-16", "input_cost_per_token": 1e-05, "litellm_provider": "github_copilot", "max_input_tokens": 200000, From 29edcee9ebff4b7d759eafa2610d90633ca4b652 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:08:37 -0700 Subject: [PATCH 026/155] fix(pricing): set azure gpt-chat-latest max_input_tokens to 128000 (#48252) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 6 +++--- model_prices_and_context_window.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 058d49133f0..02eaa906522 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4343,7 +4343,7 @@ "deprecation_date": "2027-05-18", "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", - "max_input_tokens": 272000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -9236,7 +9236,7 @@ "deprecation_date": "2027-05-18", "input_cost_per_token": 5e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -9683,7 +9683,7 @@ "deprecation_date": "2027-05-18", "input_cost_per_token": 5.5e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 058d49133f0..02eaa906522 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4343,7 +4343,7 @@ "deprecation_date": "2027-05-18", "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", - "max_input_tokens": 272000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -9236,7 +9236,7 @@ "deprecation_date": "2027-05-18", "input_cost_per_token": 5e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -9683,7 +9683,7 @@ "deprecation_date": "2027-05-18", "input_cost_per_token": 5.5e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", From 172aa72ad2dc9f044b38d0fe7f250293a8fda2ef Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Sat, 24 Oct 2026 15:12:07 -0700 Subject: [PATCH 027/155] feat(responses): honor max_agentic_loops for the streamed MCP auto-execute round cap (#47688) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../responses/mcp/mcp_streaming_iterator.py | 13 ++++-- .../mcp/test_aresponses_api_with_mcp.py | 32 +++++++++++--- .../mcp/test_mcp_streaming_iterator.py | 43 +++++++++++++------ 3 files changed, 65 insertions(+), 23 deletions(-) diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index 030911238ec..9e51e2533ea 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Final, cast from litellm._logging import verbose_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.agentic_loop_settings import validated_max_agentic_loops from litellm.responses.litellm_completion_transformation.streaming_iterator import LiteLLMCompletionStreamingIterator from litellm.responses.mcp.request_context import MCPRequestContext from litellm.responses.mcp.stream_round_billing import StreamedRoundBilling @@ -289,6 +290,10 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): self.mcp_tools_with_litellm_proxy = mcp_tools_with_litellm_proxy or [] self.user_api_key_auth = user_api_key_auth self.original_request_params = original_request_params or {} + configured_rounds: Final = validated_max_agentic_loops( + self.original_request_params.get("max_agentic_loops"), field="litellm_params.max_agentic_loops" + ) + self.max_tool_call_rounds: int = MAX_MCP_TOOL_CALL_ROUNDS if configured_rounds is None else configured_rounds self.should_auto_execute = self._should_auto_execute_tools() # Streaming state management @@ -509,7 +514,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): 2. mcp_discovery - Emit MCP discovery events (after response.output_item.added) 3. continue_initial_response - Stream the current round's response (initial or follow-up). On completion, if auto-execute is on and the response contains tool calls, loops back - through tool_execution/follow-up instead of ending, up to MAX_MCP_TOOL_CALL_ROUNDS. + through tool_execution/follow-up instead of ending, up to self.max_tool_call_rounds. 4. tool_execution - Emit tool execution events 5. finished - End iteration """ @@ -1015,14 +1020,14 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): # Remove tool_choice to avoid forcing more tool calls follow_up_params.pop("tool_choice", None) - if self.tool_call_round >= MAX_MCP_TOOL_CALL_ROUNDS: + if self.tool_call_round >= self.max_tool_call_rounds: # Hit the round cap: drop tools entirely so the model must # answer in text instead of emitting another (unexecuted) # tool call that would otherwise end the stream in silence. follow_up_params.pop("tools", None) verbose_logger.warning( - "MCP auto-execute hit MAX_MCP_TOOL_CALL_ROUNDS=%s; forcing a text-only follow-up.", - MAX_MCP_TOOL_CALL_ROUNDS, + "MCP auto-execute hit max_agentic_loops=%s; forcing a text-only follow-up.", + self.max_tool_call_rounds, ) follow_up_response: Final = await aresponses(**follow_up_params) diff --git a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py index e88d951ff09..285d90ef8de 100644 --- a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py +++ b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py @@ -29,6 +29,7 @@ from litellm.proxy._experimental.mcp_server import mcp_server_manager from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ServerListOk from litellm.proxy.utils import _estimate_dispatched_failure_usage, _failure_fields_to_lift from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler +from litellm.responses.mcp.mcp_streaming_iterator import MCPEnhancedStreamingIterator from litellm.types.llms.openai import ( ResponsesAPIResponse, ResponsesAPIStreamingResponse, @@ -2384,8 +2385,11 @@ def _first_index(events: Sequence[object], event_type: str) -> int: async def _scripted_mcp_stream( - model: str, upstream_url: str, rounds: Sequence[bytes] -) -> tuple[tuple[object, ...], respx.Route, AsyncMock, AsyncMock]: + model: str, + upstream_url: str, + rounds: Sequence[bytes], + max_agentic_loops: int | None = None, +) -> tuple[tuple[object, ...], respx.Route, AsyncMock, AsyncMock, MCPEnhancedStreamingIterator]: upstream: Final = respx.post(upstream_url).mock( side_effect=[httpx.Response(200, content=body, headers=SSE_HEADERS) for body in rounds] ) @@ -2402,9 +2406,11 @@ async def _scripted_mcp_stream( tool_choice="required", input=[{"role": "user", "type": "message", "content": "What's the weather in San Francisco?"}], stream=True, + **({"max_agentic_loops": max_agentic_loops} if max_agentic_loops is not None else {}), ) + assert isinstance(response, MCPEnhancedStreamingIterator) events: Final = tuple([event async for event in response]) - return events, upstream, fetch, execute + return events, upstream, fetch, execute, response @pytest.mark.asyncio @@ -2417,7 +2423,7 @@ async def test_streaming_responses_api_with_mcp_tools( model, upstream_url, tool_round, answer_round = SCRIPTED_PROVIDERS[provider] with caplog.at_level(logging.ERROR): - events, upstream, fetch, execute = await _scripted_mcp_stream( + events, upstream, fetch, execute, _ = await _scripted_mcp_stream( model, upstream_url, (tool_round(), answer_round()) ) @@ -2436,6 +2442,22 @@ async def test_streaming_responses_api_with_mcp_tools( assert not [record.getMessage() for record in caplog.records if record.levelno >= logging.ERROR] +@pytest.mark.asyncio +@respx.mock +async def test_streaming_mcp_iterator_uses_max_agentic_loops_from_request(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + model, upstream_url, tool_round, answer_round = SCRIPTED_PROVIDERS["openai"] + + _, _, _, _, iterator = await _scripted_mcp_stream( + model, + upstream_url, + (tool_round(), answer_round()), + max_agentic_loops=7, + ) + + assert iterator.max_tool_call_rounds == 7 + + @pytest.mark.asyncio @respx.mock async def test_streaming_mcp_event_order_and_response_id_consistency( @@ -2445,7 +2467,7 @@ async def test_streaming_mcp_event_order_and_response_id_consistency( model, upstream_url, tool_round, answer_round = SCRIPTED_PROVIDERS["openai"] with caplog.at_level(logging.ERROR): - events, _, _, _ = await _scripted_mcp_stream(model, upstream_url, (tool_round(), answer_round())) + events, _, _, _, _ = await _scripted_mcp_stream(model, upstream_url, (tool_round(), answer_round())) created: Final = _first_index(events, "response.created") in_progress: Final = _first_index(events, "response.in_progress") diff --git a/tests/unit/responses/mcp/test_mcp_streaming_iterator.py b/tests/unit/responses/mcp/test_mcp_streaming_iterator.py index 8eb73b7c6e1..ed39da1b7a9 100644 --- a/tests/unit/responses/mcp/test_mcp_streaming_iterator.py +++ b/tests/unit/responses/mcp/test_mcp_streaming_iterator.py @@ -2,6 +2,7 @@ import sys import types from contextlib import nullcontext from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -9,7 +10,6 @@ from mcp.types import CallToolResult, TextContent import litellm # noqa: F401 - ensures litellm.responses.main is registered in sys.modules from litellm.responses.mcp.mcp_streaming_iterator import ( - MAX_MCP_TOOL_CALL_ROUNDS, MCPEnhancedStreamingIterator, ) from litellm.types.llms.openai import ( @@ -96,7 +96,7 @@ def _mock_mcp_environment(monkeypatch: pytest.MonkeyPatch) -> AsyncMock: return call_tool -def _make_iterator(initial_chunks) -> MCPEnhancedStreamingIterator: +def _make_iterator(initial_chunks, max_agentic_loops: int | str | None = None) -> MCPEnhancedStreamingIterator: return MCPEnhancedStreamingIterator( base_iterator=_FakeAsyncStream(initial_chunks), mcp_events=[], @@ -107,6 +107,7 @@ def _make_iterator(initial_chunks) -> MCPEnhancedStreamingIterator: "model": "gpt-4", "input": "what is berriai/litellm?", "tools": [{"type": "mcp"}], + **({"max_agentic_loops": max_agentic_loops} if max_agentic_loops is not None else {}), }, ) @@ -157,35 +158,43 @@ async def test_second_round_tool_call_is_executed_and_reaches_final_text(monkeyp @pytest.mark.asyncio -async def test_tool_call_rounds_are_capped(monkeypatch): +@pytest.mark.parametrize( + ("max_agentic_loops", "expected_cap"), + ( + pytest.param(None, 5, id="unset"), + pytest.param(7, 7, id="integer"), + pytest.param("7", 7, id="integer-string"), + pytest.param(2, 2, id="custom-cap"), + ), +) +async def test_tool_call_rounds_are_capped(monkeypatch, max_agentic_loops, expected_cap): """ A model that keeps calling tools every round must not loop forever — - auto-execution stops at MAX_MCP_TOOL_CALL_ROUNDS, and the follow-up made + auto-execution stops at the configured round cap, and the follow-up made at the cap drops "tools" from the request so the model is forced to answer in text instead of the stream just hanging. """ call_tool = _mock_mcp_environment(monkeypatch) - # Rounds 2..MAX_MCP_TOOL_CALL_ROUNDS keep calling the tool; the call made - # once the cap is hit returns a text-only response. - tool_call_streams = [ - _tool_call_stream(f"call_{i}", "read_wiki_contents") for i in range(2, MAX_MCP_TOOL_CALL_ROUNDS + 1) - ] - aresponses_mock = AsyncMock(side_effect=[*tool_call_streams, _text_only_stream("giving up on tools")]) + tool_call_streams: Final = tuple( + _tool_call_stream(f"call_{i}", "read_wiki_contents") for i in range(2, expected_cap + 1) + ) + aresponses_mock: Final = AsyncMock(side_effect=(*tool_call_streams, _text_only_stream("giving up on tools"))) monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) iterator = _make_iterator( [ _output_item_added_chunk(), _completed_chunk([_function_call("call_1", "read_wiki_contents")]), - ] + ], + max_agentic_loops=max_agentic_loops, ) _ = [chunk async for chunk in iterator] - assert iterator.tool_call_round == MAX_MCP_TOOL_CALL_ROUNDS - assert call_tool.call_count == MAX_MCP_TOOL_CALL_ROUNDS - assert aresponses_mock.call_count == MAX_MCP_TOOL_CALL_ROUNDS + assert iterator.tool_call_round == expected_cap + assert call_tool.call_count == expected_cap + assert aresponses_mock.call_count == expected_cap # Every follow-up before the cap still offered tools; only the capped one drops them. for call in aresponses_mock.call_args_list[:-1]: @@ -193,6 +202,12 @@ async def test_tool_call_rounds_are_capped(monkeypatch): assert "tools" not in aresponses_mock.call_args_list[-1].kwargs +@pytest.mark.parametrize(("max_agentic_loops", "expected_error"), ((0, ValueError), ("three", TypeError))) +def test_invalid_max_agentic_loops_raises_when_creating_iterator(max_agentic_loops, expected_error): + with pytest.raises(expected_error, match=r"litellm_params\.max_agentic_loops"): + _make_iterator([], max_agentic_loops=max_agentic_loops) + + @pytest.mark.asyncio async def test_continuation_id_is_final_round_not_interim_tool_call(monkeypatch): """ From 1418c1990fcc403d7598d97832f1272f45049299 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:33:17 -0700 Subject: [PATCH 028/155] feat(guardrails): add native Reco guardrail provider (pre_call), fail closed by default (#48243) * feat(guardrails): add native Reco guardrail provider (pre_call) Registers guardrail: reco as a first-class vendor integration, built on the existing generic guardrail API wire contract rather than reimplementing the HTTP client and payload mapping. Maps reco_tenant_id to the X-Reco-Tenant-Id header and hardcodes fail_open/fail_on_error since those should not be customer-configurable. v1 only supports mode: pre_call; post_call and streaming redaction already work against the existing generic pilot and are deferred. Also overrides apply_guardrail to re-raise blocked-request errors with the guardrail's own configured name: the base class hardcodes a module-level name in that exception, which would otherwise mislabel every blocked Reco request in logs. * feat(guardrails): configurable Reco fail modes Expose unreachable_fallback and fail_on_error on the Reco guardrail and keep fail-open as the default. Values under optional_params win. Top-level litellm_params values apply only when set explicitly, since their defaults are fail-closed and would otherwise flip the default Also type RecoGuardrail's forwarded kwargs with an Unpack'd TypedDict instead of silencing ANN003 Signed-off-by: MickeyShnaiderman-RecoLabs Co-authored-by: Cursor * chore(guardrails): align Reco with main's CI gates Upstream moved unit tests from tests/test_litellm to tests/unit, so test_reco.py was invoked by no CI job. Move it next to the other guardrail tests Replace RecoGuardrail's forwarded kwargs with explicit keyword params, read optional_params through a pydantic adapter instead of untyped dict/getattr access, freeze RecoOptionalParams, and drop mutable-ok markers that suppress nothing, to stay within main's type-discipline and basedpyright budgets Signed-off-by: MickeyShnaiderman-RecoLabs Co-authored-by: Cursor * feat(guardrails): fail closed by default in the Reco guardrail Matches GenericGuardrailAPI and every other guardrail here: a Reco error or outage now blocks the request unless the admin sets fail_on_error: false. Tests cover the default, explicit fail-open, and top-level vs optional_params precedence Co-authored-by: RecoLabs-YaronBratspiess <208601538+RecoLabs-YaronBratspiess@users.noreply.github.com> Co-authored-by: MickeyShnaiderman-RecoLabs <77006830+MickeyShnaiderman-RecoLabs@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(guardrails): honor explicit fail-open on Reco when optional_params is a model model_dump(exclude_unset=True) so pydantic defaults in optional_params no longer override an explicit top-level fail_on_error. Also drop the duplicate log_guardrail_information decorator, call the parent by keyword, freeze RecoConfigModel, and type the tests Co-authored-by: RecoLabs-YaronBratspiess <208601538+RecoLabs-YaronBratspiess@users.noreply.github.com> Co-authored-by: MickeyShnaiderman-RecoLabs <77006830+MickeyShnaiderman-RecoLabs@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(guardrails): keep Reco config model assignable and the hook decorator the gate requires frozen=True on RecoConfigModel froze the shared LitellmParams mixin, which the guardrail registry assigns resolved secrets on, so every guardrail failed to load. Dropped it and added a test that LitellmParams stays assignable. Restored log_guardrail_information on the Reco override since tests/code_coverage_tests/check_guardrail_apply_decorator.py requires it on every hook. Test params typed as Mapping for LIT001 Co-authored-by: RecoLabs-YaronBratspiess <208601538+RecoLabs-YaronBratspiess@users.noreply.github.com> Co-authored-by: MickeyShnaiderman-RecoLabs <77006830+MickeyShnaiderman-RecoLabs@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(guardrails): format the Reco test file Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Signed-off-by: MickeyShnaiderman-RecoLabs Co-authored-by: Yaron Bratspiess Co-authored-by: MickeyShnaiderman-RecoLabs Co-authored-by: Cursor Co-authored-by: joshua Co-authored-by: MickeyShnaiderman-RecoLabs <77006830+MickeyShnaiderman-RecoLabs@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../guardrail_hooks/reco/__init__.py | 72 +++++ .../guardrails/guardrail_hooks/reco/reco.py | 84 ++++++ litellm/types/guardrails.py | 5 + .../proxy/guardrails/guardrail_hooks/reco.py | 48 +++ .../guardrails/guardrail_hooks/test_reco.py | 285 ++++++++++++++++++ 5 files changed, 494 insertions(+) create mode 100644 litellm/proxy/guardrails/guardrail_hooks/reco/__init__.py create mode 100644 litellm/proxy/guardrails/guardrail_hooks/reco/reco.py create mode 100644 litellm/types/proxy/guardrails/guardrail_hooks/reco.py create mode 100644 tests/unit/proxy/guardrails/guardrail_hooks/test_reco.py diff --git a/litellm/proxy/guardrails/guardrail_hooks/reco/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/reco/__init__.py new file mode 100644 index 00000000000..a15a372c462 --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/reco/__init__.py @@ -0,0 +1,72 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from pydantic import BaseModel, TypeAdapter + +from litellm.secret_managers.main import get_secret_str +from litellm.types.guardrails import SupportedGuardrailIntegrations +from litellm.types.proxy.guardrails.guardrail_hooks.reco import RecoOptionalParams + +from .reco import RecoGuardrail + +if TYPE_CHECKING: + from litellm.types.guardrails import Guardrail, LitellmParams + +_CONFIG_MAPPING: Final = TypeAdapter(Mapping[str, object]) + + +def _optional_params_mapping(optional_params: object) -> Mapping[str, object]: + """optional_params arrives as a pydantic object from YAML, or as a plain dict when reconciled from the DB.""" + if optional_params is None: + return MappingProxyType({}) + raw: Final = ( + optional_params.model_dump(exclude_unset=True) if isinstance(optional_params, BaseModel) else optional_params + ) + return _CONFIG_MAPPING.validate_python(raw) + + +def _first_configured(field_name: str, sources: tuple[Mapping[str, object], ...]) -> object: + value: Final = next((source[field_name] for source in sources if source.get(field_name) is not None), None) + if isinstance(value, str) and value.startswith("os.environ/"): + return get_secret_str(value) + return value + + +def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail") -> RecoGuardrail: + import litellm + + # Top-level fields only count when set explicitly: their defaults are fail-closed and would flip Reco's. + explicit_top_level: Final = _CONFIG_MAPPING.validate_python( + litellm_params.model_dump(include=litellm_params.model_fields_set & RecoOptionalParams.model_fields.keys()) + ) + sources: Final = (_optional_params_mapping(litellm_params.optional_params), explicit_top_level) + configured: Final = ( + (field_name, _first_configured(field_name, sources)) for field_name in RecoOptionalParams.model_fields + ) + reco_params: Final = RecoOptionalParams.model_validate( + {field_name: value for field_name, value in configured if value is not None} + ) + + _reco_callback: Final = RecoGuardrail( + guardrail_name=guardrail["guardrail_name"], + reco_tenant_id=reco_params.reco_tenant_id, + api_base=reco_params.api_base, + unreachable_fallback=reco_params.unreachable_fallback, + fail_on_error=reco_params.fail_on_error, + event_hook=litellm_params.mode, + default_on=bool(litellm_params.default_on), + ) + litellm.logging_callback_manager.add_litellm_callback(_reco_callback) + + return _reco_callback + + +guardrail_initializer_registry: Final = { + SupportedGuardrailIntegrations.RECO.value: initialize_guardrail, +} + + +guardrail_class_registry: Final = { + SupportedGuardrailIntegrations.RECO.value: RecoGuardrail, +} diff --git a/litellm/proxy/guardrails/guardrail_hooks/reco/reco.py b/litellm/proxy/guardrails/guardrail_hooks/reco/reco.py new file mode 100644 index 00000000000..c8998a20367 --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/reco/reco.py @@ -0,0 +1,84 @@ +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Final, Literal, Optional + +from litellm.exceptions import GuardrailRaisedException +from litellm.integrations.custom_guardrail import ( + log_guardrail_information, # pyright: ignore[reportUnknownVariableType] # decorator is untyped +) +from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api.generic_guardrail_api import ( + GenericGuardrailAPI, +) +from litellm.types.guardrails import GuardrailEventHooks, Mode +from litellm.types.proxy.guardrails.guardrail_hooks.reco import validate_reco_tenant_id +from litellm.types.utils import GenericGuardrailAPIInputs + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel + from litellm.types.proxy.guardrails.guardrail_hooks.reco import RecoOptionalParams + + +class RecoGuardrail(GenericGuardrailAPI): + """Reco guardrail integration for LiteLLM, built on the Generic Guardrail API wire contract.""" + + def __init__( + self, + reco_tenant_id: str | None, + api_base: str | None, + headers: Mapping[str, str] | None = None, + unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_closed", + fail_on_error: bool = True, + guardrail_name: str | None = None, + event_hook: str | Sequence[str] | Mode | None = None, + default_on: bool = False, + ) -> None: + if not reco_tenant_id: + raise ValueError("reco_tenant_id is required for the Reco guardrail") + if not api_base: + raise ValueError("api_base is required for the Reco guardrail") + + validated_tenant_id: Final = validate_reco_tenant_id(reco_tenant_id) + + super().__init__( # pyright: ignore[reportUnknownMemberType] # parent takes untyped extra_headers and **kwargs + headers={**(headers or {}), "X-Reco-Tenant-Id": validated_tenant_id}, + api_base=api_base, + unreachable_fallback=unreachable_fallback, + fail_on_error=fail_on_error, + guardrail_name=guardrail_name, + event_hook=event_hook, + default_on=default_on, + ) + + @log_guardrail_information + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], # mutable-ok: must match GenericGuardrailAPI.apply_guardrail's own param type + input_type: Literal["request", "response"], + logging_obj: Optional["LiteLLMLoggingObj"] = None, + ) -> GenericGuardrailAPIInputs: + # GenericGuardrailAPI raises with a hardcoded module-level guardrail_name on BLOCKED, + # not the guardrail's own configured name. Re-raise with the right one instead of + # patching the shared base class. + try: + return await super().apply_guardrail( # pyright: ignore[reportUnknownMemberType] # parent annotates request_data as a bare dict + inputs=inputs, request_data=request_data, input_type=input_type, logging_obj=logging_obj + ) + except GuardrailRaisedException as e: + raise GuardrailRaisedException( + guardrail_name=self.guardrail_name, + message=e.message, + should_wrap_with_default_message=False, + status_code=e.status_code, + blocked_content=e.blocked_content, + ) from e + + @staticmethod + def get_config_model() -> type["GuardrailConfigModel[RecoOptionalParams]"] | None: + from litellm.types.proxy.guardrails.guardrail_hooks.reco import RecoConfigModel + + return RecoConfigModel + + @classmethod + def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]: # mutable-ok: fixed by base class + return [GuardrailEventHooks.pre_call] diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index ff063d2e233..e1b891caaaf 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -59,6 +59,9 @@ from litellm.types.proxy.guardrails.guardrail_hooks.qohash import ( from litellm.types.proxy.guardrails.guardrail_hooks.qualifire import ( QualifireGuardrailConfigModel, ) +from litellm.types.proxy.guardrails.guardrail_hooks.reco import ( + RecoConfigModel, +) from litellm.types.proxy.guardrails.guardrail_hooks.repelloai import ( RepelloAIGuardrailConfigModel, ) @@ -153,6 +156,7 @@ class SupportedGuardrailIntegrations(Enum): TYPESAFE = "typesafe" STRAIKER = "straiker" ALICE = "alice" + RECO = "reco" AGENT_365 = "agent_365" LLM_SHIELD_PROXY = "llm_shield_proxy" CONDUCT = "conduct" @@ -1398,6 +1402,7 @@ class LitellmParams( # pyright: ignore[reportIncompatibleVariableOverride] # o QostodianNexusConfigModel, VigilGuardGuardrailConfigModel, SingulrGuardrailConfigModel, + RecoConfigModel, Agent365GuardrailConfigModel, WingbackGuardrailConfigModel, ): diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/reco.py b/litellm/types/proxy/guardrails/guardrail_hooks/reco.py new file mode 100644 index 00000000000..ef3b3935340 --- /dev/null +++ b/litellm/types/proxy/guardrails/guardrail_hooks/reco.py @@ -0,0 +1,48 @@ +import uuid +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from .base import GuardrailConfigModel + + +def validate_reco_tenant_id(value: str) -> str: + try: + uuid.UUID(value) + except ValueError as e: + raise ValueError(f"reco_tenant_id must be a valid UUID, got {value!r}") from e + return value + + +class RecoOptionalParams(BaseModel): + """Configuration parameters for the Reco guardrail""" + + model_config = ConfigDict(frozen=True) + + reco_tenant_id: str = Field( + description="Tenant identifier for the Reco account, as a UUID. Sent as the X-Reco-Tenant-Id header on every guardrail request.", + ) + api_base: str = Field( + description="Base URL for the Reco guardrail API. Reco's endpoint is per-region and per-silo, so this must be set explicitly rather than defaulted.", + ) + unreachable_fallback: Literal["fail_closed", "fail_open"] = Field( + default="fail_closed", + description="Behavior when Reco is unreachable. 'fail_closed' (default) blocks the request; 'fail_open' lets it through uninspected. Only applies while fail_on_error is true.", + ) + fail_on_error: bool = Field( + default=True, + description="Behavior on any Reco error, including unreachability. True (default) blocks the request; False lets it through uninspected.", + ) + + @field_validator("reco_tenant_id") + @classmethod + def _check_reco_tenant_id(cls, v: str) -> str: + return validate_reco_tenant_id(v) + + +class RecoConfigModel(GuardrailConfigModel[RecoOptionalParams]): + """Configuration parameters for the Reco guardrail""" + + @staticmethod + def ui_friendly_name() -> str: + return "Reco" diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_reco.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_reco.py new file mode 100644 index 00000000000..9a7670dbe97 --- /dev/null +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_reco.py @@ -0,0 +1,285 @@ +""" +Tests for the Reco guardrail integration. +""" + +from collections.abc import Mapping +from typing import Final +from unittest.mock import MagicMock, patch + +import httpx +import pytest +from pydantic import ValidationError + +from litellm.exceptions import GuardrailRaisedException +from litellm.proxy.guardrails.guardrail_hooks.reco import ( + RecoGuardrail, + initialize_guardrail, +) +from litellm.types.guardrails import GuardrailEventHooks, LitellmParams +from litellm.types.proxy.guardrails.guardrail_hooks.reco import ( + RecoConfigModel, + RecoOptionalParams, +) +from litellm.types.utils import GenericGuardrailAPIInputs + + +@pytest.fixture +def reco_guardrail() -> RecoGuardrail: + return RecoGuardrail( + reco_tenant_id="11111111-1111-1111-1111-111111111111", + api_base="https://edge1.us.reco.ai", + guardrail_name="my-reco-guardrail", + event_hook="pre_call", + default_on=True, + ) + + +class TestRecoGuardrailConfiguration: + def test_tenant_id_forwarded_as_header(self, reco_guardrail: RecoGuardrail) -> None: + assert reco_guardrail.headers == {"X-Reco-Tenant-Id": "11111111-1111-1111-1111-111111111111"} + + def test_tenant_header_merged_with_existing_headers(self) -> None: + guardrail: Final = RecoGuardrail( + reco_tenant_id="11111111-1111-1111-1111-111111111111", + api_base="https://edge1.us.reco.ai", + headers={"X-Custom": "value"}, + ) + assert guardrail.headers == { + "X-Custom": "value", + "X-Reco-Tenant-Id": "11111111-1111-1111-1111-111111111111", + } + + def test_missing_reco_tenant_id_raises(self) -> None: + with pytest.raises(ValueError, match="reco_tenant_id"): + RecoGuardrail(reco_tenant_id=None, api_base="https://edge1.us.reco.ai") + + def test_missing_api_base_raises(self) -> None: + with pytest.raises(ValueError, match="api_base"): + RecoGuardrail(reco_tenant_id="11111111-1111-1111-1111-111111111111", api_base=None) + + def test_non_uuid_reco_tenant_id_raises(self) -> None: + with pytest.raises(ValueError, match="UUID"): + RecoGuardrail(reco_tenant_id="tenant-123", api_base="https://edge1.us.reco.ai") + + def test_optional_params_rejects_non_uuid_tenant_id(self) -> None: + with pytest.raises(ValidationError, match="UUID"): + RecoOptionalParams(reco_tenant_id="tenant-123", api_base="https://edge1.us.reco.ai") + + def test_get_supported_event_hooks_returns_pre_call_only(self) -> None: + assert RecoGuardrail.get_supported_event_hooks() == [GuardrailEventHooks.pre_call] + + def test_get_config_model_returns_reco_config_model(self) -> None: + assert RecoGuardrail.get_config_model() is RecoConfigModel + + def test_config_model_ui_friendly_name(self) -> None: + assert RecoConfigModel.ui_friendly_name() == "Reco" + + def test_litellm_params_stay_assignable_for_registry_secret_resolution(self) -> None: + litellm_params: Final = LitellmParams(guardrail="reco", mode="pre_call") + + litellm_params.api_base = "https://edge1.us.reco.ai" + + assert litellm_params.api_base == "https://edge1.us.reco.ai" + + +class TestRecoGuardrailInitializer: + def test_initialize_guardrail_wires_optional_params(self) -> None: + litellm_params: Final = LitellmParams( + guardrail="reco", + mode="pre_call", + optional_params={ + "reco_tenant_id": "22222222-2222-2222-2222-222222222222", + "api_base": "https://edge2.eu.reco.ai", + }, + ) + + guardrail: Final = initialize_guardrail(litellm_params, {"guardrail_name": "prod-reco"}) + + assert isinstance(guardrail, RecoGuardrail) + assert guardrail.guardrail_name == "prod-reco" + assert guardrail.headers == {"X-Reco-Tenant-Id": "22222222-2222-2222-2222-222222222222"} + assert guardrail.api_base == "https://edge2.eu.reco.ai/beta/litellm_basic_guardrail_api" + + def test_initialize_guardrail_reads_plain_dict_optional_params(self) -> None: + """Guardrails reconciled from the DB can carry `optional_params` as a + plain dict rather than a pydantic object; attribute-only access would + silently read None for every field.""" + litellm_params: Final = LitellmParams.model_construct( + guardrail="reco", + optional_params={ + "reco_tenant_id": "33333333-3333-3333-3333-333333333333", + "api_base": "https://edge3.eu.reco.ai", + }, + mode="pre_call", + default_on=True, + ) + + guardrail: Final = initialize_guardrail(litellm_params, {"guardrail_name": "prod-reco"}) + + assert guardrail.headers == {"X-Reco-Tenant-Id": "33333333-3333-3333-3333-333333333333"} + assert guardrail.api_base == "https://edge3.eu.reco.ai/beta/litellm_basic_guardrail_api" + + def test_initialize_guardrail_resolves_env_var_reference(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("RECO_TEST_TENANT_ID", "44444444-4444-4444-4444-444444444444") + litellm_params: Final = LitellmParams( + guardrail="reco", + optional_params={ + "reco_tenant_id": "os.environ/RECO_TEST_TENANT_ID", + "api_base": "https://edge4.eu.reco.ai", + }, + mode="pre_call", + default_on=True, + ) + + guardrail: Final = initialize_guardrail(litellm_params, {"guardrail_name": "prod-reco"}) + + assert guardrail.headers == {"X-Reco-Tenant-Id": "44444444-4444-4444-4444-444444444444"} + + +def _litellm_params(top_level: Mapping[str, object], nested: Mapping[str, object]) -> LitellmParams: + return LitellmParams( + guardrail="reco", + mode="pre_call", + optional_params={ + "reco_tenant_id": "22222222-2222-2222-2222-222222222222", + "api_base": "https://edge2.eu.reco.ai", + **nested, + }, + **top_level, + ) + + +async def _apply_with_reco_unreachable(guardrail: RecoGuardrail) -> GenericGuardrailAPIInputs: + with patch.object(guardrail.async_handler, "post", side_effect=httpx.ConnectError("connection refused")): + return await guardrail.apply_guardrail( + inputs={"texts": ["hello"]}, + request_data={"metadata": {}}, + input_type="request", + ) + + +class TestRecoGuardrailFailureModes: + @pytest.mark.asyncio + async def test_unreachable_reco_blocks_request_by_default(self, reco_guardrail: RecoGuardrail) -> None: + with pytest.raises(Exception, match="connection refused"): + await _apply_with_reco_unreachable(reco_guardrail) + + @pytest.mark.asyncio + async def test_fail_open_lets_request_through_when_reco_unreachable(self) -> None: + guardrail: Final = RecoGuardrail( + reco_tenant_id="11111111-1111-1111-1111-111111111111", + api_base="https://edge1.us.reco.ai", + fail_on_error=False, + ) + + result: Final = await _apply_with_reco_unreachable(guardrail) + + assert result["texts"] == ["hello"] + + @pytest.mark.asyncio + async def test_fail_closed_blocks_request_when_reco_unreachable(self) -> None: + guardrail: Final = RecoGuardrail( + reco_tenant_id="11111111-1111-1111-1111-111111111111", + api_base="https://edge1.us.reco.ai", + unreachable_fallback="fail_closed", + fail_on_error=True, + ) + + with pytest.raises(Exception, match="connection refused"): + await _apply_with_reco_unreachable(guardrail) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("top_level", "nested"), + [ + pytest.param({}, {}, id="defaults"), + pytest.param({}, {"unreachable_fallback": "fail_closed", "fail_on_error": True}, id="optional-params"), + pytest.param({"unreachable_fallback": "fail_closed", "fail_on_error": True}, {}, id="top-level"), + ], + ) + async def test_initialize_guardrail_fails_closed_when_configured( + self, top_level: Mapping[str, object], nested: Mapping[str, object] + ) -> None: + guardrail: Final = initialize_guardrail(_litellm_params(top_level, nested), {"guardrail_name": "prod-reco"}) + + with pytest.raises(Exception, match="connection refused"): + await _apply_with_reco_unreachable(guardrail) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("top_level", "nested"), + [ + pytest.param({}, {"fail_on_error": False}, id="optional-params"), + pytest.param({"fail_on_error": False}, {}, id="top-level"), + pytest.param( + {"unreachable_fallback": "fail_closed", "fail_on_error": True}, + {"fail_on_error": False}, + id="optional-params-wins-over-top-level", + ), + ], + ) + async def test_initialize_guardrail_fails_open_when_configured( + self, top_level: Mapping[str, object], nested: Mapping[str, object] + ) -> None: + guardrail: Final = initialize_guardrail(_litellm_params(top_level, nested), {"guardrail_name": "prod-reco"}) + + result: Final = await _apply_with_reco_unreachable(guardrail) + + assert result["texts"] == ["hello"] + + @pytest.mark.asyncio + async def test_initialize_guardrail_keeps_top_level_fail_open_when_nested_params_are_a_model(self) -> None: + nested: Final = RecoOptionalParams( + reco_tenant_id="22222222-2222-2222-2222-222222222222", api_base="https://edge2.eu.reco.ai" + ) + litellm_params: Final = LitellmParams.model_construct( + guardrail="reco", mode="pre_call", optional_params=nested, fail_on_error=False + ) + + guardrail: Final = initialize_guardrail(litellm_params, {"guardrail_name": "prod-reco"}) + result: Final = await _apply_with_reco_unreachable(guardrail) + + assert result["texts"] == ["hello"] + + +class TestRecoGuardrailBlocking: + @pytest.mark.asyncio + async def test_action_blocked_raises_exception_with_configured_guardrail_name( + self, reco_guardrail: RecoGuardrail + ) -> None: + mock_response: Final = MagicMock() + mock_response.json.return_value = { + "action": "BLOCKED", + "blocked_reason": "Sensitive data detected", + } + mock_response.raise_for_status = MagicMock() + + with patch.object(reco_guardrail.async_handler, "post", return_value=mock_response) as mock_post: + with pytest.raises(GuardrailRaisedException) as exc_info: + await reco_guardrail.apply_guardrail( + inputs={"texts": ["some sensitive prompt"]}, + request_data={"metadata": {}}, + input_type="request", + ) + + assert exc_info.value.guardrail_name == "my-reco-guardrail" + assert str(exc_info.value) == "Sensitive data detected" + assert exc_info.value.blocked_content is True + + _, call_kwargs = mock_post.call_args + assert call_kwargs["headers"]["X-Reco-Tenant-Id"] == "11111111-1111-1111-1111-111111111111" + + @pytest.mark.asyncio + async def test_action_none_allows_content(self, reco_guardrail: RecoGuardrail) -> None: + mock_response: Final = MagicMock() + mock_response.json.return_value = {"action": "NONE", "texts": ["hello"]} + mock_response.raise_for_status = MagicMock() + + with patch.object(reco_guardrail.async_handler, "post", return_value=mock_response): + result: Final = await reco_guardrail.apply_guardrail( + inputs={"texts": ["hello"]}, + request_data={"metadata": {}}, + input_type="request", + ) + + assert result["texts"] == ["hello"] From 34ec3fcdfae89cac4edb873b6e80027789a05ea4 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:11:33 -0700 Subject: [PATCH 029/155] fix(router): skip declared text-only deployments for image requests on every routing strategy (#48241) * fix(router): skip declared text-only deployments for image requests without enable_pre_call_checks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): run the vision filter after the cooldown filter so a cooled vision leg falls back to the text leg Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): apply the vision filter after pre-call checks and tidy the new tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): mark the remaining locals in the vision filter tests Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: joshua Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 13 ++- litellm/router.py | 54 +++++++---- .../proxy/test_common_request_processing.py | 21 +++++ tests/unit/test_router/test_router.py | 93 ++++++++++++++----- 4 files changed, 135 insertions(+), 46 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 23faa964d5f..cec176ac559 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -4821,10 +4821,15 @@ class ProxyBaseLLMRequestProcessing: model_info = metadata.get("model_info") or {} model_id = model_info.get("id", None) - # 3. Final fallback to self.data["litellm_metadata"] (for routes like /v1/responses that populate data before error) + # 3. Final fallback to the request metadata the router updates in place with the selected deployment if not model_id: - litellm_metadata: Final = self.data.get("litellm_metadata", {}) or {} - model_info = litellm_metadata.get("model_info", {}) or {} - model_id = model_info.get("id", None) + model_id = self._metadata_model_id("litellm_metadata") or self._metadata_model_id("metadata") return model_id + + def _metadata_model_id(self, metadata_key: str) -> str | None: + match self.data.get(metadata_key): + case {"model_info": {"id": str() as request_model_id}}: + return request_model_id + case _: + return None diff --git a/litellm/router.py b/litellm/router.py index 96102b678c3..7993b5b921c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -14315,6 +14315,30 @@ class Router: return healthy_deployments return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates + @staticmethod + def _without_declared_text_only_deployments( + healthy_deployments: _RouteDeploymentsT, + group: str, + messages: list[dict[str, str]] | None, # mutable-ok: handed to _request_carries_image as-is + request_kwargs: Mapping[str, object] | None, + ) -> _RouteDeploymentsT: + if isinstance(healthy_deployments, Mapping) or not Router._request_carries_image( + messages, request_kwargs or {} + ): + return healthy_deployments + + vision_capable_deployments: Final = [ + deployment + for deployment in healthy_deployments + if not Router._deployment_declares_no_vision( + cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape + group, + ) + ] + if not vision_capable_deployments: + return healthy_deployments + return cast(_RouteDeploymentsT, vision_capable_deployments) # cast-ok: same list shape as the input candidates + @staticmethod def _request_carries_image( messages: list[dict[str, str]] | None, # mutable-ok: handed to resolve_structured_messages as-is @@ -14355,7 +14379,6 @@ class Router: - model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models - filter models above rpm limits - if region given, filter out models not in that region / unknown region - - request carries an image and the deployment declares supports_vision false, unless every candidate does - [TODO] function call and model doesn't support function calling """ @@ -14515,20 +14538,9 @@ class Router: model=model, llm_provider="", ) - remaining: Final = tuple( - (idx, deployment) for idx, deployment in param_candidates if idx not in excluded_indices - ) - text_only_indices: Final[frozenset[int]] = ( - frozenset(idx for idx, deployment in remaining if self._deployment_declares_no_vision(deployment, model)) - if remaining and self._request_carries_image(messages, request_settings) - else frozenset() - ) - dropped_indices: Final[frozenset[int]] = excluded_indices | ( - text_only_indices if len(text_only_indices) < len(remaining) else frozenset() - ) - if len(dropped_indices) > 0: + if len(excluded_indices) > 0: # Single-pass filter using set for O(1) lookups (avoids O(n^2) from repeated pops) - _returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in dropped_indices] + _returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in excluded_indices] return _returned_deployments @@ -15051,8 +15063,11 @@ class Router: ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) - batch_candidates: Final = cast( # cast-ok: router deployments are plain dicts - list[dict[str, object]], healthy_deployments + batch_candidates: Final = self._without_declared_text_only_deployments( + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + model, + messages, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict ) embedding_batch_admitting: Final = deployments_admitting_embedding_batch( batch_candidates, @@ -16291,8 +16306,11 @@ class Router: ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) - batch_candidates: Final = cast( # cast-ok: router deployments are plain dicts - list[dict[str, object]], healthy_deployments + batch_candidates: Final = self._without_declared_text_only_deployments( + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + model, + messages, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict ) embedding_batch_admitting: Final = deployments_admitting_embedding_batch( batch_candidates, diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 92bf2b45fab..82dd3479321 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -12027,3 +12027,24 @@ class TestResponseCostFromLoggingObjForCacheHits: ) assert cost == "" + + +def test_maybe_get_model_id_reads_the_chat_metadata_the_router_updated_in_place(): + processing_obj: Final = ProxyBaseLLMRequestProcessing(data={"metadata": {"model_info": {"id": "declared-text"}}}) + + assert processing_obj.maybe_get_model_id(None) == "declared-text" + + +def test_maybe_get_model_id_prefers_litellm_metadata_over_caller_metadata_without_a_deployment(): + processing_obj: Final = ProxyBaseLLMRequestProcessing( + data={ + "metadata": {"trace_id": "t-1"}, + "litellm_metadata": {"model_info": {"id": "vision"}}, + } + ) + + assert processing_obj.maybe_get_model_id(None) == "vision" + + +def test_maybe_get_model_id_returns_none_before_the_router_picked_a_deployment(): + assert ProxyBaseLLMRequestProcessing(data={"metadata": {}}).maybe_get_model_id(None) is None diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 2288d4311a9..568a05e8ff4 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -30022,64 +30022,109 @@ _DECLARED_TEXT_LEG: Final[dict[str, object]] = { } -def _pre_call_router(*deployments: dict[str, object], routing_strategy: str = "latency-based-routing") -> litellm.Router: +def _mixed_router(*deployments: dict[str, object], routing_strategy: str = "latency-based-routing") -> litellm.Router: return litellm.Router( model_list=[{"model_name": "mixed", **deployment} for deployment in deployments], routing_strategy=routing_strategy, - enable_pre_call_checks=True, ) -def _pre_call_ids(router: litellm.Router, deployments: tuple[dict[str, object], ...], **request: object) -> list[str]: - return [d["model_info"]["id"] for d in router._pre_call_checks(model="mixed", healthy_deployments=list(deployments), **request)] +def _candidate_ids( + router: litellm.Router, + messages: list[dict[str, object]] | None = None, + request_kwargs: dict[str, object] | None = None, +) -> list[str]: + candidates: Final = router._without_declared_text_only_deployments( + router.get_model_list(model_name="mixed"), "mixed", messages, request_kwargs + ) + return [d["model_info"]["id"] for d in candidates] -def test_pre_call_checks_drop_declared_text_only_legs_for_image_requests(): - router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG) - assert _pre_call_ids(router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=_IMAGE_CHAT) == ["vision"] +def test_router_picks_the_vision_leg_for_image_requests_without_enable_pre_call_checks(): + router: Final = _mixed_router(_VISION_LEG, _DECLARED_TEXT_LEG) + assert router.enable_pre_call_checks is False + for _ in range(5): + assert router.get_available_deployment(model="mixed", messages=_IMAGE_CHAT)["model_info"]["id"] == "vision" -def test_pre_call_checks_keep_declared_text_only_legs_for_text_requests(): - router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG) - assert _pre_call_ids(router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=[{"role": "user", "content": "Say hi"}]) == [ +@pytest.mark.asyncio +async def test_async_router_picks_the_vision_leg_for_responses_images_without_enable_pre_call_checks(): + router: Final = _mixed_router(_VISION_LEG, _DECLARED_TEXT_LEG) + for _ in range(5): + deployment: Final = await router.async_get_available_deployment( + model="mixed", request_kwargs={"input": _IMAGE_RESPONSES_INPUT} + ) + assert deployment["model_info"]["id"] == "vision" + + +def test_router_sends_an_image_to_the_text_only_leg_when_pre_call_checks_drop_the_vision_leg(): + router: Final = litellm.Router( + model_list=[ + {"model_name": "mixed", **_VISION_LEG, "model_info": {"id": "vision", "max_input_tokens": 1}}, + {"model_name": "mixed", **_DECLARED_TEXT_LEG}, + ], + routing_strategy="latency-based-routing", + enable_pre_call_checks=True, + ) + assert router.get_available_deployment(model="mixed", messages=_IMAGE_CHAT)["model_info"]["id"] == "declared-text" + + +def test_router_sends_an_image_to_the_text_only_leg_when_the_vision_leg_is_cooling_down(): + router: Final = _mixed_router(_VISION_LEG, _DECLARED_TEXT_LEG) + router.cooldown_cache.add_deployment_to_cooldown( + model_id="vision", original_exception=Exception("bad key"), exception_status=401, cooldown_time=60 + ) + assert router.get_available_deployment(model="mixed", messages=_IMAGE_CHAT)["model_info"]["id"] == "declared-text" + + +def test_without_declared_text_only_deployments_passes_a_chosen_deployment_through(): + assert litellm.Router._without_declared_text_only_deployments(_DECLARED_TEXT_LEG, "mixed", _IMAGE_CHAT, None) is _DECLARED_TEXT_LEG + + +def test_image_routing_drop_declared_text_only_legs_for_image_requests(): + router: Final = _mixed_router(_VISION_LEG, _DECLARED_TEXT_LEG) + assert _candidate_ids(router, messages=_IMAGE_CHAT) == ["vision"] + + +def test_image_routing_keep_declared_text_only_legs_for_text_requests(): + router: Final = _mixed_router(_VISION_LEG, _DECLARED_TEXT_LEG) + assert _candidate_ids(router, messages=[{"role": "user", "content": "Say hi"}]) == [ "vision", "declared-text", ] -def test_pre_call_checks_drop_declared_text_only_legs_for_responses_input_image(): - router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG) - assert _pre_call_ids( - router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=None, request_kwargs={"input": _IMAGE_RESPONSES_INPUT} - ) == ["vision"] +def test_image_routing_drop_declared_text_only_legs_for_responses_input_image(): + router: Final = _mixed_router(_VISION_LEG, _DECLARED_TEXT_LEG) + assert _candidate_ids(router, request_kwargs={"input": _IMAGE_RESPONSES_INPUT}) == ["vision"] -def test_pre_call_checks_send_image_to_the_group_when_every_leg_is_declared_text_only(): +def test_image_routing_send_image_to_the_group_when_every_leg_is_declared_text_only(): other_text_leg = {**_DECLARED_TEXT_LEG, "model_info": {"id": "declared-text-2", "supports_vision": False}} - router = _pre_call_router(_DECLARED_TEXT_LEG, other_text_leg) - assert _pre_call_ids(router, (_DECLARED_TEXT_LEG, other_text_leg), messages=_IMAGE_CHAT) == [ + router: Final = _mixed_router(_DECLARED_TEXT_LEG, other_text_leg) + assert _candidate_ids(router, messages=_IMAGE_CHAT) == [ "declared-text", "declared-text-2", ] -def test_pre_call_checks_keep_undeclared_custom_legs_for_image_requests(): +def test_image_routing_keep_undeclared_custom_legs_for_image_requests(): custom_leg = { "litellm_params": {"model": "hosted_vllm/Qwen2.5-VL-7B-Instruct", "api_base": "http://localhost:8000"}, "model_info": {"id": "custom"}, } - router = _pre_call_router(_VISION_LEG, custom_leg) - assert _pre_call_ids(router, (_VISION_LEG, custom_leg), messages=_IMAGE_CHAT) == ["vision", "custom"] + router: Final = _mixed_router(_VISION_LEG, custom_leg) + assert _candidate_ids(router, messages=_IMAGE_CHAT) == ["vision", "custom"] -def test_pre_call_checks_resolve_vision_through_the_cost_map_and_base_model(): +def test_image_routing_resolve_vision_through_the_cost_map_and_base_model(): catalog_text_leg = {"litellm_params": {"model": "bedrock/deepseek.v3.2"}, "model_info": {"id": "catalog-text"}} azure_text_leg = { "litellm_params": {"model": "azure/my-audio-deployment", "api_base": "https://x.openai.azure.com"}, "model_info": {"id": "azure-text", "base_model": "azure/gpt-audio"}, } - router = _pre_call_router(_VISION_LEG, catalog_text_leg, azure_text_leg) - assert _pre_call_ids(router, (_VISION_LEG, catalog_text_leg, azure_text_leg), messages=_IMAGE_CHAT) == ["vision"] + router: Final = _mixed_router(_VISION_LEG, catalog_text_leg, azure_text_leg) + assert _candidate_ids(router, messages=_IMAGE_CHAT) == ["vision"] @pytest.mark.parametrize( From 6268c0a281f05671c98528cb6b59e9c3fd2a06f0 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:15:10 -0700 Subject: [PATCH 030/155] feat(proxy): add disable_pre_routing_error_logs to skip spend rows for pre-routing rejections (#48239) * feat(proxy): add disable_pre_routing_error_logs to skip spend rows for pre-routing rejections Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): decide pre-routing rejections by pipeline stage, not exception type Guardrails such as AIM raise ProxyException(400), the same shape as an unparsable body, so the status code list skipped their spend rows. The pipeline stamps litellm_call_id only after auth, body parsing and model resolution, so a failure reaching the hook without one died before any provider was considered. _handle_logging_proxy_only_error flags those requests when it mints the call id, and is_pre_routing_rejection keys off that flag plus ProxyModelNotFoundError. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): mark pre-routing rejections with a server-owned sentinel request_data is the client body, so a boolean marker or a litellm_call_id in it can be sent by the client to hide a guardrail block or force a spend row. The failure handler now stores an object() sentinel whenever it finds no logging object, and is_pre_routing_rejection checks identity, which JSON cannot forge. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: joshua Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/hooks/proxy_track_cost_callback.py | 40 +++++- litellm/proxy/utils.py | 8 ++ .../hooks/test_proxy_track_cost_callback.py | 115 +++++++++++++++++- .../test_post_call_failure_hook.py | 59 ++++++++- 4 files changed, 211 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 6801c8521e5..c11b59a809a 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -38,6 +38,7 @@ from litellm.proxy.db.db_spend_update_writer import ( ) from litellm.proxy.hooks.search_free_tier import apply_search_free_tier from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_route +from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.spend_tracking.spend_counter_batch import post_call_counter_keys, spend_counter_batch_scope from litellm.proxy.spend_tracking.spend_event import ( ObjectMapping, @@ -60,7 +61,12 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 # sanitize_error_information_for_spend_logs, should_store_prompts_and_responses_in_spend_logs, ) -from litellm.proxy.utils import BilledFailedAttempt, ProxyUpdateSpend +from litellm.proxy.utils import ( + REJECTED_BEFORE_REQUEST_SETUP, + REJECTED_BEFORE_REQUEST_SETUP_KEY, + BilledFailedAttempt, + ProxyUpdateSpend, +) from litellm.types.router import Deployment from litellm.types.utils import ( CallTypes, @@ -396,6 +402,8 @@ class _ProxyDBLogger(CustomLogger): request_route: Final = user_api_key_dict.request_route if ( ProxyDBLogger._should_track_errors_in_db() is False + or ProxyDBLogger._skips_pre_routing_rejections() + and is_pre_routing_rejection(original_exception, _REQUEST_DATA.validate_python(request_data)) or request_route is not None and not ( RouteChecks.is_llm_api_route(route=request_route) or RouteChecks.is_info_route(route=request_route) @@ -862,16 +870,40 @@ class _ProxyDBLogger(CustomLogger): If users want to disable error tracking, they can set the disable_error_logs flag in the general_settings """ - from litellm.proxy.proxy_server import general_settings - - if general_settings.get("disable_error_logs") is True: + if _general_setting_enabled("disable_error_logs"): return False return + @staticmethod + def _skips_pre_routing_rejections() -> bool: + """``general_settings.disable_pre_routing_error_logs`` keeps requests the proxy rejected before + selecting a deployment out of the spend logs, so a client retrying a bad key or an exhausted + budget cannot flood ``LiteLLM_SpendLogs`` with zero-spend rows.""" + return _general_setting_enabled("disable_pre_routing_error_logs") + + +def _general_setting_enabled(name: str) -> bool: + from litellm.proxy.proxy_server import general_settings + + return name in general_settings and general_settings[name] is True + ProxyDBLogger: Final = _ProxyDBLogger +_REQUEST_DATA: Final = TypeAdapter(dict[str, object]) + + +def is_pre_routing_rejection(exception: Exception, request_data: Mapping[str, object]) -> bool: + """The pipeline sets up the logging object only after auth, body parsing and model resolution succeed, and + before guardrails or routing run. ``ProxyLogging._handle_logging_proxy_only_error`` marks a failure that + arrives without one with the ``REJECTED_BEFORE_REQUEST_SETUP`` sentinel, so that request never had a provider + considered. An unknown model is the one later rejection that still never reached a provider.""" + return request_data.get(REJECTED_BEFORE_REQUEST_SETUP_KEY) is REJECTED_BEFORE_REQUEST_SETUP or isinstance( + exception, ProxyModelNotFoundError + ) + + def _write_spend_metadata_to_kwargs(kwargs: dict, metadata: dict) -> None: patch = {k: v for k, v in metadata.items() if (k.startswith("user_api_key") or k == "tags") and v is not None} if not patch: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 8212e7cbe74..ce7e6afde24 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1476,6 +1476,13 @@ def _call_type_for_route(route: str | None) -> str | None: return call_types[0].value if len(operations) == 1 else None +REJECTED_BEFORE_REQUEST_SETUP_KEY: Final = "litellm_rejected_before_request_setup" +REJECTED_BEFORE_REQUEST_SETUP: Final = object() +"""Stored under ``REJECTED_BEFORE_REQUEST_SETUP_KEY`` on ``request_data`` when the failure handler finds no logging +object, meaning the request died (auth, body parsing, model resolution) before the pipeline set one up. A sentinel +object rather than ``True`` because ``request_data`` is the client body: JSON cannot forge this identity, so a +client can neither hide a guardrail block nor force a row by sending the key or a ``litellm_call_id`` itself.""" + _PROXY_ONLY_LLM_API_ERRORS: Final = (HTTPException, ProxyException, GuardrailRaisedException) _LOG_DB_METRICS_CALL_TYPES: Final = frozenset(("get_data", "insert_data", "update_data", "delete_data")) @@ -3888,6 +3895,7 @@ class ProxyLogging: **request_data, ) request_data["litellm_logging_obj"] = litellm_logging_obj # rebind-ok: lifted then popped by the caller + request_data.update({REJECTED_BEFORE_REQUEST_SETUP_KEY: REJECTED_BEFORE_REQUEST_SETUP}) if "metadata" not in request_data: request_data["metadata"] = {} request_data["metadata"].update(user_api_key_logged_metadata) diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 76ad1e08e28..79193d37b04 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -1,35 +1,42 @@ import asyncio import json import logging +from collections.abc import Mapping from datetime import datetime, timedelta, timezone from types import SimpleNamespace -from typing import Final +from typing import Final, cast from unittest.mock import AsyncMock, MagicMock, create_autospec, patch import httpx import pytest +from fastapi import HTTPException import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY -from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth +from litellm.proxy._types import ProxyException, SpendLogsPayload, UserAPIKeyAuth from litellm.proxy.collector import SpendEventConsumer from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter from litellm.proxy.db.model_usage_rollup import build_model_usage_transaction from litellm.proxy.db.spend_log_tool_index import response_tool_call_names from litellm.proxy.hooks.proxy_track_cost_callback import ( - _get_budget_reservation_from_metadata, ProxyDBLogger, + _get_budget_reservation_from_metadata, _should_track_cost_callback, _update_database_and_spend_counters, + is_pre_routing_rejection, run_spend_event, ) from litellm.proxy.route_llm_request import ProxyModelNotFoundError -from litellm.proxy.utils import ProxyUpdateSpend from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_event from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload +from litellm.proxy.utils import ( + REJECTED_BEFORE_REQUEST_SETUP, + REJECTED_BEFORE_REQUEST_SETUP_KEY, + ProxyUpdateSpend, +) from litellm.types.router import Deployment, LiteLLM_Params from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, PromptTokensDetailsWrapper, Usage @@ -3559,3 +3566,103 @@ async def test_batch_accounting_does_not_retry_after_a_search_allowance_update(m assert sum(counter.totals.values()) == 5 assert outcome.failed is True assert outcome.retryable_failure is False + + +_ROUTED_REQUEST: Final[Mapping[str, object]] = { + "model": "anthropic-haiku-4-5", + "messages": [{"role": "user", "content": "hi"}], + "proxy_server_request": {"request_id": "req-1"}, + "litellm_call_id": "call-1", +} +_PRE_ROUTING_REQUEST: Final[Mapping[str, object]] = { + **_ROUTED_REQUEST, + REJECTED_BEFORE_REQUEST_SETUP_KEY: REJECTED_BEFORE_REQUEST_SETUP, +} +_SPOOFED_REQUEST: Final[Mapping[str, object]] = {**_ROUTED_REQUEST, REJECTED_BEFORE_REQUEST_SETUP_KEY: True} +_INVALID_REQUEST_400: Final = ProxyException( + message="Violated guardrail policy", type="invalid_request_error", param=None, code=400 +) +_PROVIDER_AUTH_ERROR: Final = litellm.AuthenticationError( + message="bad key", llm_provider="anthropic", model="claude-haiku-4-5" +) + + +@pytest.mark.parametrize( + ("exception", "request_data", "expected"), + [ + (_INVALID_REQUEST_400, _PRE_ROUTING_REQUEST, True), + (litellm.BudgetExceededError(current_cost=1.0, max_budget=0.0), _PRE_ROUTING_REQUEST, True), + (ProxyModelNotFoundError(route="/chat/completions", model_name="4.6"), _ROUTED_REQUEST, True), + (_INVALID_REQUEST_400, _ROUTED_REQUEST, False), + (_INVALID_REQUEST_400, _SPOOFED_REQUEST, False), + (_PROVIDER_AUTH_ERROR, _ROUTED_REQUEST, False), + ( + litellm.RateLimitError(message="slow down", llm_provider="anthropic", model="claude-haiku-4-5"), + _ROUTED_REQUEST, + False, + ), + (HTTPException(status_code=429, detail="Max parallel request limit reached"), _ROUTED_REQUEST, False), + ], + ids=[ + "bad-body", + "budget", + "unknown-model", + "guardrail-block", + "guardrail-block-with-client-spoofed-marker", + "provider-auth", + "provider-429", + "proxy-429", + ], +) +def test_is_pre_routing_rejection_is_decided_by_pipeline_stage_not_exception_type( + exception: Exception, request_data: Mapping[str, object], expected: bool +) -> None: + assert is_pre_routing_rejection(exception, request_data) is expected + + +class _RecordingSpendWriter: + def __init__(self) -> None: + self.rows: list[Mapping[str, object]] = [] # mutable-ok: test double records the spend rows written + + async def update_database(self, **kwargs: object) -> None: + self.rows.append(kwargs) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("flag_enabled", "exception", "request_data", "rows_written"), + [ + (True, _INVALID_REQUEST_400, _PRE_ROUTING_REQUEST, 0), + (True, litellm.BudgetExceededError(current_cost=1.0, max_budget=0.0), _PRE_ROUTING_REQUEST, 0), + (True, ProxyModelNotFoundError(route="/chat/completions", model_name="4.6"), _ROUTED_REQUEST, 0), + (True, _INVALID_REQUEST_400, _ROUTED_REQUEST, 1), + (True, _INVALID_REQUEST_400, _SPOOFED_REQUEST, 1), + (True, _PROVIDER_AUTH_ERROR, _ROUTED_REQUEST, 1), + (False, _INVALID_REQUEST_400, _PRE_ROUTING_REQUEST, 1), + (False, litellm.BudgetExceededError(current_cost=1.0, max_budget=0.0), _PRE_ROUTING_REQUEST, 1), + ], + ids=[ + "flag-bad-body", + "flag-budget", + "flag-unknown-model", + "flag-guardrail-block-kept", + "flag-guardrail-block-with-spoofed-marker-kept", + "flag-provider-error-kept", + "off-bad-body", + "off-budget", + ], +) +async def test_async_post_call_failure_hook_disable_pre_routing_error_logs( + flag_enabled: bool, exception: Exception, request_data: Mapping[str, object], rows_written: int +) -> None: + writer: Final = _RecordingSpendWriter() + user_api_key_dict: Final = UserAPIKeyAuth( + api_key="test_api_key", user_id="u", team_id="t", request_route="/chat/completions" + ) + with patch("litellm.proxy.proxy_server.general_settings", {"disable_pre_routing_error_logs": flag_enabled}): + await ProxyDBLogger(spend_writer=lambda: cast(DBSpendUpdateWriter, writer)).async_post_call_failure_hook( + request_data={**request_data, "metadata": dict[str, object]()}, + original_exception=exception, + user_api_key_dict=user_api_key_dict, + ) + assert len(writer.rows) == rows_written diff --git a/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py b/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py index e83efa21dcc..6557f0d1446 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py +++ b/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py @@ -18,7 +18,7 @@ from litellm.constants import PROXY_REJECTED_BEFORE_ROUTING_KEY from litellm.exceptions import GuardrailRaisedException from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import ProxyErrorTypes, UserAPIKeyAuth -from litellm.proxy.utils import ProxyLogging +from litellm.proxy.utils import REJECTED_BEFORE_REQUEST_SETUP, REJECTED_BEFORE_REQUEST_SETUP_KEY, ProxyLogging from litellm.types.utils import CachingDetails @@ -194,7 +194,9 @@ async def test_post_call_failure_hook_attributes_single_router_deployment( @pytest.mark.asyncio -async def test_pre_routing_reject_spend_log_keeps_public_model_group(proxy_logging, make_user_api_key_auth, monkeypatch): +async def test_pre_routing_reject_spend_log_keeps_public_model_group( + proxy_logging, make_user_api_key_auth, monkeypatch +): from litellm.proxy import proxy_server from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload @@ -1001,7 +1003,9 @@ async def test_post_call_failure_hook_guardrail_block_fires_failure_callback( self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime ) -> None: standard_logging_object = kwargs.get("standard_logging_object") - recorded.append(standard_logging_object.get("status") if isinstance(standard_logging_object, dict) else None) + recorded.append( + standard_logging_object.get("status") if isinstance(standard_logging_object, dict) else None + ) monkeypatch.setattr(litellm, "_async_failure_callback", [_StatusRecorder()]) logging_obj = LiteLLMLoggingObj( @@ -1029,3 +1033,52 @@ async def test_post_call_failure_hook_guardrail_block_fires_failure_callback( await asyncio.sleep(0) await asyncio.sleep(0) assert recorded == ["failure"] + + +@pytest.mark.asyncio +async def test_handle_logging_proxy_only_path_flags_requests_that_died_before_request_setup( + proxy_logging, make_user_api_key_auth +): + """A reject before the pipeline set up logging (bad key, bad body) reaches the failure hook with no logging + object. The handler mints a ``litellm_call_id`` so error logs can carry it, and marks the request with a + server-owned sentinel so spend tracking can tell it apart from a guardrail block, which arrives with the + pipeline's logging object. ``request_data`` is the client body, so a client-sent call id or marker value must + not change either outcome.""" + rejected_before_setup: dict[str, object] = { + "model": "m", + "messages": [{"role": "user", "content": "x"}], + "metadata": {}, + "litellm_call_id": "client-supplied", + } + await proxy_logging._handle_logging_proxy_only_error( + request_data=rejected_before_setup, + user_api_key_dict=make_user_api_key_auth(), + route="/chat/completions", + original_exception=HTTPException(status_code=401, detail="bad key"), + ) + + logging_obj = MagicMock() + logging_obj.call_type = "acompletion" + logging_obj.model_call_details = {} + logging_obj.async_failure_handler = AsyncMock() + blocked_by_guardrail: dict[str, object] = { + "litellm_logging_obj": logging_obj, + "litellm_call_id": "call-1", + "model": "m", + "messages": [{"role": "user", "content": "x"}], + "metadata": {}, + REJECTED_BEFORE_REQUEST_SETUP_KEY: True, + } + await proxy_logging._handle_logging_proxy_only_error( + request_data=blocked_by_guardrail, + user_api_key_dict=make_user_api_key_auth(), + route="/chat/completions", + original_exception=HTTPException(status_code=400, detail="blocked"), + ) + + assert ( + rejected_before_setup.get(REJECTED_BEFORE_REQUEST_SETUP_KEY) is REJECTED_BEFORE_REQUEST_SETUP, + rejected_before_setup["litellm_call_id"], + blocked_by_guardrail.get(REJECTED_BEFORE_REQUEST_SETUP_KEY) is REJECTED_BEFORE_REQUEST_SETUP, + blocked_by_guardrail["litellm_call_id"], + ) == (True, "client-supplied", False, "call-1") From 9b7f2c8dead2a2ea5a0b6b4afbcf065d33feb8bd Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sat, 24 Oct 2026 16:44:29 -0700 Subject: [PATCH 031/155] fix(opencode): bill OpenCode Go requests from the reported list_cost (#48254) * fix(opencode): bill OpenCode Go requests from the reported list_cost OpenCode returns usage.opencode.list_cost on every response. Read it into the provider-reported cost the cost calculator checks before the price map, for chat and Responses, streamed and not. Keep the static rows as the fallback, and stop the generic usage.cost path from billing Go requests at 0 (usage.cost is what hit the prepaid balance, 0 inside the plan). Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(core): resolve reported cost breakdowns without recursion Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(opencode): cover reported list_cost spend through the proxy Replaces the recursive _reported_cost_value with a flat _cost_number helper so the code-quality gate passes, and adds an integration test that runs the proxy against a scripted OpenCode upstream for a non-stream and a stream chat completion, asserting the response cost header and the spend log row Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(opencode): bill native /v1/messages from the reported list_cost The native Anthropic Messages route priced OpenCode Go bodies from the price rows twice over: the non-stream body is a TypedDict the response-cost header path priced as-is, and the stream assembled its ModelResponse with the rows cost already stamped. Both now read usage.opencode.list_cost, the non-stream body through billable_result and the stream from the final message_delta. The integration test now covers chat, responses and messages, stream and non-stream, through the proxy against a scripted OpenCode upstream Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(opencode): keep the stream cost stamp inside the lint gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(opencode): read list_cost off dict chunks and trust it only from OpenCode on /v1/messages Stream builders receive chunks both as mappings and as ModelResponseStream objects, so the list_cost scan now reads usage through chunk_usage. The native Messages route only trusts a reported cost from opencode_go, every other upstream keeps the price map, which also keeps a zero usage.cost from being billed as free. Served bodies without stop_sequence still price, and the new code stays inside the basedpyright gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/core_helpers.py | 65 +++ litellm/litellm_core_utils/litellm_logging.py | 27 +- .../streaming_chunk_builder_utils.py | 40 +- .../litellm_core_utils/streaming_handler.py | 23 +- litellm/llms/opencode/chat/transformation.py | 57 ++- litellm/llms/opencode/common_utils.py | 11 + .../llms/opencode/responses/transformation.py | 40 +- .../anthropic_passthrough_logging_handler.py | 56 ++- .../test_opencode_go_reported_cost_wire.py | 198 ++++++++ .../test_litellm_logging.py | 276 +++++++----- .../llms/opencode/chat/test_transformation.py | 180 ++++++++ ...t_anthropic_passthrough_logging_handler.py | 422 ++++++++---------- 12 files changed, 1024 insertions(+), 371 deletions(-) create mode 100644 tests/integration/pricing/test_opencode_go_reported_cost_wire.py diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 4be5cabac04..00c6ce97e78 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -928,6 +928,71 @@ class _CarriesHiddenParams(Protocol): def hidden_params(self, hidden_params: dict[str, object]) -> None: ... # mutable-ok: API requires mutation +_USAGE_BLOCK_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + + +def _usage_field(usage: object, name: str) -> object: + if isinstance(usage, Mapping): + return _USAGE_BLOCK_ADAPTER.validate_python(usage).get(name) + return getattr(usage, name, None) + + +def _cost_number(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, int | float): + return None + return float(value) + + +def _reported_cost_value(value: object) -> float | None: + """A reported cost is a number, or a breakdown whose total lives under ``total_cost``.""" + raw: Final[object] = value + if isinstance(value, Mapping): + return _cost_number(_usage_field(raw, "total_cost")) + return _cost_number(value) + + +def response_cost_in_hidden_params(response: _CarriesHiddenParams) -> float | None: + additional_headers: Final[object] = response.hidden_params.get("additional_headers") + if not isinstance(additional_headers, Mapping): + return None + return _cost_number(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER)) + + +def override_stamped_response_cost(response: _CarriesHiddenParams, cost: float) -> None: + """Replace the cost an earlier builder stamped so every reader of the hidden params agrees with the header.""" + set_response_cost_in_hidden_params(response, cost) + hidden_params: Final = response.hidden_params + hidden_params["response_cost"] = cost + + +def chunk_usage(chunk: object) -> object: + """Stream chunks reach the builders both as mappings and as ModelResponseStream objects.""" + if isinstance(chunk, Mapping): + return _USAGE_BLOCK_ADAPTER.validate_python(chunk).get("usage") + return getattr(chunk, "usage", None) + + +_MESSAGES_REPORTED_COST_PROVIDERS: Final = frozenset({"opencode_go"}) + + +def native_messages_reported_cost(usage: object, custom_llm_provider: object) -> float | None: + """Only a Messages upstream known to price its own responses is trusted, every other one keeps the price map.""" + if not isinstance(custom_llm_provider, str) or custom_llm_provider not in _MESSAGES_REPORTED_COST_PROVIDERS: + return None + return provider_reported_usage_cost(usage, custom_llm_provider) + + +def provider_reported_usage_cost(usage: object, custom_llm_provider: str | None) -> float | None: + """The request cost a provider states on its usage block, as a number or a breakdown with ``total_cost``. + + OpenCode Go reports the model's list price (peak hours and long-context tiers included) under + ``usage.opencode.list_cost``, while its ``usage.cost`` is what hit the prepaid balance, 0 inside the plan. + """ + if custom_llm_provider == "opencode_go": + return _reported_cost_value(_usage_field(_usage_field(usage, "opencode"), "list_cost")) + return _reported_cost_value(_usage_field(usage, "cost")) + + def set_response_cost_in_hidden_params(response: _CarriesHiddenParams, cost: float | None) -> None: """Record a provider-reported cost where the cost calculator looks before the price map.""" if cost is None: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index fc8d1ecc8c1..9e0461e951d 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -77,6 +77,8 @@ from litellm.litellm_core_utils.classifier_logging import ( from litellm.litellm_core_utils.core_helpers import ( get_provider_response_headers_from_hidden_params, is_expected_client_error, + native_messages_reported_cost, + provider_reported_usage_cost, proxy_stamped_kubernetes_pod_routing, proxy_stamped_used_client_oauth_token, reconstruct_model_name, @@ -4756,7 +4758,12 @@ class Logging(LiteLLMLoggingBaseClass): ): ## return unified Usage object if isinstance(result.response.usage, ResponseAPIUsage): - set_response_cost_in_hidden_params(result.response, result.response.usage.cost) + set_response_cost_in_hidden_params( + result.response, + provider_reported_usage_cost( + result.response.usage, self.model_call_details.get("custom_llm_provider") + ), + ) transformed_usage: Final = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage( result.response.usage ) @@ -4822,8 +4829,11 @@ class Logging(LiteLLMLoggingBaseClass): bridge is replaced by the ModelResponse its spend row is built from, anything else is priced as is.""" served: Final[object] = result - if self.bridged_model_response is None or not isinstance(result, dict): - return result + call_type: Final[object] = getattr(self, "call_type", None) + if not isinstance(result, dict): + return served + if self.bridged_model_response is None and call_type != CallTypes.anthropic_messages.value: + return served return self._anthropic_messages_logged_response(result=served) def _anthropic_messages_logged_response(self, result: object) -> ModelResponse: @@ -4866,7 +4876,13 @@ class Logging(LiteLLMLoggingBaseClass): if isinstance(result, ResponsesAPIResponse): return self._translate_responses_api_response_to_model_response(result) - provider_response_id: Final = _provider_response_id(result) + raw_body: Final[object] = result + served_body: Final = _LITELLM_PARAMS.validate_python(raw_body) if isinstance(result, dict) else None + reported_cost: Final = native_messages_reported_cost( + served_body.get("usage") if served_body is not None else None, + self.model_call_details.get("custom_llm_provider"), + ) + provider_response_id: Final = _provider_response_id(raw_body) bridged: Final = self.bridged_model_response if bridged is not None and isinstance(result, dict): return bridged if provider_response_id is None else bridged.model_copy(update={"id": provider_response_id}) @@ -4891,7 +4907,7 @@ class Logging(LiteLLMLoggingBaseClass): else: from litellm.types.llms.anthropic import AnthropicResponse - pydantic_result: Final = AnthropicResponse.model_validate(result) + pydantic_result: Final = AnthropicResponse.model_validate({"stop_sequence": None, **(served_body or {})}) import httpx result = litellm.AnthropicConfig().transform_parsed_response( @@ -4904,6 +4920,7 @@ class Logging(LiteLLMLoggingBaseClass): json_mode=None, speed=self.optional_params.get("speed") if self.optional_params else None, ) + set_response_cost_in_hidden_params(result, reported_cost) return result def _translate_responses_api_response_to_model_response(self, result: ResponsesAPIResponse) -> ModelResponse: diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 50789406055..68cdc204af4 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict @@ -1352,14 +1353,27 @@ def stream_chunk_builder_text_completion( _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) -def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool: +def _logged_provider(logging_obj: Optional["Logging"]) -> object: if logging_obj is None: - return False + return None provider: Final[object] = logging_obj.model_call_details.get("custom_llm_provider") - return provider in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS + return provider -def _stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> float | None: +def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool: + return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS + + +def _stream_builder_response_cost( + response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object] +) -> float | None: + if _logged_provider(logging_obj) == LlmProviders.OPENCODE_GO.value: + # usage.opencode only survives on the chunks, the assembled Usage keeps declared fields + reported: Final = ( + provider_reported_usage_cost(chunk_usage(chunk), LlmProviders.OPENCODE_GO.value) + for chunk in reversed(chunks) + ) + return next((cost for cost in reported if cost is not None), None) usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None) if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj): return float(usage_cost) @@ -1397,8 +1411,10 @@ def _set_hidden_response_cost(response: ModelResponse, response_cost: float) -> hidden_params["response_cost"] = response_cost -def _set_stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> None: - response_cost: Final = _stream_builder_response_cost(response, logging_obj) +def _set_stream_builder_response_cost( + response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object] +) -> None: + response_cost: Final = _stream_builder_response_cost(response, logging_obj, chunks) if response_cost is None: return _set_hidden_response_cost(response, response_cost) @@ -1550,7 +1566,11 @@ def stream_chunk_builder( break _stamp_streaming_usage_cost(usage, response, logging_obj) - _set_stream_builder_response_cost(response, logging_obj) + _set_stream_builder_response_cost( + response, + logging_obj, + chunks, # pyright: ignore[reportUnknownArgumentType] # legacy chunk list is untyped + ) processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj) return response @@ -1734,7 +1754,11 @@ def stream_chunk_builder( break _stamp_streaming_usage_cost(usage, response, logging_obj) - _set_stream_builder_response_cost(response, logging_obj) + _set_stream_builder_response_cost( + response, + logging_obj, + chunks, # pyright: ignore[reportUnknownArgumentType] # legacy chunk list is untyped + ) processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj) return response diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index fdcdca151be..2df470edc59 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -24,6 +24,7 @@ import litellm from litellm import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost from litellm.litellm_core_utils.hidden_params import ( HIDDEN_PARAMS_ATTR, get_hidden_params, @@ -72,7 +73,9 @@ _SYNC_ITER_EXHAUSTED: Final = object() _GCHUNK_FIELDS: Final[frozenset] = frozenset(GChunk.__annotations__) _CompleteStreamingResponse = ModelResponse | TextCompletionResponse | ModelResponseStream | None -_USAGE_COST_HEADER_PROVIDERS: Final[frozenset[str]] = frozenset({LlmProviders.OPENROUTER.value}) +_USAGE_COST_HEADER_PROVIDERS: Final[frozenset[str]] = frozenset( + {LlmProviders.OPENROUTER.value, LlmProviders.OPENCODE_GO.value} +) def _next_sync_or_exhausted(it: Iterator[object]) -> object: @@ -1953,11 +1956,17 @@ class CustomStreamWrapper: def _propagate_usage_cost_to_hidden_params( response: "ModelResponse", custom_llm_provider: str | None, + chunks: Sequence[object] = (), ) -> None: + """The builder's usage only keeps declared fields, so provider blocks like usage.opencode are read off the chunks.""" if custom_llm_provider not in _USAGE_COST_HEADER_PROVIDERS: return - _usage: Final[Usage | None] = getattr(response, "usage", None) - _cost: Final = CustomStreamWrapper._resolve_provider_reported_cost(getattr(_usage, "cost", None)) + usages: Final = ( + getattr(response, "usage", None), + *(chunk_usage(chunk) for chunk in reversed(chunks)), + ) + costs: Final = (provider_reported_usage_cost(usage, custom_llm_provider) for usage in usages) + _cost: Final = next((cost for cost in costs if cost is not None), None) if _cost is not None: if "additional_headers" not in response.hidden_params: response.hidden_params["additional_headers"] = {} @@ -2099,7 +2108,9 @@ class CustomStreamWrapper: complete_streaming_response: Final = self._build_complete_streaming_response() response = self._new_usage_chunk() if complete_streaming_response is not None: - self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider) + self._propagate_usage_cost_to_hidden_params( + complete_streaming_response, self.custom_llm_provider, self.chunks + ) setattr( response, @@ -2318,7 +2329,9 @@ class CustomStreamWrapper: complete_streaming_response: Final = await self._async_build_complete_streaming_response() response: Final = self._new_usage_chunk() if complete_streaming_response is not None: - self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider) + self._propagate_usage_cost_to_hidden_params( + complete_streaming_response, self.custom_llm_provider, self.chunks + ) setattr( response, diff --git a/litellm/llms/opencode/chat/transformation.py b/litellm/llms/opencode/chat/transformation.py index 40b99dbdd76..dcb30b5835c 100644 --- a/litellm/llms/opencode/chat/transformation.py +++ b/litellm/llms/opencode/chat/transformation.py @@ -1,6 +1,7 @@ from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Final +import httpx from pydantic import TypeAdapter import litellm @@ -9,7 +10,14 @@ from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig -from litellm.llms.opencode.common_utils import go_api_base, go_api_key, go_endpoint, go_headers, go_model_info +from litellm.llms.opencode.common_utils import ( + attach_go_reported_cost, + go_api_base, + go_api_key, + go_endpoint, + go_headers, + go_model_info, +) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse @@ -18,7 +26,46 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging -class OpenCodeGoChatConfig(OpenAILikeChatConfig): +class _GoReportedCostMixin: + """Record usage.opencode.list_cost so response_cost reflects OpenCode's price, not the static rows.""" + + go_usage_key: str = "usage" + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: "Logging", + request_data: Mapping[str, object], + messages: Sequence[AllMessageValues], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + encoding: object, + api_key: str | None = None, + json_mode: bool | None = None, + ) -> ModelResponse: + response: Final = super().transform_response( # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # mixin precedes a BaseConfig subclass + model=model, + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data=dict(request_data), + messages=list(messages), + optional_params=dict(optional_params), + litellm_params=dict(litellm_params), + encoding=encoding, + api_key=api_key, + json_mode=json_mode, + ) + priced: Final = TypeAdapter(ModelResponse).validate_python(response) + attach_go_reported_cost( + priced, TypeAdapter(dict[str, object]).validate_json(raw_response.content).get(self.go_usage_key) + ) + return priced + + +class OpenCodeGoChatConfig(_GoReportedCostMixin, OpenAILikeChatConfig): @property def custom_llm_provider(self) -> str: return "opencode_go" @@ -60,7 +107,7 @@ class OpenCodeGoChatConfig(OpenAILikeChatConfig): return [f"opencode_go/{entry['id']}" for entry in models if isinstance(entry.get("id"), str)] -class OpenCodeGoMessagesChatConfig(AnthropicConfig): +class OpenCodeGoMessagesChatConfig(_GoReportedCostMixin, AnthropicConfig): @property def custom_llm_provider(self) -> str: return "opencode_go" @@ -110,7 +157,9 @@ class OpenCodeGoMessagesChatConfig(AnthropicConfig): return TypeAdapter(dict[str, object]).validate_python(raw) -class OpenCodeGoGeminiChatConfig(GoogleAIStudioGeminiConfig): +class OpenCodeGoGeminiChatConfig(_GoReportedCostMixin, GoogleAIStudioGeminiConfig): + go_usage_key: str = "usageMetadata" + @property def supports_stream_param_in_request_body(self) -> bool: return False diff --git a/litellm/llms/opencode/common_utils.py b/litellm/llms/opencode/common_utils.py index 46e382d8e65..82cd957d23c 100644 --- a/litellm/llms/opencode/common_utils.py +++ b/litellm/llms/opencode/common_utils.py @@ -4,8 +4,11 @@ from typing import Final from pydantic import TypeAdapter +from litellm.litellm_core_utils.core_helpers import provider_reported_usage_cost, set_response_cost_in_hidden_params from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.utils import ModelResponse def is_opencode_api_base(api_base: str | None) -> bool: @@ -67,3 +70,11 @@ def go_headers( params, "x-opencode-session", ) + + +def go_reported_cost(usage: object) -> float | None: + return provider_reported_usage_cost(usage, "opencode_go") + + +def attach_go_reported_cost(response: ModelResponse | ResponsesAPIResponse, usage: object) -> None: + set_response_cost_in_hidden_params(response, go_reported_cost(usage)) diff --git a/litellm/llms/opencode/responses/transformation.py b/litellm/llms/opencode/responses/transformation.py index ca75a319c7e..615ca60b45b 100644 --- a/litellm/llms/opencode/responses/transformation.py +++ b/litellm/llms/opencode/responses/transformation.py @@ -1,11 +1,24 @@ from collections.abc import Mapping +from typing import TYPE_CHECKING, Final + +import httpx from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig -from litellm.llms.opencode.common_utils import go_api_base, go_headers +from litellm.llms.opencode.common_utils import attach_go_reported_cost, go_api_base, go_headers +from litellm.types.llms.openai import ( + ResponseCompletedEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, + ResponsesAPIResponse, + ResponsesAPIStreamingResponse, +) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging + class OpenCodeGoResponsesConfig(OpenAIResponsesAPIConfig): @property @@ -25,3 +38,28 @@ class OpenCodeGoResponsesConfig(OpenAIResponsesAPIConfig): litellm_params: GenericLiteLLMParams | None, ) -> dict[str, object]: # mutable-ok: protocol handlers mutate the returned request and headers return go_headers(headers, litellm_params, litellm_params.api_key if litellm_params else None, model) + + def transform_response_api_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "Logging", + ) -> ResponsesAPIResponse: + response: Final = super().transform_response_api_response( + model=model, raw_response=raw_response, logging_obj=logging_obj + ) + attach_go_reported_cost(response, response.usage) + return response + + def transform_streaming_response( + self, + model: str, + parsed_chunk: Mapping[str, object], + logging_obj: "Logging", + ) -> ResponsesAPIStreamingResponse: + event: Final = super().transform_streaming_response( # pyright: ignore[reportUnknownMemberType] # base takes an untyped dict + model=model, parsed_chunk=dict(parsed_chunk), logging_obj=logging_obj + ) + if isinstance(event, ResponseCompletedEvent | ResponseFailedEvent | ResponseIncompleteEvent): + attach_go_reported_cost(event.response, event.response.usage) + return event diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 23ef89b3c5c..15f70073c7a 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -1,15 +1,20 @@ import asyncio import json -from collections.abc import Mapping, Sequence +from collections.abc import Iterator, Mapping, Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, Final, cast import httpx -from pydantic import ConfigDict, TypeAdapter +from pydantic import ConfigDict, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import ANTHROPIC_BATCHES_ROUTE +from litellm.litellm_core_utils.core_helpers import ( + native_messages_reported_cost, + override_stamped_response_cost, + response_cost_in_hidden_params, +) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -53,6 +58,38 @@ else: _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +def _event_payload(event: str) -> Mapping[str, object] | None: + data_lines: Final = tuple( + line.strip()[len("data:") :].strip() for line in event.splitlines() if line.strip().startswith("data:") + ) + if not data_lines or data_lines[0] in ("", "[DONE]"): + return None + try: + return _JSON_OBJECT.validate_json(data_lines[0]) + except ValidationError: + return None + + +def _message_delta_usages(all_chunks: Sequence[str | bytes]) -> Iterator[object]: + for chunk in reversed(all_chunks): + yield from ( + payload.get("usage") + for payload in map(_event_payload, reversed(split_sse_chunk_into_events(chunk))) + if payload is not None and payload.get("type") == "message_delta" + ) + + +def _stamp_reported_stream_cost( + response: ModelResponse, all_chunks: Sequence[str | bytes], logging_obj: LiteLLMLoggingObj +) -> None: + """The final ``message_delta`` is where a provider that prices its own responses reports the cost.""" + provider: Final = logging_obj.model_call_details.get("custom_llm_provider") + costs: Final = (native_messages_reported_cost(usage, provider) for usage in _message_delta_usages(all_chunks)) + reported: Final = next((cost for cost in costs if cost is not None), None) + if reported is not None: + override_stamped_response_cost(response, reported) + + class AnthropicPassthroughLoggingHandler: @staticmethod def anthropic_passthrough_handler( @@ -222,6 +259,7 @@ class AnthropicPassthroughLoggingHandler: ) if not isinstance(response, ModelResponse): return response + _stamp_reported_stream_cost(response, all_chunks, litellm_logging_obj) recovered_usage: Final = AnthropicPassthroughLoggingHandler._recover_interrupted_stream_output_tokens( response=response, all_chunks=all_chunks, model=model ) @@ -286,13 +324,21 @@ class AnthropicPassthroughLoggingHandler: if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/") else model ) + custom_pricing: Final = use_custom_pricing_for_model( + litellm_params=(logging_obj.litellm_params if hasattr(logging_obj, "litellm_params") else None) + ) + reported_cost: Final = ( + response_cost_in_hidden_params(litellm_model_response) + if isinstance(litellm_model_response, ModelResponse) + else None + ) + if reported_cost is not None and not custom_pricing: + return reported_cost return litellm.completion_cost( completion_response=litellm_model_response, model=model_for_cost, custom_llm_provider=custom_llm_provider, - custom_pricing=use_custom_pricing_for_model( - litellm_params=(logging_obj.litellm_params if hasattr(logging_obj, "litellm_params") else None) - ), + custom_pricing=custom_pricing, router_model_id=logging_obj.get_router_model_id(), ) diff --git a/tests/integration/pricing/test_opencode_go_reported_cost_wire.py b/tests/integration/pricing/test_opencode_go_reported_cost_wire.py new file mode 100644 index 00000000000..6ac3bdccf27 --- /dev/null +++ b/tests/integration/pricing/test_opencode_go_reported_cost_wire.py @@ -0,0 +1,198 @@ +import json +from collections.abc import Mapping +from typing import Final + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_LIST_COST: Final = 0.00412 +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + +# OpenCode's documented usage block (opencode.ai/v2/docs/console/api/inference, "Cost", read 2026-10-24): +# usage.cost is what hit the prepaid balance (0 inside the Go plan), usage.opencode.list_cost is the model price +_GO_BLOCK: Final[Mapping[str, JsonValue]] = { + "cost": 0, + "opencode": {"billing_source": "go", "list_cost": _LIST_COST, "allowance": _LIST_COST}, +} +_CHAT_USAGE: Final[Mapping[str, JsonValue]] = { + "prompt_tokens": 1200, + "completion_tokens": 300, + "total_tokens": 1500, + **_GO_BLOCK, +} +_RESPONSES_USAGE: Final[Mapping[str, JsonValue]] = { + "input_tokens": 1200, + "output_tokens": 300, + "total_tokens": 1500, + **_GO_BLOCK, +} +_MESSAGES_USAGE: Final[Mapping[str, JsonValue]] = {"input_tokens": 1200, "output_tokens": 300, **_GO_BLOCK} + + +def _approx(value: float) -> object: + return pytest.approx(value, rel=1e-6) # pyright: ignore[reportUnknownMemberType] # pytest lacks typed approx stubs + + +def _data_frame(event: Mapping[str, JsonValue]) -> bytes: + return f"data: {json.dumps(event)}\n\n".encode() + + +def _event_frame(event: Mapping[str, JsonValue]) -> bytes: + return f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() + + +def _chat_completion() -> bytes: + return json.dumps( + { + "id": "chatcmpl-go", + "object": "chat.completion", + "created": 1, + "model": "glm-5.3", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "ok"}}], + "usage": dict(_CHAT_USAGE), + } + ).encode() + + +def _chat_stream() -> tuple[bytes, ...]: + text: Final = { + "id": "chatcmpl-go", + "object": "chat.completion.chunk", + "created": 1, + "model": "glm-5.3", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}, "finish_reason": None}], + } + finish: Final = {**text, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]} + final_usage: Final = {**text, "choices": [], "usage": dict(_CHAT_USAGE)} + return (*(_data_frame(event) for event in (text, finish, final_usage)), b"data: [DONE]\n\n") + + +def _responses_response() -> Mapping[str, JsonValue]: + return { + "id": "resp_go", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-6-luna", + "output": [ + { + "id": "msg_go", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": dict(_RESPONSES_USAGE), + } + + +def _responses_stream() -> tuple[bytes, ...]: + response: Final = _responses_response() + return ( + _event_frame( + {"type": "response.created", "sequence_number": 0, "response": {**response, "status": "in_progress"}} + ), + _event_frame( + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_go", + "output_index": 0, + "content_index": 0, + "delta": "ok", + } + ), + _event_frame({"type": "response.completed", "sequence_number": 2, "response": response}), + ) + + +def _anthropic_message() -> Mapping[str, JsonValue]: + return { + "id": "msg_go", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": dict(_MESSAGES_USAGE), + } + + +def _anthropic_stream() -> tuple[bytes, ...]: + start: Final = {**_anthropic_message(), "content": [], "stop_reason": None} + return ( + _event_frame( + {"type": "message_start", "message": {**start, "usage": {"input_tokens": 1200, "output_tokens": 0}}} + ), + _event_frame({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}), + _event_frame({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "ok"}}), + _event_frame({"type": "content_block_stop", "index": 0}), + _event_frame( + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": dict(_MESSAGES_USAGE), + } + ), + _event_frame({"type": "message_stop"}), + ) + + +def _respond(request: Request) -> Reply: + assert request.method == "POST", request.method + streaming: Final = _JSON_OBJECT.validate_json(request.body).get("stream") is True + if request.target.endswith("/chat/completions"): + return ( + Reply(content_type="text/event-stream", chunks=_chat_stream()) + if streaming + else Reply(body=_chat_completion()) + ) + if request.target.endswith("/responses"): + if streaming: + return Reply(content_type="text/event-stream", chunks=_responses_stream()) + return Reply(body=json.dumps(_responses_response()).encode()) + assert request.target.endswith("/v1/messages"), request.target + if streaming: + return Reply(content_type="text/event-stream", chunks=_anthropic_stream()) + return Reply(body=json.dumps(_anthropic_message()).encode()) + + +def _logged_spend(call_id: str) -> float: + rows: Final = eventually( + lambda: read_rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE litellm_call_id = %s', (call_id,)), + lambda values: len(values) == 1, + seconds=70, + ) + return float(rows[0]["spend"]) + + +def _body(path: str, model: str, stream: bool) -> Mapping[str, JsonValue]: + if path == "/v1/responses": + return {"model": model, "input": "hi", "stream": stream} + return {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}], "stream": stream} + + +@pytest.mark.parametrize("stream", (False, True), ids=("non-stream", "stream")) +@pytest.mark.parametrize( + ("path", "model"), + ( + ("/v1/chat/completions", "opencode_go/glm-5.3"), + ("/v1/responses", "opencode_go/gpt-6-luna"), + ("/v1/messages", "opencode_go/claude-haiku-5-5"), + ), + ids=("chat", "responses", "messages"), +) +def test_request_is_billed_at_the_reported_list_cost(gateway: Gateway, path: str, model: str, stream: bool) -> None: + with wire_server(_respond) as wire, gateway.scenario() as scenario: + deployment: Final = scenario.model(model=model, api_base=wire.url, api_key="synthetic-go-key") + response: Final = gateway.request("POST", path, _body(path, deployment, stream)) + assert response.status_code == 200, response.text + assert "ok" in response.text + if not stream: + assert float(response.headers["x-litellm-response-cost"]) == _approx(_LIST_COST) + assert _logged_spend(response.headers["x-litellm-call-id"]) == _approx(_LIST_COST) diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 8509b1d6f7d..561ac9b2b33 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -76,6 +76,7 @@ import inspect from typing import List, Optional from litellm.types.utils import LiteLLMCommonStrings + @pytest.fixture def logging_obj(): return LitellmLogging( @@ -346,8 +347,10 @@ async def test_mcp_native_structured_replacement_must_match_returned_content( ) logging_obj.dynamic_success_callbacks = [NativeReplacement()] returned = await logging_obj.async_post_mcp_tool_call_hook( - kwargs={"original_response": result}, response_obj=result, - start_time=datetime.datetime.now(), end_time=datetime.datetime.now(), + kwargs={"original_response": result}, + response_obj=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), ) assert returned is result assert result.content == [TextContent(type="text", text="native-safe" if same_content else "final-safe")] @@ -356,7 +359,6 @@ async def test_mcp_native_structured_replacement_must_match_returned_content( assert "SECRET-1234" not in result.model_dump_json() - @pytest.mark.asyncio @pytest.mark.parametrize("mode", ["none", "wrapper", "exception"]) @pytest.mark.parametrize("structured", [False, True]) @@ -374,8 +376,10 @@ async def test_mcp_direct_content_edit_invalidates_stale_structured_data(logging ) logging_obj.dynamic_success_callbacks = [DirectRedactor()] returned = await logging_obj.async_post_mcp_tool_call_hook( - kwargs={"original_response": result}, response_obj=result, - start_time=datetime.datetime.now(), end_time=datetime.datetime.now(), + kwargs={"original_response": result}, + response_obj=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), ) assert returned is result assert result.content == [TextContent(type="text", text="[REDACTED]")] @@ -2620,7 +2624,6 @@ async def test_completed_success_selects_one_implemented_hook( assert (thread_id == loop_thread) is (kind != "sync") - @pytest.mark.parametrize("for_async_request,split_registries", [(False, False), (True, False), (True, True)]) @pytest.mark.parametrize("private_first", [False, True]) @pytest.mark.parametrize("response_kind", ["responses", "completion"]) @@ -3312,8 +3315,13 @@ async def test_shadow_snapshot_stays_private_and_is_invalidated_before_logging_g class RecordingShadowLogger(ShadowEvalLogger): async def async_log_success_event( - self, kwargs: Mapping[str, object], response_obj: object, start_time: object, - end_time: object, *, guardrail_snapshot: GuardrailRequestSnapshot | None = None, + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: object, + end_time: object, + *, + guardrail_snapshot: GuardrailRequestSnapshot | None = None, ) -> None: shadow_snapshots.append(guardrail_snapshot) await super().async_log_success_event( @@ -3322,13 +3330,20 @@ async def test_shadow_snapshot_stays_private_and_is_invalidated_before_logging_g class RecordingLogger(CustomLogger): async def async_log_success_event( - self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object, + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: object, + end_time: object, ) -> None: other_payloads.append(kwargs) class LoggingGuardrail(CustomGuardrail): async def async_logging_hook( - self, kwargs: dict[str, object], result: object, call_type: str, + self, + kwargs: dict[str, object], + result: object, + call_type: str, ) -> tuple[dict[str, object], object]: hook_snapshots.append(logging_obj.shadow_eval_request_snapshot) if hook_mode == "raises": @@ -3340,26 +3355,36 @@ async def test_shadow_snapshot_stays_private_and_is_invalidated_before_logging_g "user_api_key_hash": "test-key", } snapshot: Final = GuardrailRequestSnapshot.capture( - {"messages": [{"role": "user", "content": "snapshot-only"}]}, metadata, + {"messages": [{"role": "user", "content": "snapshot-only"}]}, + metadata, ) assert snapshot is not None shadow: Final = RecordingShadowLogger(prisma_provider=no_prisma, jobs_cache=InMemoryCache()) guardrail: Final = LoggingGuardrail( - guardrail_name="late-mask", default_on=True, + guardrail_name="late-mask", + default_on=True, event_hook=GuardrailEventHooks.pre_call if hook_mode == "disabled" else GuardrailEventHooks.logging_only, ) monkeypatch.setattr(litellm, "_async_success_callback", []) logging_obj: Final = LitellmLogging( - model="test-model", messages=[], stream=stream, call_type="anthropic_messages", - start_time=datetime.datetime.now(), litellm_call_id="private-snapshot", function_id="private-snapshot", + model="test-model", + messages=[], + stream=stream, + call_type="anthropic_messages", + start_time=datetime.datetime.now(), + litellm_call_id="private-snapshot", + function_id="private-snapshot", dynamic_async_success_callbacks=[shadow, RecordingLogger(), guardrail], ) logging_obj.update_messages([{"role": "user", "content": "logged input"}]) logging_obj.update_environment_variables(litellm_params={"metadata": metadata}, optional_params={}) logging_obj.shadow_eval_request_snapshot = snapshot payload: Final = { - "id": "private-snapshot", "call_type": "anthropic_messages", "metadata": metadata, - "model_group": "test-model", "model_parameters": {}, + "id": "private-snapshot", + "call_type": "anthropic_messages", + "metadata": metadata, + "model_group": "test-model", + "model_parameters": {}, } await logging_obj.async_success_handler(result=ModelResponse(), standard_logging_object=payload) @@ -3816,7 +3841,9 @@ def test_sentry_event_scrubber_initialization(monkeypatch): call_args = mock_init.call_args[1] assert call_args["send_default_pii"] is False assert call_args["event_scrubber"].recursive is True - assert {name.lower() for name in SENTRY_PII_DENYLIST} <= {name.lower() for name in call_args["event_scrubber"].denylist} + assert {name.lower() for name in SENTRY_PII_DENYLIST} <= { + name.lower() for name in call_args["event_scrubber"].denylist + } assert call_args["before_send"] is call_args["before_send_transaction"] @@ -3833,7 +3860,9 @@ def test_sentry_send_default_pii_opt_in(monkeypatch): call_args = mock_init.call_args[1] assert call_args["send_default_pii"] is True - assert not {name.lower() for name in SENTRY_PII_DENYLIST} & {name.lower() for name in call_args["event_scrubber"].denylist} + assert not {name.lower() for name in SENTRY_PII_DENYLIST} & { + name.lower() for name in call_args["event_scrubber"].denylist + } def test_get_masked_values(): @@ -5103,12 +5132,10 @@ def test_reported_cost_respects_configured_pricing_and_additional_charges( else {} ), } - } - } - ) - logging_obj.model_call_details.update( - {"additional_response_cost": additional_cost, "cache_hit": cache_hit} + }, + }, ) + logging_obj.model_call_details.update({"additional_response_cost": additional_cost, "cache_hit": cache_hit}) logging_obj._provider_reported_cost = reported_cost response: Final = ModelResponse( model=pricing_model, @@ -7055,8 +7082,11 @@ async def test_background_interaction_completion_rebills_after_in_progress_succe class SyncRecorder(CustomLogger): def log_success_event( - self, kwargs: dict[str, object], response_obj: object, - start_time: dt.datetime, end_time: dt.datetime, + self, + kwargs: dict[str, object], + response_obj: object, + start_time: dt.datetime, + end_time: dt.datetime, ) -> None: events.put((response_obj, kwargs.get("response_cost"))) logged.set() @@ -7089,11 +7119,11 @@ async def test_background_interaction_completion_rebills_after_in_progress_succe assert logging_obj.model_call_details["response_cost"] > 0 assert logging_obj.model_call_details["standard_logging_object"]["total_tokens"] == 175 - assert events.qsize() == 2 assert events.get_nowait() == (in_progress, None) assert events.get_nowait() == (completed, logging_obj.model_call_details["response_cost"]) + @pytest.mark.asyncio async def test_background_interaction_completion_prices_the_settled_body_itself(): """ @@ -8127,7 +8157,9 @@ def test_debugging_log_with_json_logs_tolerates_missing_headers(logging_obj, mon logging_obj.litellm_request_debug = True with patch("litellm.litellm_core_utils.litellm_logging.verbose_logger.warning") as warning: - logging_obj._print_llm_call_debugging_log(api_base="https://api.openai.com/v1", headers=None, additional_args={}) + logging_obj._print_llm_call_debugging_log( + api_base="https://api.openai.com/v1", headers=None, additional_args={} + ) assert "https://api.openai.com/v1" in warning.call_args.kwargs["extra"]["api_base"] @@ -10378,8 +10410,13 @@ class TestAzurePTUSpilloverCost: assert payload is not None model_map_value: Final = payload["model_map_information"]["model_map_value"] assert model_map_value is not None - assert model_map_value["input_cost_per_token"] == litellm.model_cost[self.SERVED_MODEL]["input_cost_per_token"] - assert model_map_value["output_cost_per_token"] == litellm.model_cost[self.SERVED_MODEL]["output_cost_per_token"] + assert ( + model_map_value["input_cost_per_token"] == litellm.model_cost[self.SERVED_MODEL]["input_cost_per_token"] + ) + assert ( + model_map_value["output_cost_per_token"] + == litellm.model_cost[self.SERVED_MODEL]["output_cost_per_token"] + ) assert response_cost == pytest.approx( 100 * model_map_value["input_cost_per_token"] + 50 * model_map_value["output_cost_per_token"] ) @@ -10494,8 +10531,6 @@ def test_get_assembled_streaming_response_bills_a_provider_reported_usage_cost() assert logging_obj.response_cost_calculator(result=assembled) == 0.0042 - - def test_response_cost_calculator_prices_terminal_responses_event_from_its_response(): logging_obj: Final = _responses_stream_logging_obj() inner_response: Final = ResponsesAPIResponse( @@ -10725,9 +10760,7 @@ def test_standard_logging_retries(): ] ) - with patch.object( - customHandler, "log_failure_event", new=MagicMock() - ) as mock_client: + with patch.object(customHandler, "log_failure_event", new=MagicMock()) as mock_client: try: router.completion( model="gpt-3.5-turbo", @@ -10739,19 +10772,10 @@ def test_standard_logging_retries(): pass assert mock_client.call_count == 2 + assert mock_client.call_args_list[0].kwargs["kwargs"]["standard_logging_object"]["trace_id"] is not None assert ( - mock_client.call_args_list[0].kwargs["kwargs"]["standard_logging_object"][ - "trace_id" - ] - is not None - ) - assert ( - mock_client.call_args_list[0].kwargs["kwargs"]["standard_logging_object"][ - "trace_id" - ] - == mock_client.call_args_list[1].kwargs["kwargs"][ - "standard_logging_object" - ]["trace_id"] + mock_client.call_args_list[0].kwargs["kwargs"]["standard_logging_object"]["trace_id"] + == mock_client.call_args_list[1].kwargs["kwargs"]["standard_logging_object"]["trace_id"] ) @@ -11018,9 +11042,7 @@ def test_signoz_dispatch_requires_an_endpoint(monkeypatch): custom_logger_init_args={}, ) assert created is None - assert not [ - cb for cb in logging_module._in_memory_loggers if getattr(cb, "callback_name", None) == "signoz" - ] + assert not [cb for cb in logging_module._in_memory_loggers if getattr(cb, "callback_name", None) == "signoz"] finally: logging_module._in_memory_loggers.clear() monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) @@ -11162,7 +11184,11 @@ def test_standard_logging_payload_reports_the_deployment_entry_the_request_was_b } } logging_obj.update_environment_variables( - model="gpt-5.4-nano", user="", optional_params={}, litellm_params=litellm_params, custom_llm_provider="openai" + model="gpt-5.4-nano", + user="", + optional_params={}, + litellm_params=litellm_params, + custom_llm_provider="openai", ) response: Final = ModelResponse( model="gpt-5.4-nano", @@ -11474,10 +11500,7 @@ def test_enterprise_alerting_loggers_falls_back_without_disabling_callback_contr ) logging_module._enterprise_alerting_loggers.cache_clear() for module_name in tuple(sys.modules): - if any( - module_name == prefix or module_name.startswith(f"{prefix}.") - for prefix in module_prefixes - ): + if any(module_name == prefix or module_name.startswith(f"{prefix}.") for prefix in module_prefixes): monkeypatch.delitem(sys.modules, module_name, raising=False) monkeypatch.setattr(sys, "meta_path", [BlockPagerDutyFinder(), *sys.meta_path]) @@ -11597,13 +11620,16 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest_asyncio.fixture(loop_scope="function") async def drain_logging_worker(isolate_litellm_state: None) -> AsyncIterator[None]: yield await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS) + LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS: Final = LOGGING_WORKER_MAX_TIME_PER_COROUTINE + 5.0 + @pytest.fixture(scope="function") def isolate_litellm_state(): """ @@ -11641,6 +11667,7 @@ def isolate_litellm_state(): if attr in _DEFAULTS: setattr(litellm, attr, _DEFAULTS[attr]) + _LIST_ATTRS = ( "callbacks", "success_callback", @@ -11668,6 +11695,7 @@ _SCALAR_ATTRS = ( _DEFAULTS: dict = {} + @pytest.fixture(scope="module") def setup_and_teardown(): """ @@ -11690,6 +11718,7 @@ def setup_and_teardown(): litellm.in_memory_llm_clients_cache.flush_cache() yield + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.parametrize( "response_obj,expected_values", @@ -11737,6 +11766,7 @@ def test_get_usage(response_obj, expected_values): assert usage.completion_tokens == expected_values[1] assert usage.total_tokens == expected_values[2] + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_usage_from_image_generation_response(): """ @@ -11784,6 +11814,7 @@ def test_get_usage_from_image_generation_response(): assert usage.completion_tokens_details.image_tokens == 272 assert usage.completion_tokens_details.text_tokens == 100 + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_additional_headers(): additional_headers = { @@ -11824,10 +11855,12 @@ def test_get_additional_headers(): assert additional_logging_headers.get("llm_provider-request-id") == "req_01F6CycZZPSHKRCCctcS1Vto" assert additional_logging_headers.get("llm_provider-anthropic-ratelimit-requests-reset") == "2024-10-29T23:57:40Z" + def all_fields_present(standard_logging_metadata: StandardLoggingMetadata): for field in StandardLoggingMetadata.__annotations__.keys(): assert field in standard_logging_metadata + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.parametrize( "metadata_key, metadata_value", @@ -11859,6 +11892,7 @@ def test_get_standard_logging_metadata(metadata_key, metadata_value): # Assert that the specific metadata field is set correctly assert standard_logging_metadata[metadata_key] == metadata_value + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_standard_logging_metadata_user_api_key_hash(): valid_hash = "a" * 64 # 64 character string @@ -11866,6 +11900,7 @@ def test_get_standard_logging_metadata_user_api_key_hash(): result = StandardLoggingPayloadSetup.get_standard_logging_metadata(metadata) assert result["user_api_key_hash"] == valid_hash + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_standard_logging_metadata_invalid_user_api_key(): invalid_hash = "not_a_valid_hash" @@ -11874,6 +11909,7 @@ def test_get_standard_logging_metadata_invalid_user_api_key(): all_fields_present(result) assert result["user_api_key_hash"] is None + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_standard_logging_metadata_non_string_user_api_key(): """Non-string user_api_key should not be set as user_api_key_hash.""" @@ -11882,6 +11918,7 @@ def test_get_standard_logging_metadata_non_string_user_api_key(): all_fields_present(result) assert result["user_api_key_hash"] is None + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_standard_logging_metadata_none_user_api_key(): """None user_api_key should not be set as user_api_key_hash.""" @@ -11890,6 +11927,7 @@ def test_get_standard_logging_metadata_none_user_api_key(): all_fields_present(result) assert result["user_api_key_hash"] is None + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_standard_logging_metadata_invalid_keys(): metadata = { @@ -11903,6 +11941,7 @@ def test_get_standard_logging_metadata_invalid_keys(): assert "invalid_key" not in result assert "another_invalid_key" not in result + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_cleanup_timestamps(): """Test cleanup_timestamps with different input types""" @@ -11937,6 +11976,7 @@ def test_cleanup_timestamps(): with pytest.raises(ValueError, match="start_time is required, got=invalid of type "): StandardLoggingPayloadSetup.cleanup_timestamps("invalid", end_float, completion_float) + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_model_cost_information(): """Test get_model_cost_information with different inputs""" @@ -11969,6 +12009,7 @@ def test_get_model_cost_information(): # assert all fields in StandardLoggingModelInformation are present assert all(field in result for field in StandardLoggingModelInformation.__annotations__) + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_model_cost_information_custom_pricing_uses_base_model(): result = StandardLoggingPayloadSetup.get_model_cost_information( @@ -11980,6 +12021,7 @@ def test_get_model_cost_information_custom_pricing_uses_base_model(): assert result["model_map_value"] is not None assert result["model_map_key"] != "invoke_test_claude" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_standard_logging_payload_uses_deployment_when_no_base_model(): """metadata["deployment"] is used for cost-map lookup when base_model is not set.""" @@ -12037,6 +12079,7 @@ def test_standard_logging_payload_uses_deployment_when_no_base_model(): assert payload["model_map_information"]["model_map_value"] is not None assert payload["model_map_information"]["model_map_key"] != "invoke_test_claude" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_hidden_params(): """Test get_hidden_params with different inputs""" @@ -12072,6 +12115,7 @@ def test_get_hidden_params(): # assert all fields in StandardLoggingHiddenParams are present assert all(field in result for field in StandardLoggingHiddenParams.__annotations__) + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_final_response_obj(): """Test get_final_response_obj with different input types and redaction scenarios""" @@ -12103,6 +12147,7 @@ def test_get_final_response_obj(): # Reset litellm.turn_off_message_logging to its original value litellm.turn_off_message_logging = False + def testget_standard_logging_payload_trace_id(): """Test get_standard_logging_payload_trace_id with different input scenarios""" # Test case 1: When litellm_trace_id is provided in litellm_params @@ -12140,6 +12185,7 @@ def testget_standard_logging_payload_trace_id(): assert result == "12345" assert isinstance(result, str) + def testget_standard_logging_payload_trace_id_prioritizes_trace_id_when_flag_on(monkeypatch): """With request_correlation_in_logs on, an explicit litellm_trace_id wins over litellm_session_id.""" from unittest.mock import MagicMock @@ -12154,6 +12200,7 @@ def testget_standard_logging_payload_trace_id_prioritizes_trace_id_when_flag_on( ) assert result == "the-trace-id" + def testget_standard_logging_payload_trace_id_prioritizes_session_id_when_flag_off(monkeypatch): """With request_correlation_in_logs off (default), legacy behavior is preserved: litellm_session_id still wins over litellm_trace_id.""" @@ -12169,6 +12216,7 @@ def testget_standard_logging_payload_trace_id_prioritizes_session_id_when_flag_o ) assert result == "the-session-id" + def testget_standard_logging_payload_session_id_when_flag_on(monkeypatch): """Test get_standard_logging_payload_session_id with different input scenarios, flag enabled""" from unittest.mock import MagicMock @@ -12221,6 +12269,7 @@ def testget_standard_logging_payload_session_id_when_flag_on(monkeypatch): ) assert result == "" + def testget_standard_logging_payload_session_id_empty_when_flag_off(monkeypatch): """When request_correlation_in_logs is off (default), session_id is always empty, even if litellm_session_id was explicitly supplied - preserves the pre-existing @@ -12237,6 +12286,7 @@ def testget_standard_logging_payload_session_id_empty_when_flag_off(monkeypatch) ) assert result == "" + @pytest.fixture def restore_verbose_logger_level() -> Iterator[None]: original_level: Final = verbose_logger.level @@ -12279,6 +12329,7 @@ def test_truncate_standard_logging_payload(): assert len(str(truncated["response"])) < 10_500 assert len(str(truncated["error_str"])) < 10_500 + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_truncate_standard_logging_payload_keeps_a_partial_payload_intact(): """A payload built with only some of its fields comes back with exactly those keys and values""" @@ -12287,12 +12338,14 @@ def test_truncate_standard_logging_payload_keeps_a_partial_payload_intact(): assert _custom_logger.truncate_standard_logging_payload_content(partial_payload) == partial_payload + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_strip_trailing_slash(): common_api_base = "https://api.test.com" assert StandardLoggingPayloadSetup.strip_trailing_slash(common_api_base + "/") == common_api_base assert StandardLoggingPayloadSetup.strip_trailing_slash(common_api_base) == common_api_base + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_error_information(): """Test get_error_information with different types of exceptions""" @@ -12329,6 +12382,7 @@ def test_get_error_information(): assert result["llm_provider"] == "openai" assert result["error_message"] == "litellm.RateLimitError: Test error" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_response_time(): """Test get_response_time with different streaming scenarios""" @@ -12364,6 +12418,7 @@ def test_get_response_time(): # For streaming, should return completion_start_time - start_time assert response_time == 2.0 + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.parametrize( "metadata, expected_requester_metadata", @@ -12386,6 +12441,7 @@ def test_standard_logging_metadata_requester_metadata(metadata, expected_request result = StandardLoggingPayloadSetup.get_standard_logging_metadata(metadata) assert result["requester_metadata"] == expected_requester_metadata + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_cost_breakdown_in_standard_logging_payload(): """ @@ -12475,6 +12531,7 @@ def test_cost_breakdown_in_standard_logging_payload(): print("✅ Cost breakdown test passed!") + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_cost_breakdown_missing_in_standard_logging_payload(): """ @@ -12535,6 +12592,7 @@ def test_cost_breakdown_missing_in_standard_logging_payload(): print("✅ Cost breakdown missing test passed!") + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.parametrize( "use_combined_usage_object", @@ -12622,6 +12680,7 @@ def test_usage_dict_roundtrip_in_payload(use_combined_usage_object): assert usage_obj["completion_tokens"] == 58 assert usage_obj["total_tokens"] == 100 + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_standard_logging_payload_uses_actual_model_for_azure_router(): from litellm.litellm_core_utils.litellm_logging import ( @@ -12670,6 +12729,7 @@ def test_standard_logging_payload_uses_actual_model_for_azure_router(): assert payload is not None assert payload["model"] == "azure_ai/gpt-5-nano-2025-08-07" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_standard_logging_payload_uses_actual_model_for_azure_router_with_underscore(): from litellm.litellm_core_utils.litellm_logging import ( @@ -12718,6 +12778,7 @@ def test_standard_logging_payload_uses_actual_model_for_azure_router_with_unders assert payload is not None assert payload["model"] == "azure_ai/gpt-5-nano-2025-08-07" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_merge_litellm_metadata_basic(): """ @@ -12749,6 +12810,7 @@ def test_merge_litellm_metadata_basic(): assert result["model_info"] == {"id": "model-123"} assert result["tags"] == ["tag1", "tag2"] + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_merge_litellm_metadata_precedence(): """ @@ -12775,6 +12837,7 @@ def test_merge_litellm_metadata_precedence(): # litellm_metadata values should only be included if not in metadata assert result["model_group"] == "gpt-4-group" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_merge_litellm_metadata_skip_non_serializable(): """ @@ -12807,6 +12870,7 @@ def test_merge_litellm_metadata_skip_non_serializable(): assert result["safe_field"] == "safe_value" assert result["model_group"] == "gpt-4-group" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_merge_litellm_metadata_empty_params(): """ @@ -12842,6 +12906,7 @@ def test_merge_litellm_metadata_empty_params(): result = StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params) assert result == {} + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_merge_litellm_metadata_bedrock_passthrough_scenario(): """ @@ -12901,8 +12966,10 @@ def test_merge_litellm_metadata_bedrock_passthrough_scenario(): # Verify total number of fields (9 user fields + 4 model fields = 13) assert len(result) == 13 + service_logger = ServiceLogging() + def setup_logging(): return Logging( model="gpt-5.5", @@ -12992,6 +13059,7 @@ def test_get_callback_name(): # Test string callback assert logging._get_callback_name("callback_string") == "callback_string" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_is_internal_litellm_proxy_callback(): """ @@ -13012,6 +13080,7 @@ def test_is_internal_litellm_proxy_callback(): # Test string callback assert logging._is_internal_litellm_proxy_callback("callback_string") == False + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_should_run_sync_callbacks_for_async_calls(): """ @@ -13037,6 +13106,7 @@ def test_should_run_sync_callbacks_for_async_calls(): litellm.success_callback = [PROXY_MaxIterationsHandler] assert logging._should_run_sync_callbacks_for_async_calls() == False + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_remove_internal_litellm_callbacks(): logging = setup_logging() @@ -13058,6 +13128,7 @@ def test_remove_internal_litellm_callbacks(): assert PROXY_MaxIterationsHandler not in filtered assert PROXY_CacheControlCheck not in filtered + @pytest.mark.asyncio async def test_background_interaction_completion_logs_while_in_progress_handler_is_parked(monkeypatch): """ @@ -13160,14 +13231,9 @@ class CompletionCustomHandler( assert metadata_value is None or isinstance(metadata_value, dict) if metadata_value is not None: if litellm.turn_off_message_logging is True: - assert ( - metadata_value["raw_request"] - is LiteLLMCommonStrings.redacted_by_litellm.value - ) + assert metadata_value["raw_request"] is LiteLLMCommonStrings.redacted_by_litellm.value else: - assert "raw_request" not in metadata_value or isinstance( - metadata_value["raw_request"], str - ) + assert "raw_request" not in metadata_value or isinstance(metadata_value["raw_request"], str) except Exception: print(f"Assertion Error: {traceback.format_exc()}") self.errors.append(traceback.format_exc()) @@ -13216,23 +13282,18 @@ class CompletionCustomHandler( assert isinstance(response_obj, litellm.ModelResponseStream) ## KWARGS assert isinstance(kwargs["model"], str) - assert isinstance(kwargs["messages"], list) and isinstance( - kwargs["messages"][0], dict - ) + assert isinstance(kwargs["messages"], list) and isinstance(kwargs["messages"][0], dict) assert isinstance(kwargs["optional_params"], dict) assert isinstance(kwargs["litellm_params"], dict) assert isinstance(kwargs["start_time"], (datetime, type(None))) assert isinstance(kwargs["stream"], bool) assert isinstance(kwargs["user"], (str, type(None))) - assert ( - isinstance(kwargs["input"], list) - and isinstance(kwargs["input"][0], dict) - ) or isinstance(kwargs["input"], (dict, str)) + assert (isinstance(kwargs["input"], list) and isinstance(kwargs["input"][0], dict)) or isinstance( + kwargs["input"], (dict, str) + ) assert isinstance(kwargs["api_key"], (str, type(None))) assert ( - isinstance( - kwargs["original_response"], (str, litellm.CustomStreamWrapper) - ) + isinstance(kwargs["original_response"], (str, litellm.CustomStreamWrapper)) or inspect.isasyncgen(kwargs["original_response"]) or inspect.iscoroutine(kwargs["original_response"]) ) @@ -13265,9 +13326,7 @@ class CompletionCustomHandler( ) ## KWARGS assert isinstance(kwargs["model"], str) - assert isinstance(kwargs["messages"], list) and isinstance( - kwargs["messages"][0], dict - ) + assert isinstance(kwargs["messages"], list) and isinstance(kwargs["messages"][0], dict) assert isinstance(kwargs["optional_params"], dict) assert isinstance(kwargs["litellm_params"], dict) assert isinstance(kwargs["litellm_params"]["api_base"], str) @@ -13277,10 +13336,7 @@ class CompletionCustomHandler( assert isinstance(kwargs["user"], (str, type(None))) assert ( isinstance(kwargs["input"], list) - and ( - isinstance(kwargs["input"][0], dict) - or isinstance(kwargs["input"][0], str) - ) + and (isinstance(kwargs["input"][0], dict) or isinstance(kwargs["input"][0], str)) ) or isinstance(kwargs["input"], (dict, str)) assert isinstance(kwargs["api_key"], (str, type(None))) assert isinstance( @@ -13308,9 +13364,7 @@ class CompletionCustomHandler( assert response_obj == None ## KWARGS assert isinstance(kwargs["model"], str) - assert isinstance(kwargs["messages"], list) and isinstance( - kwargs["messages"][0], dict - ) + assert isinstance(kwargs["messages"], list) and isinstance(kwargs["messages"][0], dict) assert isinstance(kwargs["optional_params"], dict) assert isinstance(kwargs["litellm_params"], dict) @@ -13318,15 +13372,12 @@ class CompletionCustomHandler( assert isinstance(kwargs["start_time"], (datetime, type(None))) assert isinstance(kwargs["stream"], bool) assert isinstance(kwargs["user"], (str, type(None))) - assert ( - isinstance(kwargs["input"], list) - and isinstance(kwargs["input"][0], dict) - ) or isinstance(kwargs["input"], (dict, str)) + assert (isinstance(kwargs["input"], list) and isinstance(kwargs["input"][0], dict)) or isinstance( + kwargs["input"], (dict, str) + ) assert isinstance(kwargs["api_key"], (str, type(None))) assert ( - isinstance( - kwargs["original_response"], (str, litellm.CustomStreamWrapper) - ) + isinstance(kwargs["original_response"], (str, litellm.CustomStreamWrapper)) or kwargs["original_response"] == None ) assert isinstance(kwargs["additional_args"], (dict, type(None))) @@ -13344,9 +13395,7 @@ class CompletionCustomHandler( assert isinstance(messages, list) and isinstance(messages[0], dict) ## KWARGS assert isinstance(kwargs["model"], str) - assert isinstance(kwargs["messages"], list) and isinstance( - kwargs["messages"][0], dict - ) + assert isinstance(kwargs["messages"], list) and isinstance(kwargs["messages"][0], dict) assert isinstance(kwargs["optional_params"], dict) assert isinstance(kwargs["litellm_params"], dict) assert isinstance(kwargs["start_time"], (datetime, type(None))) @@ -13358,9 +13407,7 @@ class CompletionCustomHandler( async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): try: - print( - "in async_log_success_event", kwargs, response_obj, start_time, end_time - ) + print("in async_log_success_event", kwargs, response_obj, start_time, end_time) self.states.append("async_success") ## START TIME assert isinstance(start_time, datetime) @@ -13389,9 +13436,7 @@ class CompletionCustomHandler( assert isinstance(kwargs["input"], (list, dict, str)) assert isinstance(kwargs["api_key"], (str, type(None))) assert ( - isinstance( - kwargs["original_response"], (str, litellm.CustomStreamWrapper) - ) + isinstance(kwargs["original_response"], (str, litellm.CustomStreamWrapper)) or inspect.isasyncgen(kwargs["original_response"]) or inspect.iscoroutine(kwargs["original_response"]) ) @@ -13423,9 +13468,7 @@ class CompletionCustomHandler( assert isinstance(kwargs["input"], (list, str, dict)) assert isinstance(kwargs["api_key"], (str, type(None))) assert ( - isinstance( - kwargs["original_response"], (str, litellm.CustomStreamWrapper) - ) + isinstance(kwargs["original_response"], (str, litellm.CustomStreamWrapper)) or inspect.isasyncgen(kwargs["original_response"]) or inspect.iscoroutine(kwargs["original_response"]) or kwargs["original_response"] == None @@ -13566,3 +13609,36 @@ def test_legacy_s3_logs_a_streamed_anthropic_messages_call_handed_off_from_the_a assert s3_writer.log_event.call_count == expected_s3_writes if expected_s3_writes: assert s3_writer.log_event.call_args.kwargs["response_obj"] is complete_response + + +_GO_PRICED_MESSAGE: Final = { + "id": "msg_go", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "answer"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1200, "output_tokens": 300, "cost": 0, "opencode": {"list_cost": 0.00412}}, +} + + +@pytest.mark.parametrize( + ("usage", "expected_header"), + ( + ({"input_tokens": 1200, "output_tokens": 300, "cost": 0, "opencode": {"list_cost": 0.00412}}, 0.00412), + ({"input_tokens": 1200, "output_tokens": 300}, None), + ), + ids=("list-cost-reported", "no-list-cost-keeps-the-price-map"), +) +def test_native_messages_body_carries_the_reported_list_cost_into_the_priced_response(usage, expected_header): + logging_obj = _anthropic_messages_logging_obj() + logging_obj.stream = False + logging_obj.optional_params = {} + logging_obj.model_call_details["custom_llm_provider"] = "opencode_go" + + priced = logging_obj.billable_result({**_GO_PRICED_MESSAGE, "usage": usage}) + + assert isinstance(priced, ModelResponse) + headers = priced._hidden_params.get("additional_headers") or {} + assert headers.get("llm_provider-x-litellm-response-cost") == expected_header diff --git a/tests/unit/llms/opencode/chat/test_transformation.py b/tests/unit/llms/opencode/chat/test_transformation.py index ce8f46f3da0..0a5e49ca1b2 100644 --- a/tests/unit/llms/opencode/chat/test_transformation.py +++ b/tests/unit/llms/opencode/chat/test_transformation.py @@ -1,3 +1,4 @@ +import asyncio import json from collections.abc import Mapping from datetime import datetime @@ -6,8 +7,11 @@ from typing import Final import httpx import pytest import respx +from pydantic import TypeAdapter import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.opencode.common_utils import go_headers @@ -400,3 +404,179 @@ def test_gemini_sync_stream_preserves_tool_calls_and_usage(go_catalog: None) -> assert json.loads(tool.function.arguments) == {"city": "Paris"} assert any(c.choices[0].finish_reason == "tool_calls" for c in chunks if c.choices) assert chunks[-1].usage.total_tokens == 15 + + +GO_REPORTED_USAGE: Final = { + "cost": 0, + "opencode": {"billing_source": "go", "list_cost": 0.00412, "allowance": 0.00412}, +} +ROW_PRICE_FOR_FIXTURE_USAGE: Final = 10 * 0.000002 + 5 * 0.000005 + + +def go_priced_reply(route: str, model: str, usage_extra: Mapping[str, object]) -> Mapping[str, object]: + reply: Final = upstream_reply(route, model) + key: Final = "usageMetadata" if route == "gemini" else "usage" + usage: Final = TypeAdapter(Mapping[str, object]).validate_python(reply[key]) + return {**reply, key: {**usage, **usage_extra}} + + +class CostRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.costs: list[float | None] = [] # mutable-ok: test probe accumulates logged costs + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: # noqa: ANN001 + self.costs.append(kwargs.get("response_cost")) + + +async def logged_cost(recorder: CostRecorder) -> float | None: + for _ in range(50): + if recorder.costs: + return recorder.costs[-1] + await asyncio.sleep(0.05) + raise AssertionError("success callback never ran") + + +@pytest.fixture +def cost_recorder(monkeypatch: pytest.MonkeyPatch) -> CostRecorder: + recorder: Final = CostRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + return recorder + + +@pytest.mark.asyncio +@pytest.mark.parametrize("upstream", ["chat/completions", "messages", "responses", "gemini"]) +@pytest.mark.parametrize("ingress", ["chat", "responses"]) +async def test_reported_list_cost_is_billed_instead_of_rows( + go_catalog: None, cost_recorder: CostRecorder, upstream: str, ingress: str +) -> None: + model: Final = f"fixture-{upstream.split('/')[0]}" + with respx.mock(using="httpx") as transport: + transport.post().mock( + return_value=httpx.Response(200, json=go_priced_reply(upstream, model, GO_REPORTED_USAGE)) + ) + kwargs: Final = {"model": f"opencode_go/{model}", "api_key": "fake-go-key"} + if ingress == "chat": + response: Final = await litellm.acompletion(messages=[{"role": "user", "content": "hello"}], **kwargs) + else: + response = await litellm.aresponses(input="hello", **kwargs) + assert response._hidden_params["additional_headers"][RESPONSE_COST_HEADER] == pytest.approx(0.00412) + assert await logged_cost(cost_recorder) == pytest.approx(0.00412) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("usage_extra", [{}, {"cost": 0}, {"opencode": {"billing_source": "go"}}]) +async def test_rows_price_the_request_when_no_list_cost_is_reported( + go_catalog: None, cost_recorder: CostRecorder, usage_extra: Mapping[str, object] +) -> None: + with respx.mock(using="httpx") as transport: + transport.post().mock( + return_value=httpx.Response(200, json=go_priced_reply("chat/completions", "fixture-chat", usage_extra)) + ) + response: Final = await litellm.acompletion( + model="opencode_go/fixture-chat", api_key="fake-go-key", messages=[{"role": "user", "content": "hello"}] + ) + assert RESPONSE_COST_HEADER not in response._hidden_params.get("additional_headers", {}) + assert await logged_cost(cost_recorder) == pytest.approx(ROW_PRICE_FOR_FIXTURE_USAGE) + + +def sse(*events: Mapping[str, object]) -> str: + return "".join(f"data: {json.dumps(event)}\n\n" for event in events) + "data: [DONE]\n\n" + + +@pytest.mark.asyncio +async def test_chat_stream_bills_the_final_usage_chunks_list_cost( + go_catalog: None, cost_recorder: CostRecorder +) -> None: + chunk: Final = { + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "fixture-chat", + } + with respx.mock(using="httpx") as transport: + transport.post().mock( + return_value=httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text=sse( + {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "hello"}}]}, + {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + { + **chunk, + "choices": [], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15, **GO_REPORTED_USAGE}, + }, + ), + ) + ) + stream: Final = await litellm.acompletion( + model="opencode_go/fixture-chat", + api_key="fake-go-key", + messages=[{"role": "user", "content": "hello"}], + stream=True, + stream_options={"include_usage": True}, + ) + chunks: Final = [piece async for piece in stream] + assert "".join(piece.choices[0].delta.content or "" for piece in chunks if piece.choices) == "hello" + assert await logged_cost(cost_recorder) == pytest.approx(0.00412) + + +@pytest.mark.asyncio +async def test_responses_stream_bills_the_completed_events_list_cost( + go_catalog: None, cost_recorder: CostRecorder +) -> None: + completed: Final = go_priced_reply("responses", "fixture-responses", GO_REPORTED_USAGE) + with respx.mock(using="httpx") as transport: + transport.post().mock( + return_value=httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text=sse( + { + "type": "response.output_text.delta", + "delta": "hello", + "item_id": "msg_test", + "output_index": 0, + "content_index": 0, + "sequence_number": 1, + }, + {"type": "response.completed", "response": completed, "sequence_number": 2}, + ), + ) + ) + stream: Final = await litellm.aresponses( + model="opencode_go/fixture-responses", api_key="fake-go-key", input="hello", stream=True + ) + events: Final = [event async for event in stream] + assert events[-1].type == "response.completed" + assert events[-1].response._hidden_params["additional_headers"][RESPONSE_COST_HEADER] == pytest.approx(0.00412) + assert await logged_cost(cost_recorder) == pytest.approx(0.00412) + + +def test_stream_chunk_builder_reads_list_cost_from_dict_chunks(): + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj = Logging( + model="opencode_go/glm-5.3", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="dict-chunks", + function_id="1", + ) + logging_obj.model_call_details["custom_llm_provider"] = "opencode_go" + chunk = { + "id": "chatcmpl-go", + "object": "chat.completion.chunk", + "created": 1, + "model": "glm-5.3", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + } + final_usage = {**chunk, "choices": [], "usage": dict(GO_REPORTED_USAGE)} + + assembled = litellm.stream_chunk_builder(chunks=[chunk, final_usage], logging_obj=logging_obj) + + assert assembled is not None + assert assembled._hidden_params["response_cost"] == GO_REPORTED_USAGE["opencode"]["list_cost"] diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index cc701e1ebd4..7530703a05a 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -145,9 +145,7 @@ class TestAnthropicLoggingHandlerModelFallback: '{"type": "message_stop"}', ] - def _create_mock_logging_obj( - self, model_in_details: str = None - ) -> LiteLLMLoggingObj: + def _create_mock_logging_obj(self, model_in_details: str = None) -> LiteLLMLoggingObj: """Create a mock logging object with optional model in model_call_details""" mock_logging_obj = MagicMock() @@ -166,21 +164,13 @@ class TestAnthropicLoggingHandlerModelFallback: mock_handler = MagicMock() return mock_handler - @patch.object( - AnthropicPassthroughLoggingHandler, "build_complete_streaming_response" - ) - @patch.object( - AnthropicPassthroughLoggingHandler, "_create_anthropic_response_logging_payload" - ) - def test_model_from_request_body_used_when_present( - self, mock_create_payload, mock_build_response - ): + @patch.object(AnthropicPassthroughLoggingHandler, "build_complete_streaming_response") + @patch.object(AnthropicPassthroughLoggingHandler, "_create_anthropic_response_logging_payload") + def test_model_from_request_body_used_when_present(self, mock_create_payload, mock_build_response): """Test that model from request_body is used when present""" # Arrange request_body = {"model": "claude-3-sonnet-20240229"} - logging_obj = self._create_mock_logging_obj( - model_in_details="claude-3-haiku-20240307" - ) + logging_obj = self._create_mock_logging_obj(model_in_details="claude-3-haiku-20240307") passthrough_handler = self._create_mock_passthrough_handler() # Mock successful response building @@ -204,57 +194,37 @@ class TestAnthropicLoggingHandlerModelFallback: # Verify that _build_complete_streaming_response was called with the request_body model mock_build_response.assert_called_once() call_args = mock_build_response.call_args - assert ( - call_args[1]["model"] == "claude-3-sonnet-20240229" - ) # Should use request_body model + assert call_args[1]["model"] == "claude-3-sonnet-20240229" # Should use request_body model def test_model_fallback_logic_isolated(self): """Test just the model fallback logic in isolation""" # Test case 1: Model from request body request_body = {"model": "claude-3-sonnet-20240229"} - logging_obj = self._create_mock_logging_obj( - model_in_details="claude-3-haiku-20240307" - ) + logging_obj = self._create_mock_logging_obj(model_in_details="claude-3-haiku-20240307") # Extract the logic directly from the function model = request_body.get("model", "") - if ( - not model - and hasattr(logging_obj, "model_call_details") - and logging_obj.model_call_details.get("model") - ): + if not model and hasattr(logging_obj, "model_call_details") and logging_obj.model_call_details.get("model"): model = logging_obj.model_call_details.get("model") assert model == "claude-3-sonnet-20240229" # Should use request_body model # Test case 2: Fallback to logging obj request_body = {} - logging_obj = self._create_mock_logging_obj( - model_in_details="claude-3-haiku-20240307" - ) + logging_obj = self._create_mock_logging_obj(model_in_details="claude-3-haiku-20240307") model = request_body.get("model", "") - if ( - not model - and hasattr(logging_obj, "model_call_details") - and logging_obj.model_call_details.get("model") - ): + if not model and hasattr(logging_obj, "model_call_details") and logging_obj.model_call_details.get("model"): model = logging_obj.model_call_details.get("model") assert model == "claude-3-haiku-20240307" # Should use fallback model # Test case 3: Empty string in request body, fallback to logging obj request_body = {"model": ""} - logging_obj = self._create_mock_logging_obj( - model_in_details="claude-3-opus-20240229" - ) + logging_obj = self._create_mock_logging_obj(model_in_details="claude-3-opus-20240229") model = request_body.get("model", "") - if ( - not model - and hasattr(logging_obj, "model_call_details") - and logging_obj.model_call_details.get("model") - ): + if not model and hasattr(logging_obj, "model_call_details") and logging_obj.model_call_details.get("model"): model = logging_obj.model_call_details.get("model") assert model == "claude-3-opus-20240229" # Should use fallback model @@ -264,11 +234,7 @@ class TestAnthropicLoggingHandlerModelFallback: logging_obj = self._create_mock_logging_obj() model = request_body.get("model", "") - if ( - not model - and hasattr(logging_obj, "model_call_details") - and logging_obj.model_call_details.get("model") - ): + if not model and hasattr(logging_obj, "model_call_details") and logging_obj.model_call_details.get("model"): model = logging_obj.model_call_details.get("model") assert model == "" # Should be empty @@ -284,11 +250,7 @@ class TestAnthropicLoggingHandlerModelFallback: # Extract the logic directly from the function model = request_body.get("model", "") - if ( - not model - and hasattr(logging_obj, "model_call_details") - and logging_obj.model_call_details.get("model") - ): + if not model and hasattr(logging_obj, "model_call_details") and logging_obj.model_call_details.get("model"): model = logging_obj.model_call_details.get("model") assert model == "" # Should remain empty since no fallback available @@ -298,11 +260,7 @@ class TestAnthropicLoggingHandlerModelFallback: logging_obj = self._create_mock_logging_obj() # Empty dict model = request_body.get("model", "") - if ( - not model - and hasattr(logging_obj, "model_call_details") - and logging_obj.model_call_details.get("model") - ): + if not model and hasattr(logging_obj, "model_call_details") and logging_obj.model_call_details.get("model"): model = logging_obj.model_call_details.get("model") assert model == "" # Should remain empty @@ -311,9 +269,7 @@ class TestAnthropicLoggingHandlerModelFallback: class TestAzureAnthropicCostCalculation: """Test the custom_llm_provider cost calculation logic for Azure AI Anthropic.""" - def _create_mock_logging_obj( - self, model: str = None, custom_llm_provider: str = None - ) -> LiteLLMLoggingObj: + def _create_mock_logging_obj(self, model: str = None, custom_llm_provider: str = None) -> LiteLLMLoggingObj: """Create a mock logging object with optional model and custom_llm_provider""" mock_logging_obj = MagicMock() mock_model_call_details = {} @@ -326,9 +282,7 @@ class TestAzureAnthropicCostCalculation: return mock_logging_obj @patch("litellm.completion_cost") - def test_cost_calculation_with_azure_ai_custom_llm_provider( - self, mock_completion_cost - ): + def test_cost_calculation_with_azure_ai_custom_llm_provider(self, mock_completion_cost): """Test that custom_llm_provider is passed to completion_cost for Azure AI Anthropic""" from litellm.types.utils import ModelResponse from datetime import datetime @@ -397,9 +351,7 @@ class TestAzureAnthropicCostCalculation: assert call_kwargs["custom_llm_provider"] is None @patch("litellm.completion_cost") - def test_cost_calculation_does_not_duplicate_provider_prefix( - self, mock_completion_cost - ): + def test_cost_calculation_does_not_duplicate_provider_prefix(self, mock_completion_cost): """Test that provider prefix is not duplicated if already present in model name""" from litellm.types.utils import ModelResponse from datetime import datetime @@ -436,9 +388,7 @@ class TestAzureAnthropicCostCalculation: assert call_kwargs["custom_llm_provider"] == "azure_ai" @patch("litellm.completion_cost") - def test_cost_calculation_resolves_unknown_model_from_litellm_params( - self, mock_completion_cost - ): + def test_cost_calculation_resolves_unknown_model_from_litellm_params(self, mock_completion_cost): """When the body model is the "unknown" sentinel, the deployment model from litellm_params must be used for costing, not "unknown" (which makes completion_cost raise and the cost silently fall back to $0).""" @@ -451,9 +401,7 @@ class TestAzureAnthropicCostCalculation: logging_obj = self._create_mock_logging_obj(model="unknown") logging_obj.model_call_details["litellm_params"] = { "model": "anthropic/claude-3-5-haiku-20241022", - "metadata": { - "model_group": "passthrough/anthropic/claude-3-5-haiku-20241022" - }, + "metadata": {"model_group": "passthrough/anthropic/claude-3-5-haiku-20241022"}, } logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"] @@ -471,17 +419,12 @@ class TestAzureAnthropicCostCalculation: ) mock_completion_cost.assert_called_once() - assert ( - mock_completion_cost.call_args[1]["model"] - == "anthropic/claude-3-5-haiku-20241022" - ) + assert mock_completion_cost.call_args[1]["model"] == "anthropic/claude-3-5-haiku-20241022" assert kwargs["response_cost"] == 0.001 assert kwargs["model"] == "anthropic/claude-3-5-haiku-20241022" @patch("litellm.completion_cost") - def test_cost_calculation_resolves_unknown_model_from_model_group( - self, mock_completion_cost - ): + def test_cost_calculation_resolves_unknown_model_from_model_group(self, mock_completion_cost): """With only model_group available (no deployment litellm_params.model), the leading passthrough/ prefix must be stripped so the cost map can resolve the model.""" @@ -493,9 +436,7 @@ class TestAzureAnthropicCostCalculation: logging_obj = self._create_mock_logging_obj(model="unknown") logging_obj.model_call_details["litellm_params"] = { - "metadata": { - "model_group": "passthrough/anthropic/claude-3-5-haiku-20241022" - } + "metadata": {"model_group": "passthrough/anthropic/claude-3-5-haiku-20241022"} } logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"] @@ -513,16 +454,11 @@ class TestAzureAnthropicCostCalculation: ) mock_completion_cost.assert_called_once() - assert ( - mock_completion_cost.call_args[1]["model"] - == "anthropic/claude-3-5-haiku-20241022" - ) + assert mock_completion_cost.call_args[1]["model"] == "anthropic/claude-3-5-haiku-20241022" assert kwargs["response_cost"] == 0.002 @patch("litellm.completion_cost") - def test_cost_calculation_skips_unknown_litellm_params_model_for_model_group( - self, mock_completion_cost - ): + def test_cost_calculation_skips_unknown_litellm_params_model_for_model_group(self, mock_completion_cost): """When litellm_params.model is itself the "unknown" sentinel, the deployment-model branch must not short-circuit; resolution falls through to model_group so costing still prices the real model instead of "unknown".""" @@ -535,9 +471,7 @@ class TestAzureAnthropicCostCalculation: logging_obj = self._create_mock_logging_obj(model="unknown") logging_obj.model_call_details["litellm_params"] = { "model": "unknown", - "metadata": { - "model_group": "passthrough/anthropic/claude-3-5-haiku-20241022" - }, + "metadata": {"model_group": "passthrough/anthropic/claude-3-5-haiku-20241022"}, } logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"] @@ -555,17 +489,12 @@ class TestAzureAnthropicCostCalculation: ) mock_completion_cost.assert_called_once() - assert ( - mock_completion_cost.call_args[1]["model"] - == "anthropic/claude-3-5-haiku-20241022" - ) + assert mock_completion_cost.call_args[1]["model"] == "anthropic/claude-3-5-haiku-20241022" assert kwargs["response_cost"] == 0.003 assert kwargs["model"] == "anthropic/claude-3-5-haiku-20241022" @patch("litellm.completion_cost") - def test_streaming_cost_calculation_resolves_model_from_message_start_chunk( - self, mock_completion_cost - ): + def test_streaming_cost_calculation_resolves_model_from_message_start_chunk(self, mock_completion_cost): """On the bare /anthropic passthrough path litellm_params carries no model or model_group and the body model is the "unknown" sentinel; the model must be recovered from the message_start SSE event so completion_cost @@ -628,9 +557,7 @@ class TestAzureAnthropicCostCalculation: ), _sse("message_stop", {"type": "message_stop"}), ] - all_chunks = list( - PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(frames) - ) + all_chunks = list(PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(frames)) logging_obj = RealLoggingObj( model="unknown", @@ -674,9 +601,7 @@ class TestAzureAnthropicCostCalculation: ] assert ( - AnthropicPassthroughLoggingHandler._extract_model_from_anthropic_chunks( - chunks - ) + AnthropicPassthroughLoggingHandler._extract_model_from_anthropic_chunks(chunks) == "claude-3-5-haiku-20241022" ) @@ -692,14 +617,11 @@ class TestAzureAnthropicCostCalculation: ) assert ( - AnthropicPassthroughLoggingHandler._extract_model_from_anthropic_chunks( - [raw_event] - ) + AnthropicPassthroughLoggingHandler._extract_model_from_anthropic_chunks([raw_event]) == "claude-3-5-haiku-20241022" ) - class TestAnthropicBatchPassthroughCostTracking: """Test cases for Anthropic batch passthrough cost tracking functionality""" @@ -799,9 +721,7 @@ class TestAnthropicBatchPassthroughCostTracking: ) mock_batches_config_instance = MagicMock() - mock_batches_config_instance.transform_retrieve_batch_response.return_value = ( - mock_batch_response - ) + mock_batches_config_instance.transform_retrieve_batch_response.return_value = mock_batch_response mock_batches_config.return_value = mock_batches_config_instance # Test the handler @@ -929,9 +849,7 @@ class TestAnthropicBatchPassthroughCostTracking: "transform_retrieve_batch_response", return_value=mock_batch_response, ): - with patch.object( - AnthropicPassthroughLoggingHandler, "_store_batch_managed_object" - ): + with patch.object(AnthropicPassthroughLoggingHandler, "_store_batch_managed_object"): result = AnthropicPassthroughLoggingHandler.batch_creation_handler( httpx_response=mock_httpx_response, logging_obj=mock_logging_obj, @@ -946,10 +864,7 @@ class TestAnthropicBatchPassthroughCostTracking: # Verify unified_object_id contains anthropic/ prefix unified_object_id = result["kwargs"]["unified_object_id"] decoded = base64.urlsafe_b64decode(unified_object_id + "==").decode() - assert ( - "anthropic/claude-sonnet-4-5-20250929" in decoded - or "claude-sonnet-4-5-20250929" in decoded - ) + assert "anthropic/claude-sonnet-4-5-20250929" in decoded or "claude-sonnet-4-5-20250929" in decoded @pytest.mark.parametrize( "kwargs,expected_user_id,expected_team_id", @@ -1007,9 +922,7 @@ class TestAnthropicBatchPassthroughCostTracking: mock_managed_files_hook.store_unified_object_id = AsyncMock() with ( patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_pl, - patch( - "litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution.verbose_proxy_logger" - ), + patch("litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution.verbose_proxy_logger"), ): mock_pl.get_proxy_hook.return_value = mock_managed_files_hook AnthropicPassthroughLoggingHandler._store_batch_managed_object( @@ -1060,9 +973,7 @@ class TestAnthropicBatchPassthroughCostTracking: is never tracked. The failure has to reach the log instead of being reported as a success.""" mock_managed_files_hook = MagicMock() - mock_managed_files_hook.store_unified_object_id = AsyncMock( - side_effect=RuntimeError("db down") - ) + mock_managed_files_hook.store_unified_object_id = AsyncMock(side_effect=RuntimeError("db down")) with ( patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_pl, patch( @@ -1104,9 +1015,7 @@ class TestAnthropicBatchPassthroughCostTracking: this gate an id-scoped route reached the store with a mismatched id, where it could only either claim a row it did not create or fail the model_object_id unique constraint.""" - with patch.object( - AnthropicPassthroughLoggingHandler, "_store_batch_managed_object" - ) as mock_store: + with patch.object(AnthropicPassthroughLoggingHandler, "_store_batch_managed_object") as mock_store: AnthropicPassthroughLoggingHandler.batch_creation_handler( httpx_response=mock_httpx_response, logging_obj=mock_logging_obj, @@ -1120,9 +1029,7 @@ class TestAnthropicBatchPassthroughCostTracking: assert mock_store.call_count == (1 if registers else 0) - def test_batch_creation_handler_failure_status_code( - self, mock_logging_obj, mock_request_body - ): + def test_batch_creation_handler_failure_status_code(self, mock_logging_obj, mock_request_body): """Test batch creation handler with non-200 status code""" mock_response = MagicMock() mock_response.status_code = 400 @@ -1145,9 +1052,7 @@ class TestAnthropicBatchPassthroughCostTracking: assert result["kwargs"]["response_cost"] == 0.0 @patch("litellm.proxy.proxy_server.proxy_logging_obj") - def test_store_batch_managed_object_success( - self, mock_proxy_logging_obj, mock_logging_obj - ): + def test_store_batch_managed_object_success(self, mock_proxy_logging_obj, mock_logging_obj): """Test storing batch managed object""" from litellm.types.utils import LiteLLMBatch @@ -1178,9 +1083,7 @@ class TestAnthropicBatchPassthroughCostTracking: ) # Verify managed files hook was called - mock_proxy_logging_obj.get_proxy_hook.assert_called_once_with( - "managed_files" - ) + mock_proxy_logging_obj.get_proxy_hook.assert_called_once_with("managed_files") class TestBuildCompleteStreamingResponseRobustness: @@ -1343,11 +1246,7 @@ class TestPureTextFastPathParity: }, ) ) - frames.append( - self._sse( - "content_block_stop", {"type": "content_block_stop", "index": b} - ) - ) + frames.append(self._sse("content_block_stop", {"type": "content_block_stop", "index": b})) frames.append( self._sse( "message_delta", @@ -1405,9 +1304,7 @@ class TestPureTextFastPathParity: # And the full logged payload (sans non-deterministic response id). def _scrub(p): d = dict(p) - r = d.get("complete_streaming_response_in_db") or d.get( - "complete_streaming_response" - ) + r = d.get("complete_streaming_response_in_db") or d.get("complete_streaming_response") return d, getattr(r, "usage", None) assert _scrub(k_fast)[1] == _scrub(k_legacy)[1] @@ -1419,11 +1316,7 @@ class TestPureTextFastPathParity: self._assert_parity(self._text_stream(["Just one piece of text."])) def test_parity_cache_tokens(self): - self._assert_parity( - self._text_stream( - ["a", "b", "c"], input_tokens=20, cache_creation=5, cache_read=7 - ) - ) + self._assert_parity(self._text_stream(["a", "b", "c"], input_tokens=20, cache_creation=5, cache_read=7)) def test_parity_max_tokens_stop(self): self._assert_parity(self._text_stream(["tok"] * 8, stop_reason="max_tokens")) @@ -1484,12 +1377,7 @@ class TestPureTextFastPathParity: self._sse("message_stop", {"type": "message_stop"}), ] all_chunks = self._to_all_chunks(frames) - assert ( - AnthropicPassthroughLoggingHandler._collapse_pure_text_chunks( - list(all_chunks) - ) - is None - ) + assert AnthropicPassthroughLoggingHandler._collapse_pure_text_chunks(list(all_chunks)) is None def test_collapse_returns_none_for_thinking(self): frames = [ @@ -1526,19 +1414,12 @@ class TestPureTextFastPathParity: self._sse("message_stop", {"type": "message_stop"}), ] all_chunks = self._to_all_chunks(frames) - assert ( - AnthropicPassthroughLoggingHandler._collapse_pure_text_chunks( - list(all_chunks) - ) - is None - ) + assert AnthropicPassthroughLoggingHandler._collapse_pure_text_chunks(list(all_chunks)) is None def test_collapse_actually_shrinks_chunk_count(self): frames = self._text_stream(["a"] * 50) all_chunks = list(self._to_all_chunks(frames)) - collapsed = AnthropicPassthroughLoggingHandler._collapse_pure_text_chunks( - all_chunks - ) + collapsed = AnthropicPassthroughLoggingHandler._collapse_pure_text_chunks(all_chunks) assert collapsed is not None # 50 text deltas + 50 event markers + 1 ping collapse to far fewer. assert len(collapsed) < len(all_chunks) / 2 @@ -1604,10 +1485,7 @@ class TestPureTextFastPathParity: self._sse("message_stop", {"type": "message_stop"}), ] all_chunks = list(self._to_all_chunks(frames)) - assert ( - AnthropicPassthroughLoggingHandler._collapse_pure_text_chunks(all_chunks) - is None - ) + assert AnthropicPassthroughLoggingHandler._collapse_pure_text_chunks(all_chunks) is None class TestInterruptedStreamOutputTokenRecovery: @@ -1722,9 +1600,7 @@ class TestInterruptedStreamOutputTokenRecovery: import litellm placeholder = 2 - result = self._run( - self._interrupted_chunks(placeholder_output_tokens=placeholder) - ) + result = self._run(self._interrupted_chunks(placeholder_output_tokens=placeholder)) usage = result["result"].usage expected = litellm.token_counter( @@ -1897,9 +1773,7 @@ class TestStreamFalseDeduplication: "delta": {"type": "text_delta", "text": "Hello"}, }, ), - TestStreamFalseDeduplication._sse( - "content_block_stop", {"type": "content_block_stop", "index": 0} - ), + TestStreamFalseDeduplication._sse("content_block_stop", {"type": "content_block_stop", "index": 0}), TestStreamFalseDeduplication._sse( "message_delta", { @@ -1949,15 +1823,12 @@ class TestStreamFalseDeduplication: # The assembled response must be stored on model_call_details so callbacks # can identify this as a completed streaming call, not an in-progress one. - assert ( - logging_obj.model_call_details.get("complete_streaming_response") - is not None - ), "complete_streaming_response must be set on model_call_details after assembly" + assert logging_obj.model_call_details.get("complete_streaming_response") is not None, ( + "complete_streaming_response must be set on model_call_details after assembly" + ) # The returned result must match what was stored - assert result["result"] is logging_obj.model_call_details.get( - "complete_streaming_response" - ) + assert result["result"] is logging_obj.model_call_details.get("complete_streaming_response") def test_dedup_guard_fires_when_stream_true_on_logging_obj(self): """ @@ -1986,21 +1857,16 @@ class TestStreamFalseDeduplication: assert logging_obj._is_assembled_stream_success(result=mock_response) is True # First dispatch sets the flag - assert not logging_obj.model_call_details.get( - "has_dispatched_final_stream_success" - ) + assert not logging_obj.model_call_details.get("has_dispatched_final_stream_success") logging_obj.model_call_details["has_dispatched_final_stream_success"] = True # Second dispatch would be blocked — simulate the guard check would_skip = bool( logging_obj._is_assembled_stream_success(result=mock_response) - and logging_obj.model_call_details.get( - "has_dispatched_final_stream_success" - ) + and logging_obj.model_call_details.get("has_dispatched_final_stream_success") ) assert would_skip is True, ( - "Dedup guard must block a second dispatch_success_handlers call for the " - "same assembled streaming response" + "Dedup guard must block a second dispatch_success_handlers call for the same assembled streaming response" ) def test_sse_fallback_path_sets_stream_true_for_dedup(self): @@ -2034,9 +1900,7 @@ class TestStreamFalseDeduplication: would_skip = bool( logging_obj._is_assembled_stream_success(result=mock_response) - and logging_obj.model_call_details.get( - "has_dispatched_final_stream_success" - ) + and logging_obj.model_call_details.get("has_dispatched_final_stream_success") ) assert would_skip is True @@ -2100,9 +1964,9 @@ class TestNonStreamingResponseRedaction: logging_obj=logging_obj, ) - assert ( - "complete_streaming_response" not in logging_obj.model_call_details - ), "non-streaming responses must not populate complete_streaming_response" + assert "complete_streaming_response" not in logging_obj.model_call_details, ( + "non-streaming responses must not populate complete_streaming_response" + ) def test_streaming_sets_complete_streaming_response(self): from litellm.types.utils import ModelResponse @@ -2119,10 +1983,7 @@ class TestNonStreamingResponseRedaction: logging_obj=logging_obj, ) - assert ( - logging_obj.model_call_details.get("complete_streaming_response") - is response - ) + assert logging_obj.model_call_details.get("complete_streaming_response") is response def test_non_streaming_response_is_redacted_when_message_logging_off(self): from litellm.litellm_core_utils.redact_messages import ( @@ -2196,10 +2057,8 @@ class TestAnthropicUsageOnlyFallback: ] def test_build_usage_only_recovers_cache_inclusive_usage(self): - response = ( - AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks( - all_chunks=self._CHUNKS, model="claude-3-5-haiku-20241022" - ) + response = AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks( + all_chunks=self._CHUNKS, model="claude-3-5-haiku-20241022" ) assert response is not None usage = response.usage @@ -2259,10 +2118,8 @@ class TestAnthropicUsageOnlyFallback: } ), ] - response = ( - AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks( - all_chunks=chunks, model="unknown" - ) + response = AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks( + all_chunks=chunks, model="unknown" ) assert response is not None assert response.model == "claude-opus-4-6" @@ -2287,9 +2144,7 @@ class TestAnthropicUsageOnlyFallback: ('data: {"a": 1}', {"a": 1}), ], ) - def test_extract_sse_data_handles_malformed_and_sentinel_lines( - self, event_str, expected - ): + def test_extract_sse_data_handles_malformed_and_sentinel_lines(self, event_str, expected): assert extract_sse_data(event_str) == expected def _real_logging_obj(self): @@ -2309,12 +2164,8 @@ class TestAnthropicUsageOnlyFallback: return logging_obj @patch("litellm.completion_cost") - @patch.object( - AnthropicPassthroughLoggingHandler, "build_complete_streaming_response" - ) - def test_handler_falls_back_when_assembly_returns_none( - self, mock_assemble, mock_cost - ): + @patch.object(AnthropicPassthroughLoggingHandler, "build_complete_streaming_response") + def test_handler_falls_back_when_assembly_returns_none(self, mock_assemble, mock_cost): mock_assemble.return_value = None mock_cost.return_value = 0.0021 logging_obj = self._real_logging_obj() @@ -2335,9 +2186,7 @@ class TestAnthropicUsageOnlyFallback: assert result["kwargs"]["response_cost"] == 0.0021 @patch("litellm.completion_cost") - @patch.object( - AnthropicPassthroughLoggingHandler, "build_complete_streaming_response" - ) + @patch.object(AnthropicPassthroughLoggingHandler, "build_complete_streaming_response") def test_handler_falls_back_when_assembly_raises(self, mock_assemble, mock_cost): import litellm @@ -2367,9 +2216,7 @@ class TestAnthropicUsageOnlyFallback: assert result["result"].usage.completion_tokens == 55 assert result["kwargs"]["response_cost"] == 0.0021 - @patch.object( - AnthropicPassthroughLoggingHandler, "build_complete_streaming_response" - ) + @patch.object(AnthropicPassthroughLoggingHandler, "build_complete_streaming_response") def test_handler_returns_none_when_no_usage_recoverable(self, mock_assemble): # assembly fails AND the chunks carry no usage event, so there is nothing # to price; the handler must return None rather than fabricate a response @@ -2391,15 +2238,9 @@ class TestAnthropicUsageOnlyFallback: assert result["result"] is None assert result["kwargs"] == {} - @patch.object( - AnthropicPassthroughLoggingHandler, "build_usage_only_response_from_chunks" - ) - @patch.object( - AnthropicPassthroughLoggingHandler, "build_complete_streaming_response" - ) - def test_handler_does_not_crash_when_usage_only_fallback_raises( - self, mock_assemble, mock_fallback - ): + @patch.object(AnthropicPassthroughLoggingHandler, "build_usage_only_response_from_chunks") + @patch.object(AnthropicPassthroughLoggingHandler, "build_complete_streaming_response") + def test_handler_does_not_crash_when_usage_only_fallback_raises(self, mock_assemble, mock_fallback): # if the usage-only fallback itself raises, it must be treated as None and # drop gracefully, not propagate out and crash the success handler mock_assemble.return_value = None @@ -2427,7 +2268,6 @@ class TestAnthropicResponseCostRecordedOnModelCallDetails: builder must record it there or streaming pass-through logs $0.""" - class TestAnthropicPassthroughFastMode: """Anthropic charges a provider-specific multiplier for ``speed=fast``, applied off ``usage.speed`` and covering every token type, cache included. The response usage @@ -2436,20 +2276,20 @@ class TestAnthropicPassthroughFastMode: MODEL = "claude-opus-4-8" STREAM_CHUNKS = [ - 'event: message_start', + "event: message_start", 'data: {"type": "message_start", "message": {"id": "msg_1", "type": "message", "role": "assistant",' ' "model": "claude-opus-4-8", "content": [], "stop_reason": null,' ' "usage": {"input_tokens": 1000, "cache_read_input_tokens": 200, "output_tokens": 0}}}', - 'event: content_block_start', + "event: content_block_start", 'data: {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}', - 'event: content_block_delta', + "event: content_block_delta", 'data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "ok"}}', - 'event: content_block_stop', + "event: content_block_stop", 'data: {"type": "content_block_stop", "index": 0}', - 'event: message_delta', + "event: message_delta", 'data: {"type": "message_delta", "delta": {"stop_reason": "end_turn"},' ' "usage": {"input_tokens": 1000, "cache_read_input_tokens": 200, "output_tokens": 100}}', - 'event: message_stop', + "event: message_stop", 'data: {"type": "message_stop"}', ] @@ -2643,7 +2483,6 @@ class TestRecordPartialUsageForFailure: assert "response_cost" not in logging_obj.model_call_details - @pytest.mark.parametrize( ("all_chunks", "interrupted"), [ @@ -2664,9 +2503,7 @@ def test_stream_was_interrupted_skips_data_lines_that_are_not_json_objects( @pytest.mark.asyncio -async def test_anthropic_passthrough_handler( - mock_httpx_response, mock_response, mock_logging_obj -): +async def test_anthropic_passthrough_handler(mock_httpx_response, mock_response, mock_logging_obj): """ Unit test - Assert that the anthropic passthrough handler calls the litellm logging object's async_success_handler """ @@ -3007,3 +2844,102 @@ def test_streamed_messages_end_turn_logs_no_stop_details(mock_logging_obj): assert logged.choices[0].finish_reason == "stop" provider_fields: Final = logged.choices[0].message.provider_specific_fields or {} assert "stop_details" not in provider_fields + + +def _go_sse(event: str, data: dict) -> str: + return f"event: {event}\ndata: {json.dumps(data)}\n\n" + + +_GO_MESSAGE_DELTA_USAGE: Final = { + "input_tokens": 1200, + "output_tokens": 300, + "cost": 0, + "opencode": {"billing_source": "go", "list_cost": 0.00412}, +} + + +def _go_stream_chunks() -> list[str]: + message = { + "id": "msg_go", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 1200, "output_tokens": 0}, + } + return [ + _go_sse("message_start", {"type": "message_start", "message": message}), + _go_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _go_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "ok"}}, + ), + _go_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _go_sse( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": _GO_MESSAGE_DELTA_USAGE, + }, + ), + _go_sse("message_stop", {"type": "message_stop"}), + ] + + +def _go_logging_obj() -> LiteLLMLoggingObj: + logging_obj = LiteLLMLoggingObj( + model="opencode_go/claude-haiku-5-5", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="go-stream", + function_id="1", + ) + logging_obj.model_call_details["custom_llm_provider"] = "opencode_go" + logging_obj.model_call_details["stream"] = True + logging_obj.model_call_details["litellm_params"] = {} + logging_obj.litellm_params = {} + return logging_obj + + +def test_streamed_messages_bill_the_list_cost_reported_in_the_final_message_delta(): + result = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks( + litellm_logging_obj=_go_logging_obj(), + passthrough_success_handler_obj=MagicMock(), + url_route="/v1/messages", + request_body={"model": "opencode_go/claude-haiku-5-5", "stream": True}, + endpoint_type="messages", + start_time=datetime.now(), + all_chunks=_go_stream_chunks(), + end_time=datetime.now(), + ) + + assert result["kwargs"]["response_cost"] == 0.00412 + assert result["result"]._hidden_params["response_cost"] == 0.00412 + + +def test_streamed_messages_from_other_providers_ignore_a_zero_usage_cost_field(): + logging_obj = _go_logging_obj() + logging_obj.model_call_details["custom_llm_provider"] = "anthropic" + + result = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route="/v1/messages", + request_body={"model": "claude-haiku-5-5", "stream": True}, + endpoint_type="messages", + start_time=datetime.now(), + all_chunks=_go_stream_chunks(), + end_time=datetime.now(), + ) + + headers = result["result"]._hidden_params.get("additional_headers") or {} + assert "llm_provider-x-litellm-response-cost" not in headers + assert result["kwargs"].get("response_cost") != 0.0 From 205cfd8044269ac43c2a91c6cafac36146e29171 Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Sat, 24 Oct 2026 16:47:15 -0700 Subject: [PATCH 032/155] fix(proxy): restore Claude gateway model discovery (#48204) (#48221) * fix(proxy): restore Claude gateway model discovery (#48204) * test(proxy): cover gateway discovery after OAuth sign-in (#48204) * refactor(proxy): share model listing route policy (#48204) * test(proxy): isolate SSO settings environment (#48204) --- .../proxy/auth/route_checks.py | 4 +- litellm/proxy/_types.py | 11 +-- .../anthropic_endpoints/gateway_endpoints.py | 8 ++ litellm/proxy/auth/auth_checks.py | 7 +- litellm/proxy/auth/route_checks.py | 4 +- .../test_claude_code_gateway_oauth.py | 27 +++++- .../auth/test_managed_authorization.py | 2 +- .../test_gateway_endpoints.py | 92 +++++++++++++++++++ .../test_proxy_setting_endpoints.py | 8 ++ 9 files changed, 148 insertions(+), 15 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/auth/route_checks.py b/enterprise/litellm_enterprise/proxy/auth/route_checks.py index fc57292a8d2..479537c4a90 100644 --- a/enterprise/litellm_enterprise/proxy/auth/route_checks.py +++ b/enterprise/litellm_enterprise/proxy/auth/route_checks.py @@ -2,6 +2,8 @@ import os from fastapi import HTTPException, status +from litellm.proxy._types import MODEL_LIST_ROUTES + class EnterpriseRouteChecks: @staticmethod @@ -43,7 +45,7 @@ class EnterpriseRouteChecks: # Routes that should remain accessible even when LLM API endpoints are disabled. # These are read-only model listing routes needed by the Admin UI. - LLM_API_EXEMPT_ROUTES = ["/models", "/v1/models"] + LLM_API_EXEMPT_ROUTES = MODEL_LIST_ROUTES @staticmethod def should_call_route(route: str): diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 3a3306dfdc0..2b01e02bb06 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -219,6 +219,8 @@ from litellm.types.proxy.auth.user_roles import ( # noqa: E402 # re-export aft KeyManagementRoutes as KeyManagementRoutes, # noqa: PLC0414 # public re-export ) +MODEL_LIST_ROUTES: Final = ("/models", "/v1/models", "/claude_code_gateway/v1/models") + class LiteLLMRoutes(enum.Enum): openai_route_names = [ @@ -307,8 +309,7 @@ class LiteLLMRoutes(enum.Enum): "/threads/{thread_id}/runs", "/v1/threads/{thread_id}/runs", # models - "/models", - "/v1/models", + *MODEL_LIST_ROUTES, # token counter "/utils/token_counter", "/utils/model_info", @@ -577,8 +578,7 @@ class LiteLLMRoutes(enum.Enum): "/health/services", "/key/list", "/user/filter/ui", - "/models", - "/v1/models", + *MODEL_LIST_ROUTES, "/sso/get/ui_settings", "/get/user_banner", "/get/latest_release_info", @@ -760,8 +760,7 @@ class LiteLLMRoutes(enum.Enum): "/model/info", "/v2/model/info", "/v2/key/info", - "/models", - "/v1/models", + *MODEL_LIST_ROUTES, "/global/spend", "/global/spend/logs", "/global/spend/keys", diff --git a/litellm/proxy/anthropic_endpoints/gateway_endpoints.py b/litellm/proxy/anthropic_endpoints/gateway_endpoints.py index 4c4155379cd..ba3307bc1ad 100644 --- a/litellm/proxy/anthropic_endpoints/gateway_endpoints.py +++ b/litellm/proxy/anthropic_endpoints/gateway_endpoints.py @@ -58,6 +58,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg ) from litellm.proxy.management_endpoints.sso_helper_utils import CLI_SSO_SESSIONS_TARGET from litellm.proxy.management_endpoints.ui_sso import CliSsoTeamDetail +from litellm.proxy.proxy_server import model_list from litellm.types.llms.base import LiteLLMBaseModel GATEWAY_PREFIX: Final = "/claude_code_gateway" @@ -158,6 +159,13 @@ router: Final = APIRouter( _GATEWAY_ENABLED: Final = (Depends(ensure_gateway_enabled),) _AUTHENTICATED: Final = (Depends(user_api_key_auth),) +router.add_api_route( + "/v1/models", + model_list, + methods=["GET"], + dependencies=_GATEWAY_ENABLED, + include_in_schema=False, +) router.add_api_route( "/v1/messages", anthropic_response, diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 2c6768b8423..06c53afcf2c 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -47,6 +47,7 @@ from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.models.project import LiteLLM_ProjectTable from litellm.proxy._types import ( + MODEL_LIST_ROUTES, RBAC_ROLES, UI_TEAM_ID, CallInfo, @@ -910,8 +911,7 @@ def _global_proxy_budget_check(global_proxy_spend: float | None, skip_budget_che and not skip_budget_checks and global_proxy_spend is not None and RouteChecks.is_llm_api_route(route=route) - and route != "/v1/models" - and route != "/models" + and route not in MODEL_LIST_ROUTES ): if math.isfinite(litellm.max_budget) and global_proxy_spend > litellm.max_budget: raise litellm.BudgetExceededError( @@ -1028,8 +1028,7 @@ async def check_tools_allowlist( # budget cannot reach side-effectful routes like /health/services (Slack/email/webhook). See #27923. MODEL_DISCOVERY_ROUTES: Final = frozenset( { - "/v1/models", - "/models", + *MODEL_LIST_ROUTES, "/model/info", "/v1/model/info", "/v2/model/info", diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 3ae671d40ac..c402f111563 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -7,6 +7,7 @@ from fastapi import HTTPException, Request, status from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ( + MODEL_LIST_ROUTES, UI_TEAM_ID, CommonProxyErrors, KeyManagementRoutes, @@ -93,8 +94,7 @@ _SELF_SCOPED_CREDENTIAL_ROUTES: Final = ( _UNBILLED_ROUTES: Final[frozenset[str]] = frozenset( { - "/models", - "/v1/models", + *MODEL_LIST_ROUTES, "/utils/token_counter", "/utils/transform_request", "/responses/input_tokens", diff --git a/tests/integration/authorization/test_claude_code_gateway_oauth.py b/tests/integration/authorization/test_claude_code_gateway_oauth.py index 384bf3a2918..a80eeacad46 100644 --- a/tests/integration/authorization/test_claude_code_gateway_oauth.py +++ b/tests/integration/authorization/test_claude_code_gateway_oauth.py @@ -15,6 +15,7 @@ from urllib.parse import parse_qs, urlencode, urlsplit import httpx import pytest import yaml +from anthropic import Anthropic from anthropic.types import Message from integration._support import claude_code as cc from integration._support.client import Gateway, eventually, gateway_from_environment @@ -397,7 +398,31 @@ def test_device_authorization_polls_complete_sso_and_mints_a_model_restricted_be "anthropic-beta": cc.CLI_BETA, "x-claude-code-session-id": f"oauth-{user_id}", } - request_body: Final = {**request_template, "model": model_a} + standard_listing: Final = enabled_gateway.client.get("/v1/models", headers=bearer_headers) + gateway_listing: Final = enabled_gateway.client.get( + "/claude_code_gateway/v1/models", headers=bearer_headers + ) + assert standard_listing.status_code == 200, standard_listing.text + assert gateway_listing.status_code == 200, gateway_listing.text + assert gateway_listing.json() == standard_listing.json() + with Anthropic( + base_url=f"{str(enabled_gateway.client.base_url).rstrip('/')}/claude_code_gateway", + auth_token=access_token.access_token, + ) as sdk: + discovered_models: Final = tuple(sdk.models.list()) + assert {model.id for model in discovered_models} == {model_a, model_c} + discovered_model: Final = next(model for model in discovered_models if model.id == model_a) + openai_listing: Final = enabled_gateway.client.get( + "/claude_code_gateway/v1/models", + headers={"authorization": f"Bearer {access_token.access_token}"}, + ) + assert openai_listing.status_code == 200, openai_listing.text + assert openai_listing.json()["object"] == "list" + assert {model["id"] for model in openai_listing.json()["data"]} == {model_a, model_c} + unauthenticated: Final = enabled_gateway.client.get("/claude_code_gateway/v1/models") + assert unauthenticated.status_code == 401, unauthenticated.text + assert upstream.drain() == () + request_body: Final = {**request_template, "model": discovered_model.id} expected_message: Final = _MESSAGE.validate_python( { **_JSON_OBJECT.validate_json( diff --git a/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py b/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py index fb676ae9a61..2b682fda982 100644 --- a/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py +++ b/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py @@ -588,6 +588,6 @@ def test_registered_inference_routes_have_an_explicit_managed_access_decision(ro "/vector_stores", "/vector_store/", "/search", "/containers", "/skills", "/claude-code/", "/interactions", "/agents", "/responses/{", "/responses/input_tokens", "/realtime/client_secrets", "/realtime/calls", "/realtime/transcription_sessions", - )) or normalized in ("/models", "/cursor/models", "/cursor/v1/models") + )) or normalized in ("/models", "/cursor/models", "/cursor/v1/models", "/claude_code_gateway/v1/models") concrete: Final = route.split("?")[0].replace("{model}", "model").replace("{model_name:path}", "model") assert managed_agent_route_allowed(concrete, None) is not unsupported, route diff --git a/tests/unit/proxy/anthropic_endpoints/test_gateway_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_gateway_endpoints.py index 8ac0e2a6f6d..8c2d8e4f157 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_gateway_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_gateway_endpoints.py @@ -813,3 +813,95 @@ def test_messages_gated_by_enable_flag(): with _gateway_env(enabled=False) as (client, _): resp = client.post("/claude_code_gateway/v1/messages", json={"model": "claude-sonnet-4-5", "messages": []}) assert resp.status_code == 404 + + +@pytest.mark.parametrize("headers", [{}, {"anthropic-version": "2023-06-01"}]) +def test_gateway_model_discovery_preserves_grants_and_format( + monkeypatch: pytest.MonkeyPatch, headers: dict[str, str] +) -> None: + from litellm import Router + from litellm.proxy import proxy_server + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + + model_router: Final = Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": f"openai/{name}", "api_key": "local-placeholder"}, + } + for name in ("discovery-example", "restricted-example") + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", model_router) + monkeypatch.setattr(proxy_server, "llm_model_list", model_router.model_list) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "general_settings", {"enable_claude_code_gateway": True}) + caller: Final = UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + models=["discovery-example"], + ) + app: Final = FastAPI() + app.include_router(proxy_server.router) + app.include_router(gateway_endpoints.router) + app.dependency_overrides[gateway_endpoints.user_api_key_auth] = lambda: caller + with TestClient(app) as client: + standard: Final = client.get("/v1/models", headers=headers) + gateway: Final = client.get("/claude_code_gateway/v1/models", headers=headers) + assert standard.status_code == 200, standard.text + assert gateway.status_code == 200, gateway.text + assert gateway.json() == standard.json() + assert [model["id"] for model in gateway.json()["data"]] == ["discovery-example"] + invalid_scope: Final = client.get("/claude_code_gateway/v1/models?scope=invalid", headers=headers) + assert invalid_scope.status_code == 400 + + +def test_gateway_model_discovery_requires_authentication() -> None: + with _gateway_env(real_auth=True) as (client, _), pytest.raises(ProxyException) as exc_info: + client.get("/claude_code_gateway/v1/models") + assert exc_info.value.code == "401" + + +@pytest.mark.parametrize("settings", [{}, {"enable_claude_code_gateway": False}]) +def test_gateway_model_discovery_disabled(monkeypatch: pytest.MonkeyPatch, settings: dict[str, bool]) -> None: + from litellm.proxy import proxy_server + + with _gateway_env(real_auth=True) as (client, _): + monkeypatch.setattr(proxy_server, "general_settings", settings) + response: Final = client.get( + "/claude_code_gateway/v1/models", headers={"Authorization": f"Bearer {_MASTER_KEY}"} + ) + assert response.status_code == 404, response.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ["/models", "/v1/models", "/claude_code_gateway/v1/models"]) +async def test_gateway_model_discovery_preserves_route_exemptions(monkeypatch: pytest.MonkeyPatch, route: str) -> None: + from litellm_enterprise.proxy.auth.route_checks import EnterpriseRouteChecks + + import litellm + from litellm.proxy._types import LiteLLM_TeamTable, UserAPIKeyAuth + from litellm.proxy.auth.auth_checks import common_checks + from litellm.proxy.auth.route_checks import RouteChecks + + monkeypatch.setattr(litellm, "max_budget", 1.0) + result: Final = await common_checks( + request_body={}, + team_object=LiteLLM_TeamTable(team_id="test-team", spend=150.0, max_budget=100.0), + user_object=None, + end_user_object=None, + global_proxy_spend=2.0, + general_settings={}, + route=route, + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=UserAPIKeyAuth(team_id="test-team"), + request=MagicMock(), + ) + assert result is True + assert RouteChecks.is_unbilled_route(route) + assert RouteChecks.is_llm_api_route(route) + monkeypatch.setenv("DISABLE_LLM_API_ENDPOINTS", "true") + monkeypatch.setenv("DISABLE_ADMIN_ENDPOINTS", "true") + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + EnterpriseRouteChecks.should_call_route(route) diff --git a/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index 0d558f206d8..57b68e6db05 100644 --- a/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -1,6 +1,8 @@ import json import os +from collections.abc import Iterator from typing import Final +from unittest.mock import patch import pytest from fastapi.testclient import TestClient @@ -181,6 +183,12 @@ def mock_auth(): app.dependency_overrides.pop(user_api_key_auth, None) +@pytest.fixture(autouse=True) +def isolated_environment() -> Iterator[None]: + with patch.dict(os.environ): + yield + + @pytest.fixture(autouse=True) def fresh_settings_store(monkeypatch: pytest.MonkeyPatch) -> None: from litellm.proxy import proxy_server From 387b8b34777d0e2f61b679c7cb90524bf36a3a65 Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Sat, 24 Oct 2026 17:06:31 -0700 Subject: [PATCH 033/155] fix(anthropic): release upstream messages stream on disconnect (#48197) (#48220) * fix(anthropic): close upstream messages stream on disconnect (#48197) * test(anthropic): prove disconnect cleanup over live proxy (#48197) --- .../adapters/streaming_iterator.py | 21 +- .../test_chat_bridge_disconnect_wire.py | 232 ++++++++++++++++++ .../test_streaming_iterator_sse_stream.py | 67 +++++ 3 files changed, 319 insertions(+), 1 deletion(-) create mode 100644 tests/integration/messages_endpoint/chat_bridge/test_chat_bridge_disconnect_wire.py diff --git a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py index dadd4d8558d..4eb53f35a03 100644 --- a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py @@ -66,6 +66,11 @@ class _ChunkStream(Protocol): def __aiter__(self) -> "AsyncIterator[ModelResponseStream]": ... +@runtime_checkable +class _AsyncCloseable(Protocol): + async def aclose(self) -> None: ... + + @runtime_checkable class _FailureLoggingWaiter(Protocol): @property @@ -197,6 +202,14 @@ class _CombinedChunkSplitter: self._buffer: deque[ModelResponseStream] = deque() self._held_calls: Mapping[int, tuple[ModelResponseStream, _ToolCallPart]] = MappingProxyType({}) self._live_call_index: int | None = None + self._closed = False + + async def aclose(self) -> None: + if self._closed: + return + self._closed = True + if isinstance(self._stream, _AsyncCloseable): + await self._stream.aclose() @property def has_dispatched_failure_logging(self) -> bool: @@ -1239,6 +1252,9 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): # For non-dict chunks, forward the original value unchanged yield chunk + async def aclose(self) -> None: + await self._combined_chunk_splitter.aclose() + async def async_anthropic_sse_wrapper(self) -> AsyncIterator[bytes]: """ Async version of anthropic_sse_wrapper. @@ -1481,4 +1497,7 @@ class AnthropicSSEStream(AsyncIterator[bytes]): return await self._byte_stream.__anext__() async def aclose(self) -> None: - await self._byte_stream.aclose() + try: + await self._byte_stream.aclose() + finally: + await self._anthropic_wrapper.aclose() diff --git a/tests/integration/messages_endpoint/chat_bridge/test_chat_bridge_disconnect_wire.py b/tests/integration/messages_endpoint/chat_bridge/test_chat_bridge_disconnect_wire.py new file mode 100644 index 00000000000..100e760d5be --- /dev/null +++ b/tests/integration/messages_endpoint/chat_bridge/test_chat_bridge_disconnect_wire.py @@ -0,0 +1,232 @@ +import asyncio +import json +from collections.abc import AsyncGenerator, AsyncIterator +from contextlib import ExitStack, asynccontextmanager +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Final +from urllib.parse import urlsplit + +import httpx +import pytest +import pytest_asyncio +import yaml +from aiohttp import web + +from tests.integration._support.anthropic_sse import LIFECYCLE, delta_text, event_types, parse_sse, user_prompt +from tests.integration._support.client import JSON_OBJECT, Gateway +from tests.integration._support.openai_wire import chat_stream +from tests.integration._support.ports import allocate_port +from tests.integration._support.process import OwnedProxy, owned_proxy_process, paused + +_MODEL: Final = "disconnect-repro" +_KEY: Final = "sk-integration-master-key" +_DEADLINE: Final = 10 +_RACE_ATTEMPTS: Final = 20 + + +class _Mode(Enum): + OCCUPY = "occupy" + QUEUED = "queued" + CONTENT = "content" + COMPLETE = "complete" + + +@dataclass(frozen=True, slots=True) +class _Request: + mode: _Mode + headers_allowed: asyncio.Event = field(default_factory=asyncio.Event) + headers_sent: asyncio.Event = field(default_factory=asyncio.Event) + running: asyncio.Event = field(default_factory=asyncio.Event) + released: asyncio.Event = field(default_factory=asyncio.Event) + + +@dataclass(frozen=True, slots=True) +class _Backend: + url: str + requests: asyncio.Queue[_Request] + runner: web.AppRunner + + +@pytest_asyncio.fixture(loop_scope="function") +async def backend() -> AsyncIterator[_Backend]: + requests: Final[asyncio.Queue[_Request]] = asyncio.Queue() + worker: Final = asyncio.Semaphore(1) + + async def complete(request: web.Request) -> web.StreamResponse: + body: Final = JSON_OBJECT.validate_json(await request.read()) + assert body["model"] == "test-model" and body["stream"] is True, body + assert request.headers["Authorization"] == "Bearer integration-provider-key" + pending: Final = _Request(_Mode(user_prompt(body))) + await requests.put(pending) + response: Final = web.StreamResponse(headers={"Content-Type": "text/event-stream"}) + try: + await pending.headers_allowed.wait() + await response.prepare(request) + pending.headers_sent.set() + async with worker: + pending.running.set() + chunks: Final = chat_stream("chatcmpl-disconnect", "test-model", "hello") + if pending.mode in (_Mode.CONTENT, _Mode.COMPLETE): + for chunk in chunks[:2]: + await response.write(chunk) + if pending.mode is _Mode.COMPLETE: + await response.write(chunks[2]) + await response.write_eof() + else: + await asyncio.Event().wait() + finally: + pending.released.set() + return response + + app: Final = web.Application() + app.router.add_post("/v1/chat/completions", complete) + runner: Final = web.AppRunner(app, handler_cancellation=True, shutdown_timeout=0.1) + await runner.setup() + port: Final = allocate_port() + await web.TCPSite(runner, "127.0.0.1", port).start() + try: + yield _Backend(f"http://127.0.0.1:{port}", requests, runner) + finally: + await runner.cleanup() + + +@pytest_asyncio.fixture(loop_scope="function") +async def proxy(backend: _Backend, tmp_path: Path) -> AsyncIterator[OwnedProxy]: + config: Final = tmp_path / "disconnect.yaml" + config.write_text( + yaml.safe_dump( + { + "model_list": [ + { + "model_name": _MODEL, + "litellm_params": { + "model": "custom_openai/test-model", + "api_base": backend.url + "/v1", + "api_key": "integration-provider-key", + }, + } + ], + "litellm_settings": {"anthropic_sse_ping_interval_seconds": 0}, + "router_settings": {"num_retries": 0}, + "general_settings": {"dangerously_permit_weak_or_unset_master_key": True}, + } + ) + ) + with httpx.Client() as client: + template: Final = Gateway(client, _KEY, backend.url) + with owned_proxy_process( + template, + tmp_path, + {"STORE_MODEL_IN_DB": "False"}, + config=config, + database_setup=(), + remove_environment=("DATABASE_URL", "DATABASE_URL_READ_REPLICA", "REDIS_HOST", "REDIS_PORT"), + ) as owned: + try: + yield owned + finally: + await backend.runner.cleanup() + + +def _body(mode: _Mode) -> bytes: + return json.dumps( + {"model": _MODEL, "stream": True, "max_tokens": 16, "messages": [{"role": "user", "content": mode.value}]} + ).encode() + + +@asynccontextmanager +async def _request( + proxy: OwnedProxy, mode: _Mode, endpoint: str = "/v1/messages" +) -> AsyncGenerator[asyncio.StreamReader, None]: + address: Final = urlsplit(str(proxy.gateway.client.base_url)) + reader, writer = await asyncio.open_connection(address.hostname, address.port) + body: Final = _body(mode) + writer.write( + f"POST {endpoint} HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer {_KEY}\r\n" + f"Content-Type: application/json\r\nContent-Length: {len(body)}\r\n\r\n".encode() + + body + ) + await writer.drain() + try: + yield reader + finally: + writer.close() + await writer.wait_closed() + + +async def _next(backend: _Backend, mode: _Mode) -> _Request: + pending: Final = await asyncio.wait_for(backend.requests.get(), _DEADLINE) + assert pending.mode is mode + return pending + + +async def _headers(pending: _Request) -> None: + pending.headers_allowed.set() + await asyncio.wait_for(pending.headers_sent.wait(), _DEADLINE) + + +async def _released(pending: _Request) -> None: + try: + await asyncio.wait_for(pending.released.wait(), _DEADLINE) + except TimeoutError: + pytest.fail(f"Backend {pending.mode.value} request still active after client disconnect", pytrace=False) + + +async def _disconnect_queued_request(backend: _Backend, proxy: OwnedProxy) -> None: + with ExitStack() as scheduling: + async with _request(proxy, _Mode.QUEUED): + queued: Final = await _next(backend, _Mode.QUEUED) + scheduling.enter_context(paused(proxy.process)) + await _headers(queued) + assert not queued.running.is_set() + await _released(queued) + assert backend.requests.empty(), "Disconnect must not trigger another upstream request" + + +async def test_disconnect_with_upstream_headers_pending_releases_queued_request( + backend: _Backend, proxy: OwnedProxy +) -> None: + """Hold headers and the client FIN ready together without patching proxy code.""" + async with _request(proxy, _Mode.OCCUPY): + occupied: Final = await _next(backend, _Mode.OCCUPY) + await _headers(occupied) + await asyncio.wait_for(occupied.running.wait(), _DEADLINE) + for attempt in range(_RACE_ATTEMPTS): + await _disconnect_queued_request(backend, proxy) + assert not occupied.released.is_set(), attempt + await _released(occupied) + + +@pytest.mark.parametrize("endpoint", ["/v1/messages", "/v1/chat/completions"]) +async def test_disconnect_after_content_releases_upstream(backend: _Backend, proxy: OwnedProxy, endpoint: str) -> None: + async with _request(proxy, _Mode.CONTENT, endpoint) as response: + pending: Final = await _next(backend, _Mode.CONTENT) + await _headers(pending) + received: Final = await asyncio.wait_for(response.readuntil(b'"hello"'), _DEADLINE) + assert b"200 OK" in received + if endpoint == "/v1/messages": + assert b"content_block_delta" in received + await _released(pending) + assert backend.requests.empty(), "Disconnect must not trigger another upstream request" + + +async def test_messages_stream_completes_with_unchanged_events(backend: _Backend, proxy: OwnedProxy) -> None: + async with httpx.AsyncClient(base_url=str(proxy.gateway.client.base_url), trust_env=False) as client: + response_task: Final = asyncio.create_task( + client.post( + "/v1/messages", + headers={"Authorization": f"Bearer {_KEY}", "Content-Type": "application/json"}, + content=_body(_Mode.COMPLETE), + ) + ) + pending: Final = await _next(backend, _Mode.COMPLETE) + await _headers(pending) + response: Final = await asyncio.wait_for(response_task, _DEADLINE) + assert response.status_code == 200, response.text + events: Final = parse_sse(response.text) + assert event_types(events) == LIFECYCLE + assert delta_text(events) == "hello" + assert pending.released.is_set() + assert backend.requests.empty(), "Successful streaming must not trigger another upstream request" diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_sse_stream.py b/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_sse_stream.py index fbbbc579d94..ec57636a17b 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_sse_stream.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_sse_stream.py @@ -8,8 +8,11 @@ wrapper's chunks/messages/model so disconnect-time partial billing can read them from typing import Final from unittest.mock import MagicMock +import httpx import pytest +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( AnthropicSSEStream, AnthropicStreamWrapper, @@ -85,3 +88,67 @@ def test_sse_stream_exposes_chunks_messages_and_model(): assert stream.messages == [{"role": "user", "content": "hi"}] chunks: Final = stream.chunks assert isinstance(chunks, list) and len(chunks) == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("started", [False, True]) +async def test_sse_stream_close_releases_http_response_before_upstream_body(started: bool) -> None: + response: Final = httpx.Response(200, stream=httpx.ByteStream(b"unread upstream body")) + upstream: Final = CustomStreamWrapper( + completion_stream=response, + model="test-model", + custom_llm_provider="openai", + logging_obj=Logging( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="acompletion", + start_time=1.0, + litellm_call_id="disconnect-test", + function_id="test", + ), + ) + stream: Final = AnthropicSSEStream(AnthropicStreamWrapper(completion_stream=upstream, model="test-model")) + if started: + assert (await anext(stream)).startswith(b"event: message_start") + assert not response.is_stream_consumed + assert not response.is_closed + + await stream.aclose() + await stream.aclose() + + assert response.is_closed + assert not response.is_stream_consumed + with pytest.raises(StopAsyncIteration): + await anext(stream) + + +class _CloseableAsyncStream(_AsyncStream): + def __init__(self, items: list[MagicMock]) -> None: + super().__init__(items) + self.close_calls = 0 + + async def aclose(self) -> None: + self.close_calls += 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("finish_stream", [False, True]) +async def test_sse_stream_releases_upstream_once_after_content(finish_stream: bool) -> None: + upstream: Final = _CloseableAsyncStream( + [_make_chunk(Delta(content="hello")), _make_chunk(Delta(), finish_reason="stop")] + ) + stream: Final = AnthropicSSEStream(AnthropicStreamWrapper(completion_stream=upstream, model="test-model")) + events: Final[list[bytes]] = [] + async for event in stream: + events.append(event) + if not finish_stream and b"event: content_block_delta" in event: + break + assert any(b'"text":"hello"' in event.replace(b" ", b"") for event in events) + if finish_stream: + assert events[-1].startswith(b"event: message_stop") + + await stream.aclose() + await stream.aclose() + + assert upstream.close_calls == 1 From 5367855644b099170460a6ec9161be6effe0cd8f Mon Sep 17 00:00:00 2001 From: jesus-berri Date: Sat, 24 Oct 2026 17:09:55 -0700 Subject: [PATCH 034/155] feat(teams): show caller member spend and budget on Your Teams (#45674) * feat(teams): show caller member spend and budget on Your Teams Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(teams): use member-linked reset dates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(teams): keep type-discipline suppressions on cast lines Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(teams): preserve empty page list response Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(teams): keep the member reset date inside the spend column Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(teams): match email-only roster members for caller membership Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): show team member budget reset date in UTC Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(teams): address review on caller membership enrichment Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(ui): label the caller's team spend line My Spend Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/auth_checks.py | 52 ++- .../management_endpoints/team_endpoints.py | 241 ++++++++-- .../management_endpoints/team_endpoints.py | 12 + .../management/test_team_list_v2.py | 32 +- .../proxy/auth/test_team_member_budget.py | 55 ++- .../test_team_endpoints.py | 435 ++++++++++++++++-- .../TeamMemberSpendBudgetCell.test.tsx | 90 ++++ .../TeamsPage/TeamMemberSpendBudgetCell.tsx | 70 +++ .../TeamsPage/TeamsTable.integration.test.tsx | 24 +- .../components/TeamsPage/teamTableColumns.tsx | 12 +- .../components/key_team_helpers/key_list.tsx | 2 + .../shared/table_cells/spend_budget_cell.tsx | 4 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 16 + .../src/utils/budgetUtils.test.ts | 23 + ui/litellm-dashboard/src/utils/budgetUtils.ts | 10 + 15 files changed, 987 insertions(+), 91 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/TeamsPage/TeamMemberSpendBudgetCell.test.tsx create mode 100644 ui/litellm-dashboard/src/components/TeamsPage/TeamMemberSpendBudgetCell.tsx create mode 100644 ui/litellm-dashboard/src/utils/budgetUtils.test.ts diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 06c53afcf2c..79e385c376a 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -15,6 +15,7 @@ import re import time from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from dataclasses import dataclass +from datetime import datetime from functools import partial from itertools import chain from types import MappingProxyType @@ -1799,6 +1800,20 @@ async def get_team_member_default_budget( return budget +def resolve_effective_team_member_budget( + member_budget: LiteLLM_BudgetTable | None, + team_default_budget: LiteLLM_BudgetTable | None, + now: datetime, +) -> float | None: + if member_budget is not None and member_budget.max_budget is not None: + return member_budget.effective_max_budget(now) + if team_default_budget is None or team_default_budget.max_budget is None or team_default_budget.max_budget <= 0: + return None + return team_default_budget.max_budget + ( + member_budget.active_temp_budget_increase(now) if member_budget is not None else 0.0 + ) + + async def resolve_default_end_user_budget( prisma_client: PrismaClient, user_api_key_cache: UserApiKeyCache, @@ -7034,29 +7049,24 @@ async def _check_team_member_budget( # Per-member override wins; otherwise fall back to the team-level # default configured via team.metadata["team_member_budget_id"]. - team_member_budget: float | None = None member_budget_row: Final = loaded_membership.litellm_budget_table if loaded_membership is not None else None now: Final = get_utc_datetime() - if member_budget_row is not None and member_budget_row.max_budget is not None: - team_member_budget = member_budget_row.effective_max_budget(now=now) - else: - default_budget_id: Final = (team_object.metadata or {}).get("team_member_budget_id") - if isinstance(default_budget_id, str): - default_budget: Final = await get_team_member_default_budget( - budget_id=default_budget_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - ) - # Treat 0 on the team default as "no cap". - # Per-member rows still respect 0 as an explicit admin disable. - if ( - default_budget is not None - and default_budget.max_budget is not None - and default_budget.max_budget > 0 - ): - team_member_budget = default_budget.max_budget + ( - member_budget_row.active_temp_budget_increase(now=now) if member_budget_row is not None else 0.0 - ) + default_budget_id: Final = (team_object.metadata or {}).get("team_member_budget_id") + team_default_budget: Final = ( + await get_team_member_default_budget( + budget_id=default_budget_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + ) + if (member_budget_row is None or member_budget_row.max_budget is None) + and isinstance(default_budget_id, str) + else None + ) + team_member_budget: Final = resolve_effective_team_member_budget( + member_budget=member_budget_row, + team_default_budget=team_default_budget, + now=now, + ) if team_member_budget is not None: team_member_spend = (loaded_membership.spend if loaded_membership is not None else 0.0) or 0.0 diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 361778ab2ea..95d4d03a7f0 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -52,6 +52,7 @@ from litellm.proxy._types import ( CommonProxyErrors, DeleteTeamRequest, LiteLLM_AuditLogs, + LiteLLM_BudgetTable, LiteLLM_DeletedTeamTable, Litellm_EntityType, LiteLLM_ManagementEndpoint_MetadataFields, @@ -114,6 +115,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo get_user_object, invalidate_team_member_spend_state, jwt_key_mapping_cache_key, + resolve_effective_team_member_budget, ) from litellm.proxy.auth.auth_utils import ( enforce_batch_limits_are_admin_only, @@ -244,6 +246,7 @@ from litellm.types.proxy.management_endpoints.team_endpoints import ( BulkUpdateTeamMemberPermissionsResponse, GetTeamMemberPermissionsResponse, TeamIdSearchMatch, + TeamListCallerMembership, TeamListItem, TeamListResponse, TeamMemberAddResult, @@ -254,6 +257,7 @@ from litellm.types.proxy.management_endpoints.team_endpoints import ( UpdateTeamMemberPermissionsRequest, ) from litellm.types.utils import BudgetConfig, GenericBudgetConfigType +from litellm.utils import get_utc_datetime if TYPE_CHECKING: from prisma import Prisma @@ -302,6 +306,34 @@ class _TeamIdRow(Protocol): def team_id(self) -> str: ... +class _TeamListBudgetRow(Protocol): + @property + def budget_id(self) -> str: ... + + @property + def budget_reset_at(self) -> datetime | None: ... + + @property + def max_budget(self) -> float | None: ... + + @property + def temp_budget_expiry(self) -> datetime | None: ... + + @property + def temp_budget_increase(self) -> float | None: ... + + +class _TeamListMembershipRow(Protocol): + @property + def team_id(self) -> str: ... + + @property + def spend(self) -> float | None: ... + + @property + def litellm_budget_table(self) -> _TeamListBudgetRow | None: ... + + class _CacheableTeamRow(_TeamIdRow, _ModelDumpRow, Protocol): ... @@ -5467,16 +5499,15 @@ async def team_member_me( user_api_key_cache=user_api_key_cache, ) - caller_user_email: Final = user_api_key_dict.user_email - member_role: str | None = None - for m in team_table.members_with_roles: - # Match by user_id when present, else fall back to email — members - # added by email may have user_id=None on the stored entry. - if (m.user_id is not None and m.user_id == caller_user_id) or ( - m.user_email is not None and caller_user_email is not None and m.user_email == caller_user_email - ): - member_role = m.role - break + caller_identity: Final = _TeamListCallerIdentity( + user_id=caller_user_id, + user_email=user_api_key_dict.user_email, + ) + matched_member: Final = next( + (member for member in team_table.members_with_roles if _team_member_matches_caller(member, caller_identity)), + None, + ) + member_role: Final = matched_member.role if matched_member is not None else None if member_role is None: # Caller is not a member of this team. Even proxy admins get 404 here — @@ -5886,11 +5917,16 @@ def _convert_teams_to_response_models( members_count = len(members_with_roles) keys_count = counts.get(team_dict.get("team_id") or "", 0) team_list.append( - TeamListItem( - **team_dict, - members_count=members_count, - keys_count=keys_count, - access_group_skill_names=None, + TeamListItem.model_validate( + cast( # cast-ok: TeamListItem validates the Prisma model dump + object, + { + **team_dict, + "members_count": members_count, + "keys_count": keys_count, + "access_group_skill_names": None, + }, + ) ) ) return team_list @@ -5921,6 +5957,137 @@ async def _get_keys_count_by_team( return {row["team_id"]: row.get("_count", {}).get("team_id", 0) for row in grouped if row.get("team_id")} +def _team_list_budget_model(budget_row: _TeamListBudgetRow | None) -> LiteLLM_BudgetTable | None: + if budget_row is None: + return None + return LiteLLM_BudgetTable( + max_budget=budget_row.max_budget, + temp_budget_expiry=budget_row.temp_budget_expiry, + temp_budget_increase=budget_row.temp_budget_increase, + ) + + +@dataclass(frozen=True, slots=True) +class _TeamListCallerIdentity: + user_id: str | None + user_email: str | None + + +def _team_member_matches_caller(member: Member, caller_identity: _TeamListCallerIdentity) -> bool: + return (member.user_id is not None and member.user_id == caller_identity.user_id) or ( + member.user_email is not None + and caller_identity.user_email is not None + and member.user_email == caller_identity.user_email + ) + + +def _caller_in_roster(team: TeamListItem, caller_identity: _TeamListCallerIdentity) -> bool: + return any(_team_member_matches_caller(member, caller_identity) for member in team.members_with_roles) + + +def _team_member_default_budget_id( + team: TeamListItem, + caller_identity: _TeamListCallerIdentity, + memberships_by_team: Mapping[str, _TeamListMembershipRow], +) -> str | None: + membership_row: Final = memberships_by_team.get(team.team_id) + if membership_row is None and not _caller_in_roster(team, caller_identity): + return None + member_budget: Final = _team_list_budget_model( + membership_row.litellm_budget_table if membership_row is not None else None + ) + if member_budget is not None and member_budget.max_budget is not None: + return None + metadata: Final = cast( # cast-ok: team metadata is stored as a JSON object + "dict[str, object] | None", team.metadata + ) + if metadata is None: + return None + budget_id: Final = metadata.get("team_member_budget_id") + return budget_id if isinstance(budget_id, str) else None + + +def _team_list_caller_membership( + team: TeamListItem, + caller_identity: _TeamListCallerIdentity, + memberships_by_team: Mapping[str, _TeamListMembershipRow], + default_budget_id: str | None, + budgets_by_id: Mapping[str, _TeamListBudgetRow], + now: datetime, +) -> TeamListCallerMembership | None: + membership_row: Final = memberships_by_team.get(team.team_id) + if membership_row is None and not _caller_in_roster(team, caller_identity): + return None + member_budget_row: Final = membership_row.litellm_budget_table if membership_row is not None else None + member_budget: Final = _team_list_budget_model(member_budget_row) + default_budget_row: Final = budgets_by_id.get(default_budget_id) if default_budget_id is not None else None + team_default_budget: Final = _team_list_budget_model(default_budget_row) + max_budget: Final = resolve_effective_team_member_budget( + member_budget=member_budget, + team_default_budget=team_default_budget, + now=now, + ) + budget_reset_at: Final = ( + member_budget_row.budget_reset_at if member_budget_row is not None and max_budget is not None else None + ) + return TeamListCallerMembership( + spend=(membership_row.spend if membership_row is not None else 0.0) or 0.0, + max_budget=max_budget, + budget_reset_at=budget_reset_at, + ) + + +async def _enrich_team_list_with_caller_membership( + prisma_client: PrismaClient, + teams: Sequence[TeamListItem], + caller_identity: _TeamListCallerIdentity, + now: datetime, +) -> tuple[TeamListItem, ...]: + caller_user_id: Final = caller_identity.user_id + if caller_user_id is None: + return tuple(team.model_copy(update={"caller_membership": None}) for team in teams) + if not teams: + return () + + page_team_ids: Final = tuple(team.team_id for team in teams) + membership_rows: Final[Sequence[_TeamListMembershipRow]] = await _team_membership_db(prisma_client).find_many( + # bounded-ok: page_team_ids is capped at 100 teams by list_team_v2 + where={"team_id": {"in": page_team_ids}, "user_id": caller_user_id}, + include={"litellm_budget_table": True}, + ) + memberships_by_team: Final = {row.team_id: row for row in membership_rows} + default_budget_id_by_team: Final = MappingProxyType( + {team.team_id: _team_member_default_budget_id(team, caller_identity, memberships_by_team) for team in teams} + ) + default_budget_ids: Final = tuple( + dict.fromkeys(budget_id for budget_id in default_budget_id_by_team.values() if budget_id is not None) + ) + default_budget_rows: Final[Sequence[_TeamListBudgetRow]] = ( + await _budget_db(prisma_client).find_many( + # bounded-ok: default_budget_ids is derived from this page of at most 100 teams + where={"budget_id": {"in": default_budget_ids}}, + ) + if default_budget_ids + else () + ) + budgets_by_id: Final = {row.budget_id: row for row in default_budget_rows} + return tuple( + team.model_copy( + update={ + "caller_membership": _team_list_caller_membership( + team=team, + caller_identity=caller_identity, + memberships_by_team=memberships_by_team, + default_budget_id=default_budget_id_by_team.get(team.team_id), + budgets_by_id=budgets_by_id, + now=now, + ) + } + ) + for team in teams + ) + + async def _enforce_list_team_v2_access( user_api_key_dict: UserAPIKeyAuth, user_id: str | None, @@ -6034,7 +6201,7 @@ async def list_team_v2( sort_order: str = fastapi.Query(default="asc", description="Sort order ('asc' or 'desc')"), status: str | None = fastapi.Query(default=None, description="Filter by status (e.g. 'deleted')"), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -): +) -> TeamListResponse: """ Get a paginated list of teams with filtering and sorting options. @@ -6109,13 +6276,13 @@ async def list_team_v2( ) if where_conditions is None: - return { - "teams": [], - "total": 0, - "page": page, - "page_size": page_size, - "total_pages": 0, - } + return TeamListResponse( + teams=[], + total=0, + page=page, + page_size=page_size, + total_pages=0, + ) from litellm.proxy.management_helpers.team_list_sorting import ( COMPUTED_TEAM_SORTS, @@ -6227,13 +6394,27 @@ async def list_team_v2( team_groups, tuple(ag_id for ag_id in (team_item.access_group_ids or []) if ag_id not in ag_lookup) ) - return { - "teams": team_list, - "total": total_count, - "page": page, - "page_size": page_size, - "total_pages": total_pages, - } + caller_identity: Final = _TeamListCallerIdentity( + user_id=user_api_key_dict.user_id, + user_email=user_api_key_dict.user_email, + ) + team_list_with_caller_membership: Final = ( + await _enrich_team_list_with_caller_membership( + prisma_client=prisma_client, + teams=tuple(team for team in team_list if isinstance(team, TeamListItem)), + caller_identity=caller_identity, + now=get_utc_datetime(), + ) + if not use_deleted_table and team_list + else team_list + ) + return TeamListResponse( + teams=list(team_list_with_caller_membership), + total=total_count, + page=page, + page_size=page_size, + total_pages=total_pages, + ) async def _authorize_and_filter_teams( diff --git a/litellm/types/proxy/management_endpoints/team_endpoints.py b/litellm/types/proxy/management_endpoints/team_endpoints.py index 51c48e6ebf9..742842bd028 100644 --- a/litellm/types/proxy/management_endpoints/team_endpoints.py +++ b/litellm/types/proxy/management_endpoints/team_endpoints.py @@ -1,4 +1,5 @@ from collections.abc import Mapping +from datetime import datetime from typing import Final, Literal from pydantic import ConfigDict, Field, field_validator, model_validator @@ -83,11 +84,22 @@ class BulkUpdateTeamMemberPermissionsResponse(LiteLLMBaseModel): permissions_appended: list[str] | None = None +class TeamListCallerMembership(LiteLLMBaseModel): + """The calling user's own spend and effective member budget in a team.""" + + spend: float = 0.0 + max_budget: float | None = None + budget_reset_at: datetime | None = None + + model_config = ConfigDict(frozen=True) + + class TeamListItem(LiteLLM_TeamTable): """A team item in the paginated list response, enriched with computed fields.""" members_count: int = 0 keys_count: int = 0 + caller_membership: TeamListCallerMembership | None = None # Resources inherited from access groups (separate from direct assignments) access_group_models: list[str] | None = None access_group_mcp_server_ids: list[str] | None = None diff --git a/tests/integration/management/test_team_list_v2.py b/tests/integration/management/test_team_list_v2.py index d608d054daa..4c9171310a6 100644 --- a/tests/integration/management/test_team_list_v2.py +++ b/tests/integration/management/test_team_list_v2.py @@ -1,8 +1,8 @@ -from typing import FrozenSet, Optional +from typing import Final, FrozenSet, Optional import pytest -from tests.integration._support.actors import Actor +from tests.integration._support.actors import Actor, ActorSet, GatewayClient pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -139,3 +139,31 @@ async def test_team_list_v2_invalid_status_is_400(gateway_client, actors): headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, ) assert resp.status_code == 400, resp.text + + +async def test_team_list_v2_returns_caller_membership_only_for_members( + gateway_client: GatewayClient, + actors: ActorSet, +) -> None: + member: Final = actors.keys[Actor.TEAM_ADMIN] + member_response: Final = await gateway_client.get( + f"/v2/team/list?team_id={actors.team_alpha_id}&user_id={member.user_id}", + headers={"Authorization": f"Bearer {member.cleartext}"}, + ) + + assert member_response.status_code == 200, member_response.text + member_team: Final = member_response.json()["teams"][0] + assert member_team["caller_membership"] == { + "spend": 0.0, + "max_budget": None, + "budget_reset_at": None, + } + + admin: Final = actors.keys[Actor.PROXY_ADMIN] + admin_response: Final = await gateway_client.get( + f"/v2/team/list?team_id={actors.team_alpha_id}", + headers={"Authorization": f"Bearer {admin.cleartext}"}, + ) + + assert admin_response.status_code == 200, admin_response.text + assert admin_response.json()["teams"][0]["caller_membership"] is None diff --git a/tests/unit/proxy/auth/test_team_member_budget.py b/tests/unit/proxy/auth/test_team_member_budget.py index b38a953d189..6055e4f0356 100644 --- a/tests/unit/proxy/auth/test_team_member_budget.py +++ b/tests/unit/proxy/auth/test_team_member_budget.py @@ -3,8 +3,11 @@ Unit tests for team member budget checks in common_checks. These tests verify the team member budget enforcement without requiring a proxy server. """ -import pytest +from datetime import datetime, timezone +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch + +import pytest from fastapi import Request import litellm @@ -15,7 +18,10 @@ from litellm.proxy._types import ( LiteLLM_UserTable, UserAPIKeyAuth, ) -from litellm.proxy.auth.auth_checks import common_checks, get_team_membership +from litellm.proxy.auth.auth_checks import ( + common_checks, + resolve_effective_team_member_budget, +) @pytest.mark.asyncio @@ -436,3 +442,48 @@ async def test_team_member_budget_check_personal_key_not_team(): # Should pass and get_team_membership should not be called assert result is True mock_get_team_membership.assert_not_called() + + +def test_resolve_effective_team_member_budget_preserves_member_and_default_precedence() -> None: + now: Final = datetime(2030, 1, 1, tzinfo=timezone.utc) + member_budget: Final = LiteLLM_BudgetTable( + max_budget=100, + temp_budget_increase=10, + temp_budget_expiry=datetime(2030, 1, 2, tzinfo=timezone.utc), + ) + expired_member_budget: Final = LiteLLM_BudgetTable( + max_budget=100, + temp_budget_increase=10, + temp_budget_expiry=datetime(2029, 12, 31, tzinfo=timezone.utc), + ) + active_inherited_increase: Final = LiteLLM_BudgetTable( + max_budget=None, + temp_budget_increase=10, + temp_budget_expiry=datetime(2030, 1, 2, tzinfo=timezone.utc), + ) + expired_inherited_increase: Final = LiteLLM_BudgetTable( + max_budget=None, + temp_budget_increase=10, + temp_budget_expiry=datetime(2029, 12, 31, tzinfo=timezone.utc), + ) + + assert resolve_effective_team_member_budget(member_budget, None, now) == 110 + assert resolve_effective_team_member_budget(expired_member_budget, None, now) == 100 + assert resolve_effective_team_member_budget( + active_inherited_increase, + LiteLLM_BudgetTable(max_budget=50), + now, + ) == 60 + assert resolve_effective_team_member_budget( + expired_inherited_increase, + LiteLLM_BudgetTable(max_budget=50), + now, + ) == 50 + assert resolve_effective_team_member_budget( + LiteLLM_BudgetTable(max_budget=0), + LiteLLM_BudgetTable(max_budget=50), + now, + ) == 0 + assert resolve_effective_team_member_budget(None, LiteLLM_BudgetTable(max_budget=50), now) == 50 + assert resolve_effective_team_member_budget(None, LiteLLM_BudgetTable(max_budget=0), now) is None + assert resolve_effective_team_member_budget(None, None, now) is None diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index 37028558af3..4feb7fb5bc8 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -54,6 +54,8 @@ from litellm.proxy.management_endpoints.team_endpoints import ( _LOCK_DELETED_TEAM_MEMBERS_SQL, _STRIP_DELETED_TEAMS_FROM_USERS_SQL, DeletedTeams, + _enrich_team_list_with_caller_membership, + _TeamListCallerIdentity, GetTeamMemberPermissionsResponse, UpdateTeamMemberPermissionsRequest, _build_team_list_where_conditions, @@ -91,6 +93,8 @@ from litellm.router import Router from litellm.types.proxy.management_endpoints.team_endpoints import ( BulkTeamMemberAddRequest, BulkTeamMemberAddResponse, + TeamListCallerMembership, + TeamListItem, TeamMemberAddResult, ) from litellm.types.utils import StandardAuditLogPayload @@ -4172,6 +4176,7 @@ async def test_list_team_v2_security_check_non_admin_user_own_teams(): ] mock_db.litellm_teamtable.find_many = AsyncMock(return_value=mock_teams) mock_db.litellm_teamtable.count = AsyncMock(return_value=2) + mock_db.litellm_teammembership.find_many = AsyncMock(return_value=[]) mock_db.litellm_verificationtoken.group_by = AsyncMock(return_value=[]) with patch( @@ -4191,9 +4196,8 @@ async def test_list_team_v2_security_check_non_admin_user_own_teams(): ) # Should return results without error - assert "teams" in result - assert "total" in result - assert result["total"] == 2 + assert len(result.teams) == 2 + assert result.total == 2 @pytest.mark.asyncio @@ -4229,6 +4233,7 @@ async def test_list_team_v2_security_check_admin_user(): ] mock_db.litellm_teamtable.find_many = AsyncMock(return_value=mock_teams) mock_db.litellm_teamtable.count = AsyncMock(return_value=2) + mock_db.litellm_teammembership.find_many = AsyncMock(return_value=[]) mock_db.litellm_verificationtoken.group_by = AsyncMock(return_value=[]) # Should NOT raise an exception @@ -4242,9 +4247,8 @@ async def test_list_team_v2_security_check_admin_user(): ) # Should return results without error - assert "teams" in result - assert "total" in result - assert result["total"] == 2 + assert len(result.teams) == 2 + assert result.total == 2 @pytest.mark.asyncio @@ -4310,10 +4314,8 @@ async def test_list_team_v2_with_status_deleted(): mock_db.litellm_teamtable.count.assert_not_called() # Should return results without error - assert "teams" in result - assert "total" in result - assert result["total"] == 2 - assert len(result["teams"]) == 2 + assert len(result.teams) == 2 + assert result.total == 2 @pytest.mark.asyncio @@ -4372,6 +4374,7 @@ async def test_list_team_v2_includes_litellm_model_table(): side_effect=lambda **kw: [_team_row("team_1", kw.get("include"))] ) mock_db.litellm_teamtable.count = AsyncMock(return_value=1) + mock_db.litellm_teammembership.find_many = AsyncMock(return_value=[]) mock_db.litellm_verificationtoken.group_by = AsyncMock(return_value=[]) result = await list_team_v2( @@ -4383,8 +4386,8 @@ async def test_list_team_v2_includes_litellm_model_table(): status=None, ) - assert result["teams"][0].litellm_model_table is not None - assert result["teams"][0].litellm_model_table.model_aliases == {"my-fast-model": "fake-model"} + assert result.teams[0].litellm_model_table is not None + assert result.teams[0].litellm_model_table.model_aliases == {"my-fast-model": "fake-model"} mock_db.litellm_deletedteamtable.find_many = AsyncMock( side_effect=lambda **kw: [_team_row("team_2", kw.get("include"))] @@ -4465,6 +4468,7 @@ async def test_list_team_v2_org_admin_sees_org_teams(): } mock_db.litellm_teamtable.find_many = AsyncMock(return_value=[mock_team]) mock_db.litellm_teamtable.count = AsyncMock(return_value=1) + mock_db.litellm_teammembership.find_many = AsyncMock(return_value=[]) mock_db.litellm_verificationtoken.group_by = AsyncMock(return_value=[]) mock_db.litellm_usertable.find_unique = AsyncMock(return_value=mock_user) @@ -4482,9 +4486,9 @@ async def test_list_team_v2_org_admin_sees_org_teams(): status=None, ) - assert result["total"] == 1 - assert len(result["teams"]) == 1 - assert result["teams"][0].members_count == 1 + assert result.total == 1 + assert len(result.teams) == 1 + assert result.teams[0].members_count == 1 # Verify org-scoped where clause where = mock_db.litellm_teamtable.find_many.call_args.kwargs["where"] @@ -4565,6 +4569,7 @@ async def test_list_team_v2_org_admin_own_user_id_sees_all_org_teams(): return_value=[mock_team_1, mock_team_2] ) mock_db.litellm_teamtable.count = AsyncMock(return_value=2) + mock_db.litellm_teammembership.find_many = AsyncMock(return_value=[]) mock_db.litellm_verificationtoken.group_by = AsyncMock(return_value=[]) mock_db.litellm_usertable.find_unique = AsyncMock(return_value=mock_user) @@ -4583,8 +4588,8 @@ async def test_list_team_v2_org_admin_own_user_id_sees_all_org_teams(): status=None, ) - assert result["total"] == 2 - assert len(result["teams"]) == 2 + assert result.total == 2 + assert len(result.teams) == 2 # Verify the where clause scopes by org OR own membership — no # top-level team_id filter that would hide org teams they aren't in @@ -4689,6 +4694,7 @@ async def test_list_team_v2_org_admin_own_query_keeps_memberships_in_other_orgs( prisma_client = MagicMock() prisma_client.db.litellm_teamtable.find_many = AsyncMock(side_effect=find_many) prisma_client.db.litellm_teamtable.count = AsyncMock(side_effect=count) + prisma_client.db.litellm_teammembership.find_many = AsyncMock(return_value=[]) prisma_client.db.litellm_verificationtoken.group_by = AsyncMock(return_value=[]) prisma_client.writer_db.litellm_usertable.find_unique = AsyncMock( return_value=LiteLLM_UserTable( @@ -4721,8 +4727,8 @@ async def test_list_team_v2_org_admin_own_query_keeps_memberships_in_other_orgs( sort_order="asc", status=None, ) - assert result["total"] == len(result["teams"]) - return [t.team_id for t in result["teams"]] + assert result.total == len(result.teams) + return [t.team_id for t in result.teams] own_view = ["team_in_org_A", "other_team_in_org_A", "team_in_org_B"] assert await list_teams("org_admin_user") == own_view @@ -4950,6 +4956,7 @@ async def test_list_team_v2_org_admin_with_user_id_returns_user_teams(): } mock_db.litellm_teamtable.find_many = AsyncMock(return_value=[mock_team]) mock_db.litellm_teamtable.count = AsyncMock(return_value=1) + mock_db.litellm_teammembership.find_many = AsyncMock(return_value=[]) mock_db.litellm_verificationtoken.group_by = AsyncMock(return_value=[]) result = await list_team_v2( @@ -4966,7 +4973,7 @@ async def test_list_team_v2_org_admin_with_user_id_returns_user_teams(): status=None, ) - assert result["total"] == 1 + assert result.total == 1 # Verify the where clause filters by user's teams AND org scope where = mock_db.litellm_teamtable.find_many.call_args.kwargs["where"] @@ -5219,6 +5226,7 @@ async def test_list_team_v2_populates_keys_count(): mock_db.litellm_teamtable.find_many = AsyncMock(return_value=[team_a, team_b]) mock_db.litellm_teamtable.count = AsyncMock(return_value=2) + mock_db.litellm_teammembership.find_many = AsyncMock(return_value=[]) mock_db.litellm_verificationtoken.group_by = AsyncMock( return_value=[ {"team_id": "team_a", "_count": {"team_id": 3}}, @@ -5235,16 +5243,391 @@ async def test_list_team_v2_populates_keys_count(): status=None, ) - assert result["total"] == 2 - by_id = {t.team_id: t for t in result["teams"]} + assert result.total == 2 + by_id = {t.team_id: t for t in result.teams} assert by_id["team_a"].keys_count == 3 assert by_id["team_b"].keys_count == 0 - # The aggregate is one batched query, filtered by the page's team IDs. group_by_kwargs = mock_db.litellm_verificationtoken.group_by.call_args.kwargs assert group_by_kwargs["by"] == ["team_id"] assert group_by_kwargs["where"] == {"team_id": {"in": ["team_a", "team_b"]}} assert group_by_kwargs["count"] == {"team_id": True} + membership_where: Final = mock_db.litellm_teammembership.find_many.call_args.kwargs["where"] + assert membership_where == { + "team_id": {"in": ("team_a", "team_b")}, + "user_id": "admin_user_123", + } + + +def _team_list_prisma( + membership_rows: Sequence[LiteLLM_TeamMembership], + budget_rows: Sequence[LiteLLM_BudgetTableFull], +) -> tuple[MagicMock, AsyncMock, AsyncMock]: + prisma_client: Final = MagicMock() + membership_query: Final = AsyncMock(return_value=tuple(membership_rows)) + budget_query: Final = AsyncMock(return_value=tuple(budget_rows)) + prisma_client.db.litellm_teammembership.find_many = membership_query + prisma_client.db.litellm_budgettable.find_many = budget_query + return prisma_client, membership_query, budget_query + + +def _team_list_member( + team_id: str, + user_id: str = "caller", + default_budget_id: str | None = None, +) -> TeamListItem: + metadata: Final = {"team_member_budget_id": default_budget_id} if default_budget_id is not None else None + return TeamListItem( + team_id=team_id, + metadata=metadata, + members_with_roles=[Member(user_id=user_id, role="user")], + ) + + +def _team_list_caller_identity( + user_id: str | None = "caller", + user_email: str | None = None, +) -> _TeamListCallerIdentity: + return _TeamListCallerIdentity(user_id=user_id, user_email=user_email) + + +def _budget_table( + budget_id: str, + max_budget: float | None, + budget_reset_at: datetime | None = None, + temp_budget_increase: float | None = None, + temp_budget_expiry: datetime | None = None, +) -> LiteLLM_BudgetTableFull: + return LiteLLM_BudgetTableFull( + budget_id=budget_id, + max_budget=max_budget, + budget_reset_at=budget_reset_at, + created_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + temp_budget_increase=temp_budget_increase, + temp_budget_expiry=temp_budget_expiry, + ) + + +@pytest.mark.asyncio +async def test_team_list_caller_membership_uses_member_spend_budget_and_reset_date() -> None: + reset_at: Final = datetime(2026, 10, 20, tzinfo=timezone.utc) + member_budget: Final = _budget_table("member-budget", 100, reset_at) + membership: Final = LiteLLM_TeamMembership( + user_id="caller", + team_id="team-a", + spend=50, + litellm_budget_table=member_budget, + ) + prisma_client, _, budget_query = _team_list_prisma([membership], []) + + teams: Final = await _enrich_team_list_with_caller_membership( + prisma_client, + [_team_list_member("team-a")], + _team_list_caller_identity(), + datetime(2026, 10, 1, tzinfo=timezone.utc), + ) + + assert teams[0].caller_membership == TeamListCallerMembership( + spend=50, + max_budget=100, + budget_reset_at=reset_at, + ) + budget_query.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_team_list_caller_membership_includes_roster_members_without_membership_rows() -> None: + reset_at: Final = datetime(2026, 10, 20, tzinfo=timezone.utc) + prisma_client, membership_query, budget_query = _team_list_prisma( + (), + [_budget_table("team-budget", 50, reset_at)], + ) + + teams: Final = await _enrich_team_list_with_caller_membership( + prisma_client, + [_team_list_member("team-a", default_budget_id="team-budget")], + _team_list_caller_identity(), + datetime(2026, 10, 1, tzinfo=timezone.utc), + ) + + assert teams[0].caller_membership == TeamListCallerMembership( + spend=0.0, + max_budget=50, + budget_reset_at=None, + ) + membership_query.assert_awaited_once() + budget_query.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_team_list_caller_membership_matches_email_only_roster_member() -> None: + reset_at: Final = datetime(2026, 10, 20, tzinfo=timezone.utc) + team: Final = TeamListItem( + team_id="team-a", + metadata={"team_member_budget_id": "team-budget"}, + members_with_roles=[Member(user_id=None, user_email="caller@x.com", role="user")], + ) + prisma_client, membership_query, budget_query = _team_list_prisma( + (), + [_budget_table("team-budget", 50, reset_at)], + ) + + teams: Final = await _enrich_team_list_with_caller_membership( + prisma_client, + [team], + _team_list_caller_identity(user_email="caller@x.com"), + datetime(2026, 10, 1, tzinfo=timezone.utc), + ) + + assert teams[0].caller_membership == TeamListCallerMembership( + spend=0.0, + max_budget=50, + budget_reset_at=None, + ) + membership_query.assert_awaited_once() + budget_query.assert_awaited_once() + + +@pytest.mark.parametrize("caller_user_email", ("other@x.com", None)) +@pytest.mark.asyncio +async def test_team_list_caller_membership_ignores_email_only_roster_member_without_matching_email( + caller_user_email: str | None, +) -> None: + team: Final = TeamListItem( + team_id="team-a", + metadata={"team_member_budget_id": "team-budget"}, + members_with_roles=[Member(user_id=None, user_email="caller@x.com", role="user")], + ) + prisma_client, membership_query, budget_query = _team_list_prisma( + (), + [_budget_table("team-budget", 50)], + ) + + teams: Final = await _enrich_team_list_with_caller_membership( + prisma_client, + [team], + _team_list_caller_identity(user_email=caller_user_email), + datetime(2026, 10, 1, tzinfo=timezone.utc), + ) + + assert teams[0].caller_membership is None + membership_query.assert_awaited_once() + budget_query.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_team_list_caller_membership_uses_team_default_without_member_reset_date() -> None: + reset_at: Final = datetime(2026, 10, 20, tzinfo=timezone.utc) + membership: Final = LiteLLM_TeamMembership(user_id="caller", team_id="team-a", spend=12) + prisma_client, _, budget_query = _team_list_prisma( + [membership], + [_budget_table("team-budget", 50, reset_at)], + ) + + teams: Final = await _enrich_team_list_with_caller_membership( + prisma_client, + [_team_list_member("team-a", default_budget_id="team-budget")], + _team_list_caller_identity(), + datetime(2026, 10, 1, tzinfo=timezone.utc), + ) + + assert teams[0].caller_membership == TeamListCallerMembership( + spend=12, + max_budget=50, + budget_reset_at=None, + ) + budget_query.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_team_list_caller_membership_uses_reset_date_from_member_linked_default_budget() -> None: + reset_at: Final = datetime(2026, 10, 20, tzinfo=timezone.utc) + member_budget: Final = _budget_table("team-budget", 50, reset_at) + membership: Final = LiteLLM_TeamMembership( + user_id="caller", + team_id="team-a", + litellm_budget_table=member_budget, + ) + prisma_client, _, budget_query = _team_list_prisma([membership], []) + + teams: Final = await _enrich_team_list_with_caller_membership( + prisma_client, + [_team_list_member("team-a", default_budget_id="team-budget")], + _team_list_caller_identity(), + datetime(2026, 10, 1, tzinfo=timezone.utc), + ) + + assert teams[0].caller_membership == TeamListCallerMembership( + spend=0.0, + max_budget=50, + budget_reset_at=reset_at, + ) + budget_query.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_team_list_caller_membership_ignores_zero_team_default_budget() -> None: + membership: Final = LiteLLM_TeamMembership(user_id="caller", team_id="team-a") + prisma_client, _, _ = _team_list_prisma( + [membership], + [_budget_table("team-budget", 0, datetime(2026, 10, 20, tzinfo=timezone.utc))], + ) + + teams: Final = await _enrich_team_list_with_caller_membership( + prisma_client, + [_team_list_member("team-a", default_budget_id="team-budget")], + _team_list_caller_identity(), + datetime(2026, 10, 1, tzinfo=timezone.utc), + ) + + assert teams[0].caller_membership == TeamListCallerMembership(spend=0.0, max_budget=None, budget_reset_at=None) + + +@pytest.mark.parametrize( + ("expiry", "expected_budget"), + ( + (datetime(2030, 1, 2, tzinfo=timezone.utc), 110), + (datetime(2029, 12, 31, tzinfo=timezone.utc), 100), + ), +) +@pytest.mark.asyncio +async def test_team_list_caller_membership_applies_only_active_temporary_increases( + expiry: datetime, + expected_budget: float, +) -> None: + member_budget: Final = _budget_table( + "member-budget", + 100, + datetime(2030, 2, 1, tzinfo=timezone.utc), + temp_budget_increase=10, + temp_budget_expiry=expiry, + ) + membership: Final = LiteLLM_TeamMembership( + user_id="caller", + team_id="team-a", + litellm_budget_table=member_budget, + ) + prisma_client, _, _ = _team_list_prisma([membership], []) + + teams: Final = await _enrich_team_list_with_caller_membership( + prisma_client, + [_team_list_member("team-a")], + _team_list_caller_identity(), + datetime(2030, 1, 1, tzinfo=timezone.utc), + ) + + assert teams[0].caller_membership is not None + assert teams[0].caller_membership.max_budget == expected_budget + assert teams[0].caller_membership.budget_reset_at == datetime(2030, 2, 1, tzinfo=timezone.utc) + + +@pytest.mark.parametrize( + "member_reset_at", + ( + datetime(2030, 1, 15, tzinfo=timezone.utc), + None, + ), +) +@pytest.mark.asyncio +async def test_team_list_caller_membership_uses_private_row_reset_with_inherited_budget( + member_reset_at: datetime | None, +) -> None: + team_reset_at: Final = datetime(2030, 2, 1, tzinfo=timezone.utc) + member_budget: Final = _budget_table( + "member-budget", + None, + member_reset_at, + temp_budget_increase=10, + temp_budget_expiry=datetime(2030, 1, 2, tzinfo=timezone.utc), + ) + membership: Final = LiteLLM_TeamMembership( + user_id="caller", + team_id="team-a", + litellm_budget_table=member_budget, + ) + prisma_client, _, _ = _team_list_prisma( + [membership], + [_budget_table("team-budget", 50, team_reset_at)], + ) + + teams: Final = await _enrich_team_list_with_caller_membership( + prisma_client, + [_team_list_member("team-a", default_budget_id="team-budget")], + _team_list_caller_identity(), + datetime(2030, 1, 1, tzinfo=timezone.utc), + ) + + assert teams[0].caller_membership == TeamListCallerMembership( + spend=0.0, + max_budget=60, + budget_reset_at=member_reset_at, + ) + + +@pytest.mark.asyncio +async def test_team_list_caller_membership_is_null_for_nonmember_admins() -> None: + prisma_client, membership_query, budget_query = _team_list_prisma([], []) + team: Final = TeamListItem( + team_id="team-a", + metadata={"team_member_budget_id": "team-budget"}, + members_with_roles=[Member(user_id="another-user", role="user")], + ) + + teams: Final = await _enrich_team_list_with_caller_membership( + prisma_client, + [team], + _team_list_caller_identity(), + datetime(2026, 10, 1, tzinfo=timezone.utc), + ) + + assert teams[0].caller_membership is None + membership_query.assert_awaited_once() + budget_query.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_team_list_caller_membership_skips_queries_without_caller_user_id() -> None: + prisma_client, membership_query, budget_query = _team_list_prisma((), ()) + + teams: Final = await _enrich_team_list_with_caller_membership( + prisma_client, + [_team_list_member("team-a")], + _team_list_caller_identity(user_id=None), + datetime(2026, 10, 1, tzinfo=timezone.utc), + ) + + assert teams[0].caller_membership is None + membership_query.assert_not_awaited() + budget_query.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_team_list_caller_membership_batches_page_membership_and_budget_queries() -> None: + memberships: Final = ( + LiteLLM_TeamMembership(user_id="caller", team_id="team-a"), + LiteLLM_TeamMembership(user_id="caller", team_id="team-b"), + ) + prisma_client, membership_query, budget_query = _team_list_prisma( + memberships, + [_budget_table("budget-a", 50), _budget_table("budget-b", 75)], + ) + + teams: Final = await _enrich_team_list_with_caller_membership( + prisma_client, + [ + _team_list_member("team-a", default_budget_id="budget-a"), + _team_list_member("team-b", default_budget_id="budget-b"), + ], + _team_list_caller_identity(), + datetime(2026, 10, 1, tzinfo=timezone.utc), + ) + + assert [team.caller_membership for team in teams] == [ + TeamListCallerMembership(spend=0.0, max_budget=50, budget_reset_at=None), + TeamListCallerMembership(spend=0.0, max_budget=75, budget_reset_at=None), + ] + membership_query.assert_awaited_once() + budget_query.assert_awaited_once() @pytest.mark.asyncio @@ -5282,8 +5665,8 @@ async def test_list_team_v2_keys_count_skipped_for_empty_page(): status=None, ) - assert result["total"] == 0 - assert result["teams"] == [] + assert result.total == 0 + assert result.teams == [] mock_db.litellm_verificationtoken.group_by.assert_not_called() @@ -5332,7 +5715,7 @@ async def test_list_team_v2_keys_count_skipped_for_deleted_status(): status="deleted", ) - assert result["total"] == 1 + assert result.total == 1 mock_db.litellm_verificationtoken.group_by.assert_not_called() diff --git a/ui/litellm-dashboard/src/components/TeamsPage/TeamMemberSpendBudgetCell.test.tsx b/ui/litellm-dashboard/src/components/TeamsPage/TeamMemberSpendBudgetCell.test.tsx new file mode 100644 index 00000000000..79e3be1750d --- /dev/null +++ b/ui/litellm-dashboard/src/components/TeamsPage/TeamMemberSpendBudgetCell.test.tsx @@ -0,0 +1,90 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import { TeamMemberSpendBudgetCell } from "./TeamMemberSpendBudgetCell"; + +describe("TeamMemberSpendBudgetCell", () => { + it("shows team and member budgets with the member reset date", () => { + render( + , + ); + + expect(screen.getByText("Team")).toBeInTheDocument(); + expect(screen.getByText("$600.00")).toBeInTheDocument(); + expect(screen.getByText("/ $1,000.00")).toBeInTheDocument(); + expect(screen.getByText("My Spend")).toBeInTheDocument(); + expect(screen.getByText("$50.00")).toBeInTheDocument(); + expect(screen.getByText("/ $100.00")).toBeInTheDocument(); + expect(screen.getByText("Resets Oct 20, 2026 UTC")).toBeInTheDocument(); + expect(screen.getAllByRole("meter")[0]).toHaveAttribute("aria-valuetext", "Team $600.00 of $1,000.00"); + expect(screen.getAllByRole("meter")[1]).toHaveAttribute("aria-valuetext", "My Spend $50.00 of $100.00"); + }); + + it("shows the team budget and Unlimited for a member without a budget", () => { + render( + , + ); + + expect(screen.getByText("Team")).toBeInTheDocument(); + expect(screen.getByText("$600.00")).toBeInTheDocument(); + expect(screen.getByText("My Spend")).toBeInTheDocument(); + expect(screen.getByText("/ Unlimited")).toBeInTheDocument(); + expect(screen.getAllByRole("meter")).toHaveLength(1); + }); + + it("omits the reset text when no member reset date exists", () => { + render( + , + ); + + expect(screen.queryByText(/Resets/)).not.toBeInTheDocument(); + }); + + it("renders only the team line when caller membership is absent", () => { + render(); + + expect(screen.getByText("Team")).toBeInTheDocument(); + expect(screen.queryByText("My Spend")).not.toBeInTheDocument(); + expect(screen.getAllByRole("meter")).toHaveLength(1); + }); + + it("exposes member spend above its budget through the meter", () => { + render( + , + ); + + expect(screen.getAllByRole("meter")[1]).toHaveAttribute("aria-valuetext", "My Spend $150.00 of $100.00"); + }); + + it("does not render meters for zero or unlimited budgets", () => { + render( + , + ); + + expect(screen.queryByRole("meter")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/TeamsPage/TeamMemberSpendBudgetCell.tsx b/ui/litellm-dashboard/src/components/TeamsPage/TeamMemberSpendBudgetCell.tsx new file mode 100644 index 00000000000..9bea9892899 --- /dev/null +++ b/ui/litellm-dashboard/src/components/TeamsPage/TeamMemberSpendBudgetCell.tsx @@ -0,0 +1,70 @@ +"use client"; + +import { Meter, MeterIndicator, MeterTrack } from "@/components/shared/Meter"; +import { getSpendBudgetMeterTone } from "@/components/shared/table_cells/spend_budget_cell"; +import type { components } from "@/lib/http/schema"; +import { formatBudgetResetUtc } from "@/utils/budgetUtils"; +import { formatNumberWithCommas, getSpendString } from "@/utils/dataUtils"; + +type CallerMembership = components["schemas"]["TeamListCallerMembership"]; + +interface TeamMemberSpendBudgetCellProps { + teamSpend: number | null | undefined; + teamMaxBudget: number | null | undefined; + callerMembership?: CallerMembership | null; +} + +interface BudgetLineProps { + label: "Team" | "My Spend"; + spend: number | null | undefined; + maxBudget: number | null | undefined; + resetAt?: string | null; +} + +function BudgetLine({ label, spend, maxBudget, resetAt }: BudgetLineProps) { + const spendValue = typeof spend === "number" && Number.isFinite(spend) ? spend : 0; + const budget = typeof maxBudget === "number" && Number.isFinite(maxBudget) ? maxBudget : null; + const spendText = spendValue === 0 ? "$0.00" : getSpendString(spendValue, 2); + const budgetText = budget === null ? "Unlimited" : `$${formatNumberWithCommas(budget, 2)}`; + const resetText = formatBudgetResetUtc(resetAt); + + return ( +
+
+ {label} + + {spendText}{" "} + / {budgetText} + +
+ {budget !== null && budget > 0 && ( + + + + + + )} + {resetText &&
Resets {resetText} UTC
} +
+ ); +} + +export function TeamMemberSpendBudgetCell({ + teamSpend, + teamMaxBudget, + callerMembership, +}: TeamMemberSpendBudgetCellProps) { + return ( +
+ + {callerMembership && ( + + )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.integration.test.tsx b/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.integration.test.tsx index def6750071f..02870bc1571 100644 --- a/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.integration.test.tsx @@ -112,10 +112,30 @@ it("renders a team row with alias, organization, and spend/budget", async () => expect(screen.getByText("Acme Team")).toBeInTheDocument(); expect(screen.getByText("Test Organization")).toBeInTheDocument(); expect(screen.getByText("$42.50")).toBeInTheDocument(); - expect(screen.getByText("of $100.00")).toBeInTheDocument(); + expect(screen.getByText("/ $100.00")).toBeInTheDocument(); }); }); +it("renders caller membership spend and budget in the team row", async () => { + mockUseTeamsTable.mockReturnValue( + teamsResult([ + { + ...mockTeam, + caller_membership: { + spend: 50, + max_budget: 100, + budget_reset_at: "2026-10-20T12:00:00Z", + }, + }, + ]), + ); + renderTable(); + + expect(await screen.findByText("My Spend")).toBeInTheDocument(); + expect(screen.getByText("$50.00")).toBeInTheDocument(); + expect(screen.getByText("Resets Oct 20, 2026 UTC")).toBeInTheDocument(); +}); + it("renders the Resources cell with member, model, and key counts", () => { renderTable(); @@ -154,7 +174,7 @@ describe("sort contract – only backend-sortable columns are sortable", () => { it("sorts by the backend team_alias field (not the label) when the Team header is clicked", async () => { renderTable(); - fireEvent.click(screen.getByText("Team").closest("button") as HTMLElement); + fireEvent.click(within(screen.getByRole("columnheader", { name: "Team" })).getByRole("button")); await waitFor(() => { expect(mockUseTeamsTable).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ sortBy: "team_alias" })); diff --git a/ui/litellm-dashboard/src/components/TeamsPage/teamTableColumns.tsx b/ui/litellm-dashboard/src/components/TeamsPage/teamTableColumns.tsx index e56e0d16522..abc66333f11 100644 --- a/ui/litellm-dashboard/src/components/TeamsPage/teamTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/TeamsPage/teamTableColumns.tsx @@ -4,7 +4,7 @@ import { ColumnDef } from "@tanstack/react-table"; import { Copy, KeyRound, Layers, MoreHorizontal, Pencil, Trash2, Users } from "lucide-react"; import { DataTableMultiSortHeader, DataTableSortHeader } from "@/components/shared/DataTable"; -import { DateCell, IdentityCell, SpendBudgetCell } from "@/components/shared/table_cells"; +import { DateCell, IdentityCell } from "@/components/shared/table_cells"; import { buttonVariants } from "@/components/ui/button"; import { DropdownMenu, @@ -20,6 +20,7 @@ import { copyToClipboard, formatNumberWithCommas } from "@/utils/dataUtils"; import { Team } from "../key_team_helpers/key_list"; import { Organization } from "../networking"; +import { TeamMemberSpendBudgetCell } from "./TeamMemberSpendBudgetCell"; interface ResourceTone { icon: typeof Users; @@ -233,11 +234,10 @@ export const getTeamTableColumns = ({ size: 200, enableSorting: false, cell: ({ row }) => ( - ), }, diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx index 2f0b9c9b00e..11dd4f2c2e1 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx @@ -1,4 +1,5 @@ import { Member } from "../networking"; +import type { components } from "@/lib/http/schema"; import type { ObjectPermission } from "../object_permission_types"; import type { ModelBudgetUsage, ModelMaxBudget } from "./ModelMaxBudgetEditor"; @@ -21,6 +22,7 @@ export interface Team { keys_count?: number; members_count?: number; members_with_roles: Member[]; + caller_membership?: components["schemas"]["TeamListCallerMembership"] | null; team_member_permissions?: string[] | null; spend: number; access_group_ids?: string[]; diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/spend_budget_cell.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/spend_budget_cell.tsx index 914d3a98c81..6090b42e93d 100644 --- a/ui/litellm-dashboard/src/components/shared/table_cells/spend_budget_cell.tsx +++ b/ui/litellm-dashboard/src/components/shared/table_cells/spend_budget_cell.tsx @@ -13,7 +13,7 @@ interface SpendBudgetCellProps { budgetDecimals?: number; } -const meterTone = (pct: number): "default" | "warning" | "over" => { +export const getSpendBudgetMeterTone = (pct: number): "default" | "warning" | "over" => { if (pct > 100) return "over"; if (pct >= 80) return "warning"; return "default"; @@ -76,7 +76,7 @@ export function SpendBudgetCell({ {hasBudget && ( - + )} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index b5a86dd2187..e81fa946833 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -46319,6 +46319,21 @@ export interface components { /** Team Id */ team_id: string; }; + /** + * TeamListCallerMembership + * @description The calling user's own spend and effective member budget in a team. + */ + TeamListCallerMembership: { + /** Budget Reset At */ + budget_reset_at?: string | null; + /** Max Budget */ + max_budget?: number | null; + /** + * Spend + * @default 0 + */ + spend: number; + }; /** * TeamListItem * @description A team item in the paginated list response, enriched with computed fields. @@ -46355,6 +46370,7 @@ export interface components { budget_limits?: components["schemas"]["BudgetLimitEntry"][] | null; /** Budget Reset At */ budget_reset_at?: string | null; + caller_membership?: components["schemas"]["TeamListCallerMembership"] | null; /** Created At */ created_at?: string | null; /** Default Team Member Models */ diff --git a/ui/litellm-dashboard/src/utils/budgetUtils.test.ts b/ui/litellm-dashboard/src/utils/budgetUtils.test.ts new file mode 100644 index 00000000000..a50fecabe83 --- /dev/null +++ b/ui/litellm-dashboard/src/utils/budgetUtils.test.ts @@ -0,0 +1,23 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import { formatBudgetReset, formatBudgetResetUtc } from "./budgetUtils"; + +const originalTimezone: string | undefined = process.env.TZ; + +afterEach(() => { + if (originalTimezone === undefined) { + delete process.env.TZ; + return; + } + process.env.TZ = originalTimezone; +}); + +describe("formatBudgetResetUtc", () => { + it("formats the reset date in UTC regardless of the local timezone", () => { + process.env.TZ = "America/Los_Angeles"; + const resetAt: string = "2026-10-20T00:00:00Z"; + + expect(formatBudgetResetUtc(resetAt)).toBe("Oct 20, 2026"); + expect(formatBudgetReset(resetAt)).toBe("Oct 19, 2026"); + }); +}); diff --git a/ui/litellm-dashboard/src/utils/budgetUtils.ts b/ui/litellm-dashboard/src/utils/budgetUtils.ts index 3d3278db88f..dfc57f2fa66 100644 --- a/ui/litellm-dashboard/src/utils/budgetUtils.ts +++ b/ui/litellm-dashboard/src/utils/budgetUtils.ts @@ -1,4 +1,7 @@ import dayjs from "dayjs"; +import utc from "dayjs/plugin/utc"; + +dayjs.extend(utc); export function formatBudgetReset(iso: string | null | undefined): string | null { if (!iso) return null; @@ -6,3 +9,10 @@ export function formatBudgetReset(iso: string | null | undefined): string | null if (!resetDate.isValid()) return null; return resetDate.format("MMM D, YYYY"); } + +export function formatBudgetResetUtc(iso: string | null | undefined): string | null { + if (!iso) return null; + const resetDate = dayjs.utc(iso); + if (!resetDate.isValid()) return null; + return resetDate.format("MMM D, YYYY"); +} From 68b9370c7c280426d9d95bb98693b7eec8168e57 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 24 Oct 2026 17:11:46 -0700 Subject: [PATCH 035/155] fix(chatgpt): route uncatalogued models through responses (#48244) --- litellm/responses/bridge_check.py | 2 +- .../test_responses_api_bridge_flag.py | 90 +++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 60318d6b184..8b00cdf4813 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -175,7 +175,7 @@ def responses_api_bridge_check( pinned_mode: Final = ( None if explicit_responses_prefix else deployment_routing_mode(deployment_model_info, custom_llm_provider) ) - if explicit_responses_prefix or pinned_mode == "responses": + if custom_llm_provider == LlmProviders.CHATGPT or explicit_responses_prefix or pinned_mode == "responses": model = model.replace("responses/", "") model_info["mode"] = "responses" elif pinned_mode == "chat" and backend_serves_chat_completions(model_info): diff --git a/tests/unit/responses/test_responses_api_bridge_flag.py b/tests/unit/responses/test_responses_api_bridge_flag.py index 63204114501..278bd4c1e52 100644 --- a/tests/unit/responses/test_responses_api_bridge_flag.py +++ b/tests/unit/responses/test_responses_api_bridge_flag.py @@ -10,6 +10,7 @@ import copy import json from collections.abc import Mapping from importlib import import_module +from pathlib import Path from typing import Final, cast from unittest.mock import MagicMock, patch @@ -852,3 +853,92 @@ def test_bridge_reasoning_effort_reads_only_a_string_or_mapping_effort_nested_in extra_body: JsonValue, expected: JsonValue ) -> None: assert bridge_reasoning_effort(None, extra_body) == expected + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("prefix", ["", "responses/"]) +@pytest.mark.parametrize("catalog_mode", [None, "chat", "responses"]) +@pytest.mark.asyncio +async def test_chatgpt_completion_routes_to_responses_without_a_provider_catalog_entry( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + respx_mock: respx.MockRouter, + asynchronous: bool, + stream: bool, + prefix: str, + catalog_mode: str | None, +) -> None: + model: Final = "test-chatgpt-routing" + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setenv("CHATGPT_API_BASE", "https://chatgpt.example") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + (tmp_path / "auth.json").write_text( + json.dumps({"access_token": "fake-access-token", "expires_at": 253402300799, "account_id": "fake-account"}) + ) + monkeypatch.delitem(litellm.model_cost, f"chatgpt/{model}", raising=False) + monkeypatch.delitem(litellm.model_cost, model, raising=False) + if catalog_mode is not None: + monkeypatch.setitem(litellm.model_cost, model, {"mode": catalog_mode, "litellm_provider": "openai"}) + litellm.get_model_info.cache_clear() + payload: Final = { + "id": "resp_test", + "object": "response", + "created_at": 1, + "status": "completed", + "model": model, + "output": [ + { + "type": "message", + "id": "msg_test", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "hello", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + events: Final = ( + {"type": "response.created", "response": {**payload, "status": "in_progress", "output": []}}, + { + "type": "response.output_text.delta", + "delta": "hello", + "item_id": "msg_test", + "output_index": 0, + "content_index": 0, + }, + {"type": "response.completed", "response": payload}, + ) + upstream: Final = respx_mock.post("https://chatgpt.example/responses").respond( + 200, + headers={"content-type": "text/event-stream"}, + text="".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events), + ) + kwargs: Final = { + "model": f"chatgpt/{prefix}{model}", + "messages": [{"role": "user", "content": "Say hello"}], + "api_base": "https://chatgpt.example", + "stream": stream, + "num_retries": 0, + } + response: Final = await litellm.acompletion(**kwargs) if asynchronous else litellm.completion(**kwargs) + result: Final = ( + litellm.stream_chunk_builder([chunk async for chunk in response] if asynchronous else list(response)) + if stream + else response + ) + assert result is not None + assert result.choices[0].message.content == "hello" + assert upstream.call_count == 1 + sent: Final = json.loads(upstream.calls[0].request.content) + assert sent["model"] == model + assert sent["store"] is False + assert sent["input"] == [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello"}]} + ] + + +@pytest.mark.parametrize("provider", ["openai", "azure", "xai", "github_copilot"]) +def test_missing_catalog_entry_does_not_force_other_providers_to_responses(provider: str) -> None: + info, model = responses_api_bridge_check(model="test-chatgpt-routing-control", custom_llm_provider=provider) + assert (info.get("mode"), model) == (None, "test-chatgpt-routing-control") From 943583722a03db41619d614b47d318277b93de11 Mon Sep 17 00:00:00 2001 From: jesus-berri Date: Sat, 24 Oct 2026 17:14:16 -0700 Subject: [PATCH 036/155] fix(ui): show "Never resets" instead of "n/a" for an unset budget reset period (#47065) * fix(ui): show "Never resets" instead of "n/a" for an unset budget reset period Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): label unset reset period as Keep current/No change where blank keeps the stored value Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): show a single Never resets option in create team when no default is set Without a default team budget_duration, the create team Reset Budget dropdown offered both "Not set" and "Never resets", and both created a team that never resets. The explicit Never resets item is now only shown when there is a default to opt out of, otherwise the empty option itself reads "Never resets" Also makes the Reset Budget create tests use chooseSelectOption so they no longer flake on the select popup's pointer-events transition --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../components/AccessGroupBudgetModal.tsx | 1 + ...cessGroupBudgetsPanel.integration.test.tsx | 30 ++++++++++++++ .../users/_components/BulkEditUsers.test.tsx | 13 ++++++ .../users/_components/user_edit_view.test.tsx | 29 +++++++++++++ .../users/_components/user_edit_view.tsx | 9 +++- .../user_info_view.integration.test.tsx | 4 +- .../src/components/Teams.test.tsx | 17 ++++---- ui/litellm-dashboard/src/components/Teams.tsx | 4 +- .../budget_duration_dropdown.test.tsx | 41 +++++++++++++++++++ .../budget_duration_dropdown.tsx | 2 +- .../team/EditMembership.integration.test.tsx | 21 ++++++++++ 11 files changed, 158 insertions(+), 13 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/common_components/budget_duration_dropdown.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AccessGroupBudgetModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AccessGroupBudgetModal.tsx index a66f31a0dc0..54decbfff3d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AccessGroupBudgetModal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AccessGroupBudgetModal.tsx @@ -102,6 +102,7 @@ const AccessGroupBudgetModal: React.FC = ({ id={id} value={value || null} onChange={(next) => onChange(next ?? undefined)} + placeholder="Keep current" /> )} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AccessGroupBudgetsPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AccessGroupBudgetsPanel.integration.test.tsx index 4d0818987ce..60cd880b95e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AccessGroupBudgetsPanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AccessGroupBudgetsPanel.integration.test.tsx @@ -116,6 +116,36 @@ describe("AccessGroupBudgetsPanel", () => { expect(await screen.findByLabelText(/Max Budget/)).toHaveValue(2.5); }); + it("keeps the stored reset period when Keep current is selected", async () => { + GET.mockResolvedValue({ + data: { + access_groups: [ + { + ...BUDGETED_GROUP, + budget: { ...BUDGETED_GROUP.budget, budget_duration: "24h" }, + }, + ], + }, + }); + renderPanel(); + await openActions("premium"); + await userEvent.click(await screen.findByText("Edit budget")); + + await userEvent.click(await screen.findByRole("combobox", { name: "Reset Budget" })); + const keepCurrentOption = await screen.findByRole("option", { name: "Keep current" }); + expect(keepCurrentOption).toBeInTheDocument(); + await userEvent.click(keepCurrentOption); + await userEvent.click(screen.getByRole("button", { name: "Save Budget" })); + + await waitFor(() => + expect(PUT).toHaveBeenCalledWith("/access_group/{access_group}/budget", { + params: { path: { access_group: "premium" } }, + body: { max_budget: 2.5 }, + }), + ); + expect(PUT.mock.calls[0][1].body).not.toHaveProperty("budget_duration"); + }); + it("refuses to save a budget with every field blank", async () => { renderPanel(); await openActions("shared"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.test.tsx index f0a8873a67c..f797d6e4f36 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.test.tsx @@ -14,6 +14,7 @@ vi.mock("./user_edit_view", () => ({ UserEditView: ({ onSubmit, onCancel }: { onSubmit: (values: any) => void; onCancel: () => void }) => (
+
), @@ -204,6 +205,18 @@ describe("BulkEditUserModal", () => { }); }); + it("omits an unset reset period from the bulk update payload", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: "Submit with unset duration" })); + + await waitFor(() => { + expect(mockUserBulkUpdateUserCall).toHaveBeenCalledWith("test-token", { max_budget: 100 }, ["user1", "user2"]); + }); + expect(mockUserBulkUpdateUserCall.mock.calls[0][1]).not.toHaveProperty("budget_duration"); + }); + it("should call userBulkUpdateUserCall with allUsers flag when update all users is enabled", async () => { const user = userEvent.setup(); renderWithProviders(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/user_edit_view.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/user_edit_view.test.tsx index 156b27672d5..f71384d80d7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/user_edit_view.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/user_edit_view.test.tsx @@ -322,6 +322,35 @@ describe("UserEditView", () => { expect(callArgs.metadata).toEqual(MOCK_USER_DATA.user_info.metadata); }); + it("uses No change for an unset reset period in bulk edit", async () => { + const onSubmit = vi.fn(); + const userDataWithUnsetResetPeriod = { + ...MOCK_USER_DATA, + user_info: { + ...MOCK_USER_DATA.user_info, + max_budget: null, + budget_duration: null, + }, + }; + renderWithProviders( + , + ); + + const resetPeriod = await screen.findByRole("combobox", { name: "Reset Budget" }); + expect(resetPeriod).toHaveTextContent("No change"); + await userEvent.click(resetPeriod); + await userEvent.click(await screen.findByRole("option", { name: "No change" })); + + const unlimitedBudget = await screen.findByRole("checkbox", { name: "Unlimited Budget" }); + await waitFor(() => expect(unlimitedBudget).toBeChecked()); + await userEvent.click(unlimitedBudget); + fireEvent.change(screen.getByRole("spinbutton", { name: /max budget/i }), { target: { value: "12.5" } }); + await userEvent.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(onSubmit).toHaveBeenCalled()); + expect(onSubmit.mock.calls[0][0]).toMatchObject({ max_budget: "12.5", budget_duration: null }); + }); + it("should set max_budget to null when unlimited budget is checked on submit", async () => { const onSubmitMock = vi.fn(); const userDataWithNullBudget = { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/user_edit_view.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/user_edit_view.tsx index 647761bb44b..d4317d477f8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/user_edit_view.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/user_edit_view.tsx @@ -315,7 +315,14 @@ export function UserEditView({ - {({ id, value, onChange }) => } + {({ id, value, onChange }) => ( + + )} {!isBulkEdit && ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx index 948607bff37..d434987b873 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx @@ -201,13 +201,13 @@ describe("UserInfoView add-to-team form", () => { await openEditor(user); await user.click(screen.getByRole("combobox", { name: "Reset Budget" })); - await user.click(await screen.findByRole("option", { name: "n/a" })); + await user.click(await screen.findByRole("option", { name: "Never resets" })); await user.click(screen.getByRole("button", { name: /save changes/i })); await waitFor(() => expect(mockUserUpdateUserCall).toHaveBeenCalled()); expect(mockUserUpdateUserCall.mock.calls[0][1]).toMatchObject({ budget_duration: null }); await openEditor(user); - expect(screen.getByRole("combobox", { name: "Reset Budget" })).toHaveTextContent("n/a"); + expect(screen.getByRole("combobox", { name: "Reset Budget" })).toHaveTextContent("Never resets"); }); }); diff --git a/ui/litellm-dashboard/src/components/Teams.test.tsx b/ui/litellm-dashboard/src/components/Teams.test.tsx index 7ab6ec2679e..fcfa1ea5e80 100644 --- a/ui/litellm-dashboard/src/components/Teams.test.tsx +++ b/ui/litellm-dashboard/src/components/Teams.test.tsx @@ -730,9 +730,9 @@ describe("Teams - Reset Budget in team create", () => { it("should send an explicit null budget_duration when Never resets is selected", async () => { await openCreateModal(); + await screen.findByText("Default: monthly (30d)"); - await userEvent.click(resetBudgetSelect()); - await userEvent.click(await screen.findByText("Never resets")); + await chooseSelectOption(userEvent, resetBudgetSelect(), "Never resets"); const payload = await submitCreateModal(); @@ -752,8 +752,7 @@ describe("Teams - Reset Budget in team create", () => { it("should send the picked duration when one is selected", async () => { await openCreateModal(); - await userEvent.click(resetBudgetSelect()); - await userEvent.click(await screen.findByText("weekly")); + await chooseSelectOption(userEvent, resetBudgetSelect(), "weekly"); const payload = await submitCreateModal(); @@ -768,14 +767,18 @@ describe("Teams - Reset Budget in team create", () => { }); }); - it("should fall back to the n/a placeholder when the default settings fetch fails", async () => { + it("should offer a single Never resets option when no server default is configured", async () => { vi.mocked(getDefaultTeamSettings).mockRejectedValue(new Error("Unauthorized")); await openCreateModal(); - await waitFor(() => { - expect(screen.getByText("n/a")).toBeInTheDocument(); + expect(getDefaultTeamSettings).toHaveBeenCalled(); }); + + await userEvent.click(resetBudgetSelect()); + + expect(await screen.findAllByRole("option", { name: "Never resets" })).toHaveLength(1); + expect(screen.queryByText("Not set")).not.toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/Teams.tsx b/ui/litellm-dashboard/src/components/Teams.tsx index 9971010c3ef..3a04340d3c4 100644 --- a/ui/litellm-dashboard/src/components/Teams.tsx +++ b/ui/litellm-dashboard/src/components/Teams.tsx @@ -295,7 +295,7 @@ const Teams: React.FC = ({ accessToken, userID, userRole, isViewOnly const defaultBudgetDuration: string | undefined = defaultTeamSettings?.values?.budget_duration ?? undefined; const budgetDurationPlaceholder = defaultBudgetDuration ? `Default: ${getBudgetDurationLabel(defaultBudgetDuration)} (${defaultBudgetDuration})` - : "n/a"; + : undefined; // Add this useEffect to fetch guardrails useEffect(() => { @@ -825,7 +825,7 @@ const Teams: React.FC = ({ accessToken, userID, userRole, isViewOnly {({ id, value, onChange }) => ( onChange(next ?? undefined)} diff --git a/ui/litellm-dashboard/src/components/common_components/budget_duration_dropdown.test.tsx b/ui/litellm-dashboard/src/components/common_components/budget_duration_dropdown.test.tsx new file mode 100644 index 00000000000..184e4297a67 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/budget_duration_dropdown.test.tsx @@ -0,0 +1,41 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; + +import BudgetDurationDropdown from "./budget_duration_dropdown"; + +describe("BudgetDurationDropdown", () => { + it("labels an unset period as Never resets in the trigger and option", async () => { + const user = userEvent.setup(); + render(); + + expect(screen.getByRole("combobox")).toHaveTextContent("Never resets"); + + await user.click(screen.getByRole("combobox")); + + expect(await screen.findByRole("option", { name: "Never resets" })).toBeInTheDocument(); + }); + + it("emits null when Never resets is selected", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + render(); + + await user.click(screen.getByRole("combobox")); + await user.click(await screen.findByRole("option", { name: "Never resets" })); + + expect(onChange).toHaveBeenCalledOnce(); + expect(onChange.mock.calls[0][0]).toBeNull(); + }); + + it("uses a supplied placeholder for the unset trigger and option", async () => { + const user = userEvent.setup(); + render(); + + expect(screen.getByRole("combobox")).toHaveTextContent("Not set"); + + await user.click(screen.getByRole("combobox")); + + expect(await screen.findByRole("option", { name: "Not set" })).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/common_components/budget_duration_dropdown.tsx b/ui/litellm-dashboard/src/components/common_components/budget_duration_dropdown.tsx index 40ee857634c..bae6986ab03 100644 --- a/ui/litellm-dashboard/src/components/common_components/budget_duration_dropdown.tsx +++ b/ui/litellm-dashboard/src/components/common_components/budget_duration_dropdown.tsx @@ -27,7 +27,7 @@ const BudgetDurationDropdown: React.FC = ({ onChange, className = "", style = {}, - placeholder = "n/a", + placeholder = "Never resets", showNeverResets = false, }) => { return ( diff --git a/ui/litellm-dashboard/src/components/team/EditMembership.integration.test.tsx b/ui/litellm-dashboard/src/components/team/EditMembership.integration.test.tsx index 2c98d17f961..81106bc094a 100644 --- a/ui/litellm-dashboard/src/components/team/EditMembership.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/team/EditMembership.integration.test.tsx @@ -106,6 +106,27 @@ describe("EditMembership submit payload", () => { }); }); + it("shows Never resets and submits a null duration for a member without a reset period", async () => { + renderEdit(teamMemberConfig, { user_id: "u1", user_email: "a@b.com", role: "user" }); + + expect(screen.getByRole("combobox", { name: "Budget Reset Period" })).toHaveTextContent("Never resets"); + + save(); + + await waitFor(() => expect(onSubmit).toHaveBeenCalledOnce()); + const expectedPayload = { + user_email: "a@b.com", + user_id: "u1", + role: "user", + max_budget_in_team: null, + budget_duration: null, + tpm_limit: null, + rpm_limit: null, + allowed_models: [], + }; + expect(submitted()).toStrictEqual(expectedPayload); + }); + it("seeds the per-model budget editor from the member's stored budget and submits the edited cap", async () => { const modelBudgetConfig = { ...teamMemberConfig, From 2fa6f22aff24b08030bb756c7b772f5a1678989d Mon Sep 17 00:00:00 2001 From: pfolex <93488901+pfolex@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:16:33 -0400 Subject: [PATCH 037/155] feat(models): add Cerebras Llama 3.1 hyphenated aliases (cerebras/llama-3.1-8b, cerebras/llama-3.1-70b) (#48157) (#48158) --- ...odel_prices_and_context_window_backup.json | 22 ++++ model_prices_and_context_window.json | 22 ++++ .../test_cerebras_llama_model_metadata.py | 115 ++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 tests/unit/test_cerebras_llama_model_metadata.py diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 02eaa906522..98df2fc4204 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -15663,6 +15663,28 @@ "/v1/images/generations" ] }, + "cerebras/llama-3.1-70b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "cerebras", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "supports_function_calling": true, + "supports_tool_choice": true + }, + "cerebras/llama-3.1-8b": { + "input_cost_per_token": 1e-07, + "litellm_provider": "cerebras", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-07, + "supports_function_calling": true, + "supports_tool_choice": true + }, "cerebras/llama-3.3-70b": { "input_cost_per_token": 8.5e-07, "litellm_provider": "cerebras", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 02eaa906522..98df2fc4204 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -15663,6 +15663,28 @@ "/v1/images/generations" ] }, + "cerebras/llama-3.1-70b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "cerebras", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "supports_function_calling": true, + "supports_tool_choice": true + }, + "cerebras/llama-3.1-8b": { + "input_cost_per_token": 1e-07, + "litellm_provider": "cerebras", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-07, + "supports_function_calling": true, + "supports_tool_choice": true + }, "cerebras/llama-3.3-70b": { "input_cost_per_token": 8.5e-07, "litellm_provider": "cerebras", diff --git a/tests/unit/test_cerebras_llama_model_metadata.py b/tests/unit/test_cerebras_llama_model_metadata.py new file mode 100644 index 00000000000..c5ec096e553 --- /dev/null +++ b/tests/unit/test_cerebras_llama_model_metadata.py @@ -0,0 +1,115 @@ +# What does this test? +""" +Verifies that Cerebras Llama 3.1 models (standard hyphenated aliases): +- cerebras/llama-3.1-8b +- cerebras/llama-3.1-70b + +have correct rate card metadata, context window limits, and cost calculation parity +with the un-hyphenated variants in model_prices_and_context_window.json. +Fixes: https://github.com/BerriAI/litellm/issues/48157 +""" + +import json +from pathlib import Path + +import pytest + +import litellm + + +@pytest.fixture(scope="module") +def model_cost_maps(): + root = Path(__file__).resolve().parents[2] + main_path = root / "model_prices_and_context_window.json" + backup_path = root / "litellm" / "model_prices_and_context_window_backup.json" + + with open(main_path, "r", encoding="utf-8") as f: + main_map = json.load(f) + with open(backup_path, "r", encoding="utf-8") as f: + backup_map = json.load(f) + + return main_map, backup_map + + +CEREBRAS_MODELS_EXPECTED = { + "cerebras/llama-3.1-8b": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1e-07, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "mode": "chat", + "litellm_provider": "cerebras", + }, + "cerebras/llama-3.1-70b": { + "input_cost_per_token": 6e-07, + "output_cost_per_token": 6e-07, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "mode": "chat", + "litellm_provider": "cerebras", + }, +} + +PAIR_PARITY = [ + ("cerebras/llama-3.1-8b", "cerebras/llama3.1-8b"), + ("cerebras/llama-3.1-70b", "cerebras/llama3.1-70b"), +] + + +@pytest.mark.parametrize("model_id,expected_data", CEREBRAS_MODELS_EXPECTED.items()) +def test_cerebras_llama_model_metadata(model_cost_maps, model_id, expected_data): + """Test that hyphenated Cerebras Llama 3.1 models exist and have expected metadata in both catalogs.""" + main_map, backup_map = model_cost_maps + + assert model_id in main_map, f"Missing {model_id} in main catalog" + assert model_id in backup_map, f"Missing {model_id} in backup catalog" + + main_entry = main_map[model_id] + backup_entry = backup_map[model_id] + assert main_entry == backup_entry, f"Parity mismatch between main and backup for {model_id}" + + for key, expected_val in expected_data.items(): + assert main_entry.get(key) == expected_val, ( + f"{model_id}[{key}] got {main_entry.get(key)}, expected {expected_val}" + ) + + +@pytest.mark.parametrize("hyphenated,unhyphenated", PAIR_PARITY) +def test_cerebras_llama_parity_with_unhyphenated(model_cost_maps, hyphenated, unhyphenated): + """Test that hyphenated and un-hyphenated variants have identical pricing and context limits.""" + main_map, _ = model_cost_maps + + assert hyphenated in main_map + assert unhyphenated in main_map + + entry_hyphen = main_map[hyphenated] + entry_unhyphen = main_map[unhyphenated] + + assert entry_hyphen["input_cost_per_token"] == entry_unhyphen["input_cost_per_token"] + assert entry_hyphen["output_cost_per_token"] == entry_unhyphen["output_cost_per_token"] + assert entry_hyphen["max_input_tokens"] == entry_unhyphen["max_input_tokens"] + assert entry_hyphen["max_output_tokens"] == entry_unhyphen["max_output_tokens"] + + +def test_cerebras_llama_cost_per_token(): + """Test that litellm.cost_per_token accurately computes costs for hyphenated Cerebras models.""" + prompt_tokens = 1_000_000 + completion_tokens = 1_000_000 + + # llama-3.1-8b: $0.10 input + $0.10 output per 1M tokens + cost_prompt, cost_comp = litellm.cost_per_token( + model="cerebras/llama-3.1-8b", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + ) + assert pytest.approx(cost_prompt, rel=1e-5) == 0.10 + assert pytest.approx(cost_comp, rel=1e-5) == 0.10 + + # llama-3.1-70b: $0.60 input + $0.60 output per 1M tokens + cost_prompt_70b, cost_comp_70b = litellm.cost_per_token( + model="cerebras/llama-3.1-70b", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + ) + assert pytest.approx(cost_prompt_70b, rel=1e-5) == 0.60 + assert pytest.approx(cost_comp_70b, rel=1e-5) == 0.60 From 5171b3fbd85051969ee4dcf512dec634fd8f472b Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sat, 24 Oct 2026 17:17:09 -0700 Subject: [PATCH 038/155] feat(cost): parse 12-hour cache write rates for batch and ultrafast tiers (#48259) * feat(cost): parse 12-hour cache write rates for batch and ultrafast tiers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost): bill 12-hour batch cache writes once when only the 12hr batch key exists Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 23 +- .../litellm_core_utils/llm_cost_calc/utils.py | 11 +- litellm/types/utils.py | 8 + litellm/utils.py | 12 + .../cost_calculation/cost_tracking_case.py | 4 + .../llm_cost_calc/test_llm_cost_calc_utils.py | 80 ++ tests/unit/test_cost_calculator.py | 106 +- tests/unit/test_utils.py | 910 +++++++++--------- ui/litellm-dashboard/src/lib/http/schema.d.ts | 24 + 9 files changed, 730 insertions(+), 448 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a0da0f6ddca..8b02f93fca4 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -37,6 +37,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( _SERVICE_TIER_TO_COST_KEY_SUFFIX, BilledTokenRates, CostCalculatorUtils, + calculate_cache_writing_cost, calculate_cost_component, generic_cost_per_character, generic_cost_per_token, @@ -2969,7 +2970,12 @@ def batch_cost_calculator( total_completion_cost = 0.0 if batch_rates.input is not None: total_prompt_cost = _batch_prompt_cost( - usage, model_info, batch_rates.input, batch_rates.cache_read, batch_rates.cache_creation + usage, + model_info, + batch_rates.input, + batch_rates.cache_read, + batch_rates.cache_creation, + batch_rates.cache_creation_rates_by_ttl, ) elif input_cost_per_token: details: Final = parse_prompt_tokens_details(usage) @@ -3014,11 +3020,13 @@ def _batch_prompt_cost( input_rate: float, cache_read_rate: float | None, cache_creation_rate: float | None, + cache_creation_rates_by_ttl: Mapping[str, float], ) -> float: details: Final = parse_prompt_tokens_details(usage) cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0 cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0 - written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0 + bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl) + written_tokens: Final = details["cache_creation_tokens"] if bills_cache_writes else 0 audio_tokens, image_tokens, video_tokens = ( details["audio_tokens"], details["image_tokens"], @@ -3035,7 +3043,16 @@ def _batch_prompt_cost( + (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0) + cached_audio_tokens * _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0) - + written_tokens * (cache_creation_rate or 0.0) + + ( + calculate_cache_writing_cost( + written_tokens, + details["cache_creation_token_details"], + input_rate if cache_creation_rate is None else cache_creation_rate, + cache_creation_rates_by_ttl, + ) + if bills_cache_writes + else 0.0 + ) ) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 5d4b1a82759..3b49f239afe 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -104,7 +104,8 @@ _BATCH_RATE_PREFIXES: Final = ( "cache_read_input_token_cost", "cache_creation_input_token_cost", ) -_BATCH_TIER_KEY: Final = re.compile(rf"^({'|'.join(_BATCH_RATE_PREFIXES)})_above_(\d+k?)_tokens{_BATCH_KEY_SUFFIX}$") +_BATCH_TIER_PREFIXES: Final = (*_BATCH_RATE_PREFIXES, *_TTL_PRICED_CACHE_WRITE_RATE_KEYS.values()) +_BATCH_TIER_KEY: Final = re.compile(rf"^({'|'.join(_BATCH_TIER_PREFIXES)})_above_(\d+k?)_tokens{_BATCH_KEY_SUFFIX}$") _NON_STANDARD_THRESHOLD_SUFFIXES: Final = (*_SERVICE_TIER_SUFFIXES, _BATCH_KEY_SUFFIX) @@ -314,6 +315,7 @@ class BatchCostRates: output: float | None cache_read: float | None cache_creation: float | None + cache_creation_rates_by_ttl: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({})) def _batch_rate(model_info: ModelInfo, key: str) -> float | None: @@ -369,6 +371,13 @@ def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provide output=_batch_rate_for_prefix(model_info, "output_cost_per_token", usage, inclusive), cache_read=_batch_rate_for_prefix(model_info, "cache_read_input_token_cost", usage, inclusive), cache_creation=_batch_rate_for_prefix(model_info, "cache_creation_input_token_cost", usage, inclusive), + cache_creation_rates_by_ttl=MappingProxyType( + { + ttl: rate + for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items() + if (rate := _batch_rate_for_prefix(model_info, key, usage, inclusive)) is not None + } + ), ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ee8683f8ae8..cc7ef851111 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -328,6 +328,10 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_creation_input_token_cost_above_12hr_above_272k_tokens: ReadOnly[float | None] cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: ReadOnly[float | None] cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_batches: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_ultrafast: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast: ReadOnly[float | None] cache_creation_input_token_cost_flex: float | None # OpenAI flex service tier pricing cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing @@ -3973,6 +3977,10 @@ class MirroredPricingParams(LiteLLMBaseModel): cache_creation_input_token_cost_above_12hr_above_272k_tokens: float | None = None cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: float | None = None cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: float | None = None + cache_creation_input_token_cost_above_12hr_batches: float | None = None + cache_creation_input_token_cost_above_12hr_ultrafast: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast: float | None = None tiered_pricing: list[dict[str, Any]] | None = None diff --git a/litellm/utils.py b/litellm/utils.py index acf4594c6b9..e2820f9c682 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6842,6 +6842,18 @@ def get_model_info_helper( cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority=_model_info.get( "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority", None ), + cache_creation_input_token_cost_above_12hr_batches=_model_info.get( + "cache_creation_input_token_cost_above_12hr_batches", None + ), + cache_creation_input_token_cost_above_12hr_ultrafast=_model_info.get( + "cache_creation_input_token_cost_above_12hr_ultrafast", None + ), + cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches=_model_info.get( + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches", None + ), + cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast=_model_info.get( + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast", None + ), off_peak_pricing=_model_info.get("off_peak_pricing", None), input_cost_per_character=_model_info.get("input_cost_per_character", None), input_cost_per_token_above_128k_tokens=_model_info.get("input_cost_per_token_above_128k_tokens", None), diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index e978a9a316a..95363e92366 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -91,6 +91,10 @@ class CostMapEntry(BaseModel): cache_creation_input_token_cost_above_12hr_above_272k_tokens: float | None = None cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: float | None = None cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: float | None = None + cache_creation_input_token_cost_above_12hr_batches: float | None = None + cache_creation_input_token_cost_above_12hr_ultrafast: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast: float | None = None cache_creation_input_token_cost_above_1hr_above_200k_tokens: float | None = None cache_read_input_token_cost_above_200k_tokens: float | None = None cache_creation_input_token_cost_above_200k_tokens: float | None = None diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 1729db94ed8..21b527250c4 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2179,6 +2179,48 @@ def test_generic_cost_per_token_falls_back_to_default_for_missing_12h_rate() -> assert result[0] == pytest.approx(100 * model_info["cache_creation_input_token_cost"]) +@pytest.mark.parametrize(("prompt_tokens", "twelve_hour_rate"), [(10_000, 7e-6), (300_000, 14e-6)]) +def test_generic_cost_per_token_prices_ultrafast_12h_cache_writes_at_the_ultrafast_rate( + prompt_tokens: int, twelve_hour_rate: float +) -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_token_ultrafast": 5e-6, + "cache_creation_input_token_cost": 2e-6, + "cache_creation_input_token_cost_ultrafast": 3e-6, + "cache_creation_input_token_cost_above_12hr": 4e-6, + "cache_creation_input_token_cost_above_12hr_ultrafast": 7e-6, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 8e-6, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": 14e-6, + } + usage: Final = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=0, + total_tokens=prompt_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper( + cache_creation_tokens=5_000, + cache_creation_token_details=CacheCreationTokenDetails( + ephemeral_30m_input_tokens=3_000, ephemeral_12h_input_tokens=2_000 + ), + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model="test-ultrafast-12h-cache-write", + usage=usage, + custom_llm_provider="openai", + service_tier="ultrafast", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx( + (prompt_tokens - 5_000) * model_info["input_cost_per_token_ultrafast"] + + 3_000 * model_info["cache_creation_input_token_cost_ultrafast"] + + 2_000 * twelve_hour_rate + ) + + def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): """ Regression test: when cache_creation_tokens is 0 but cache_creation_token_details @@ -4079,6 +4121,44 @@ def test_get_batch_cost_rates_has_no_cache_write_rate_without_a_cache_write_batc assert rates.cache_creation is None +@pytest.mark.parametrize(("prompt_tokens", "expected"), [(1_000, 2.5e-7), (272_000, 2.5e-7), (300_000, 5e-7)]) +def test_get_batch_cost_rates_reads_the_12h_batch_cache_write_rate_for_the_crossed_tier( + prompt_tokens: int, expected: float +) -> None: + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates: Final = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-7, + cache_creation_input_token_cost_batches=1.25e-7, + cache_creation_input_token_cost_above_12hr_batches=2.5e-7, + cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches=5e-7, + ), + Usage(prompt_tokens=prompt_tokens, completion_tokens=1, total_tokens=prompt_tokens + 1), + "openai", + ) + + assert rates.cache_creation == 1.25e-7 + assert rates.cache_creation_rates_by_ttl == {"12hr": expected} + + +def test_get_batch_cost_rates_has_no_12h_cache_write_rate_without_a_12h_batch_key() -> None: + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates: Final = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-7, + cache_creation_input_token_cost_batches=1.25e-7, + cache_creation_input_token_cost_above_12hr=2.5e-7, + cache_creation_input_token_cost_above_12hr_ultrafast=5e-7, + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert rates.cache_creation_rates_by_ttl == {} + + @pytest.mark.parametrize("model_base", ["gpt-6-sol", "gpt-6-luna"]) def test_azure_gpt_6_foundry_price_sheet(_local_model_cost_map, model_base): """Azure Foundry hosts gpt-6-sol and gpt-6-luna at OpenAI's Global rates, with the diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 550fe5f0d1a..27b25c1f98e 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -35,7 +35,7 @@ from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import HttpxBinaryResponseContent, OpenAIRealtimeStreamList, ResponseAPIUsage, ResponsesAPIResponse from litellm.types.llms.xai import RealtimeMeteredUsage from litellm.types.rerank import RerankResponse -from litellm.types.utils import CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage +from litellm.types.utils import CacheCreationTokenDetails, CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage from litellm.types.videos.main import VideoObject from litellm.utils import supports_prompt_caching import os @@ -5947,6 +5947,110 @@ def test_batch_cost_calculator_bills_cache_write_tokens_at_the_batch_input_rate_ assert prompt_cost == pytest.approx(300_048 * 3e-6) +_TWELVE_HOUR_BATCH_MODEL_INFO: Final = cast( + ModelInfo, + { + "input_cost_per_token_batches": 1e-6, + "input_cost_per_token_above_272k_tokens_batches": 3e-6, + "cache_creation_input_token_cost_batches": 1.25e-6, + "cache_creation_input_token_cost_above_272k_tokens_batches": 3.75e-6, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-6, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 7.5e-6, + "output_cost_per_token_batches": 4e-6, + }, +) + + +def _twelve_hour_cache_write_usage(prompt_tokens: int) -> Usage: + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=4, + total_tokens=prompt_tokens + 4, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=0, + cache_creation_tokens=500, + cache_creation_token_details=CacheCreationTokenDetails( + ephemeral_30m_input_tokens=300, ephemeral_12h_input_tokens=200 + ), + ), + ) + + +@pytest.mark.parametrize( + ("prompt_tokens", "input_rate", "write_rate", "twelve_hour_rate"), + [(1_000, 1e-6, 1.25e-6, 2.5e-6), (272_000, 1e-6, 1.25e-6, 2.5e-6), (300_000, 3e-6, 3.75e-6, 7.5e-6)], +) +def test_batch_cost_calculator_bills_12h_cache_writes_at_the_12h_batch_rate_for_the_crossed_tier( + prompt_tokens: int, input_rate: float, write_rate: float, twelve_hour_rate: float +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, completion_cost_value = batch_cost_calculator( + usage=_twelve_hour_cache_write_usage(prompt_tokens), + model="lit-12h-batch-model", + custom_llm_provider="openai", + model_info=_TWELVE_HOUR_BATCH_MODEL_INFO, + ) + + assert prompt_cost == pytest.approx((prompt_tokens - 500) * input_rate + 300 * write_rate + 200 * twelve_hour_rate) + assert completion_cost_value == pytest.approx(4 * 4e-6) + + +@pytest.mark.parametrize( + ("prompt_tokens", "input_rate", "twelve_hour_write_rate"), [(1_000, 1e-6, 2.5e-6), (300_000, 3e-6, 7.5e-6)] +) +def test_batch_cost_calculator_bills_12h_cache_writes_once_when_only_the_12h_batch_key_exists( + prompt_tokens: int, input_rate: float, twelve_hour_write_rate: float +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + model_info: Final = cast( + ModelInfo, + { + key: rate + for key, rate in _TWELVE_HOUR_BATCH_MODEL_INFO.items() + if not key.startswith("cache_creation_input_token_cost_") + or key.startswith("cache_creation_input_token_cost_above_12hr") + }, + ) + + prompt_cost, _ = batch_cost_calculator( + usage=_twelve_hour_cache_write_usage(prompt_tokens), + model="lit-12h-batch-model", + custom_llm_provider="openai", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx((prompt_tokens - 200) * input_rate + 200 * twelve_hour_write_rate) + + +@pytest.mark.parametrize(("prompt_tokens", "write_rate"), [(1_000, 1.25e-6), (300_000, 3.75e-6)]) +def test_batch_cost_calculator_bills_12h_cache_writes_at_the_batch_cache_write_rate_without_a_12h_batch_key( + prompt_tokens: int, write_rate: float +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + model_info: Final = cast( + ModelInfo, + { + key: rate + for key, rate in _TWELVE_HOUR_BATCH_MODEL_INFO.items() + if not key.startswith("cache_creation_input_token_cost_above_12hr") + }, + ) + + prompt_cost, _ = batch_cost_calculator( + usage=_twelve_hour_cache_write_usage(prompt_tokens), + model="lit-12h-batch-model", + custom_llm_provider="openai", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx( + (prompt_tokens - 500) * (1e-6 if prompt_tokens <= 272_000 else 3e-6) + 500 * write_rate + ) + + def test_batch_cost_calculator_prices_modalities_and_cached_tokens_together_in_the_crossed_tier() -> None: from litellm.cost_calculator import batch_cost_calculator diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index a0b4b784a4d..55b7bc78a11 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -21,7 +21,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx import pytest import respx -from jsonschema import validate +from jsonschema import Draft7Validator, validate import litellm from litellm.exceptions import MidStreamFallbackError @@ -923,6 +923,456 @@ def validate_model_cost_values(model_data, exceptions=None): return len(violations) == 0, violations +_COST_MAP_ROW_SCHEMA: Final = { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "supports_computer_use": {"type": "boolean"}, + "cache_creation_input_audio_token_cost": {"type": "number"}, + "cache_creation_input_token_cost": {"type": "number"}, + "cache_creation_input_token_cost_above_1hr": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_flex": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_priority": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_batches": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_ultrafast": {"type": "number"}, + "cache_creation_input_token_cost_above_32k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_100k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_100k_tokens_batches": {"type": "number"}, + "cache_creation_input_token_cost_above_128k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_200k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_256k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_272k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_272k_tokens_flex": {"type": "number"}, + "cache_creation_input_token_cost_above_272k_tokens_ultrafast": {"type": "number"}, + "cache_creation_input_token_cost_above_272k_tokens_priority": {"type": "number"}, + "cache_creation_input_token_cost_above_200k_tokens_batches": {"type": "number"}, + "cache_creation_input_token_cost_above_272k_tokens_batches": {"type": "number"}, + "cache_creation_input_token_cost_batches": {"type": "number"}, + "cache_creation_input_token_cost_decisions": {"type": "number"}, + "cache_creation_input_token_cost_above_272k_tokens_decisions": {"type": "number"}, + "cache_read_input_token_cost_decisions": {"type": "number"}, + "cache_read_input_token_cost_above_272k_tokens_decisions": {"type": "number"}, + "input_cost_per_token_decisions": {"type": "number"}, + "input_cost_per_token_above_272k_tokens_decisions": {"type": "number"}, + "output_cost_per_token_decisions": {"type": "number"}, + "output_cost_per_token_above_272k_tokens_decisions": {"type": "number"}, + "cache_creation_input_token_cost_flex": {"type": "number"}, + "cache_creation_input_token_cost_priority": {"type": "number"}, + "cache_creation_input_token_cost_ultrafast": {"type": "number"}, + "cache_read_input_token_cost": {"type": "number"}, + "explicit_cache_read_input_token_cost": {"type": "number"}, + "cache_read_input_token_cost_above_32k_tokens": {"type": "number"}, + "cache_read_input_token_cost_above_100k_tokens": {"type": "number"}, + "cache_read_input_token_cost_above_100k_tokens_batches": {"type": "number"}, + "cache_read_input_token_cost_above_128k_tokens": {"type": "number"}, + "cache_read_input_token_cost_above_200k_tokens": {"type": "number"}, + "cache_read_input_token_cost_above_200k_tokens_batches": {"type": "number"}, + "cache_read_input_token_cost_above_256k_tokens": {"type": "number"}, + "cache_read_input_token_cost_above_272k_tokens": {"type": "number"}, + "cache_read_input_token_cost_above_272k_tokens_flex": {"type": "number"}, + "cache_read_input_token_cost_above_272k_tokens_ultrafast": {"type": "number"}, + "cache_read_input_token_cost_above_512k_tokens": {"type": "number"}, + "input_cost_per_token_above_272k_tokens_ultrafast": {"type": "number"}, + "cache_read_input_token_cost_batches": {"type": "number"}, + "cache_read_input_token_cost_above_272k_tokens_batches": {"type": "number"}, + "cache_creation_input_token_cost_above_1hr_above_100k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"}, + "cache_read_input_audio_token_cost": {"type": "number"}, + "cache_read_input_audio_token_cost_batches": {"type": "number"}, + "cache_read_input_audio_token_cost_flex": {"type": "number"}, + "cache_read_input_audio_token_cost_priority": {"type": "number"}, + "cache_read_input_image_token_cost": {"type": "number"}, + "audio_transcription_config": {"type": "string"}, + "deprecation_date": {"type": "string"}, + "input_cost_per_audio_per_second": {"type": "number"}, + "input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"}, + "google_maps_grounding_cost_per_query": {"type": "number"}, + "input_cost_per_audio_token": {"type": "number"}, + "input_cost_per_audio_token_flex": {"type": "number"}, + "input_cost_per_image_token": {"type": "number"}, + "input_cost_per_character": {"type": "number"}, + "input_cost_per_character_above_128k_tokens": {"type": "number"}, + "input_cost_per_image": {"type": "number"}, + "input_cost_per_image_above_128k_tokens": {"type": "number"}, + "input_cost_per_video_token": {"type": "number"}, + "input_cost_per_token_above_32k_tokens": {"type": "number"}, + "input_cost_per_token_above_100k_tokens": {"type": "number"}, + "input_cost_per_token_above_100k_tokens_batches": {"type": "number"}, + "input_cost_per_token_above_200k_tokens": {"type": "number"}, + "input_cost_per_token_above_200k_tokens_batches": {"type": "number"}, + "input_cost_per_token_above_256k_tokens": {"type": "number"}, + "input_cost_per_token_above_272k_tokens": {"type": "number"}, + "input_cost_per_token_above_512k_tokens": {"type": "number"}, + "cache_read_input_token_cost_flex": {"type": "number"}, + "cache_read_input_token_cost_priority": {"type": "number"}, + "cache_read_input_token_cost_balanced": {"type": "number"}, + "cache_read_input_token_cost_ultrafast": {"type": "number"}, + "cache_read_input_token_cost_above_200k_tokens_priority": {"type": "number"}, + "cache_read_input_token_cost_above_272k_tokens_priority": {"type": "number"}, + "input_cost_per_token_flex": {"type": "number"}, + "input_cost_per_token_priority": {"type": "number"}, + "input_cost_per_token_balanced": {"type": "number"}, + "input_cost_per_token_ultrafast": {"type": "number"}, + "input_cost_per_token_above_200k_tokens_priority": {"type": "number"}, + "input_cost_per_token_above_272k_tokens_priority": {"type": "number"}, + "input_cost_per_token_above_272k_tokens_batches": {"type": "number"}, + "input_cost_per_token_above_272k_tokens_flex": {"type": "number"}, + "input_cost_per_audio_token_priority": {"type": "number"}, + "output_cost_per_token_flex": {"type": "number"}, + "output_cost_per_token_priority": {"type": "number"}, + "output_cost_per_token_balanced": {"type": "number"}, + "output_cost_per_token_ultrafast": {"type": "number"}, + "output_cost_per_token_above_200k_tokens_priority": {"type": "number"}, + "output_cost_per_token_above_272k_tokens_priority": {"type": "number"}, + "output_cost_per_token_above_272k_tokens_ultrafast": {"type": "number"}, + "output_cost_per_token_above_272k_tokens_batches": {"type": "number"}, + "output_cost_per_token_above_272k_tokens_flex": {"type": "number"}, + "regional_endpoint_uplift_multiplier": {"type": "number"}, + "regional_processing_uplift_multiplier_eu": {"type": "number"}, + "regional_processing_uplift_multiplier_us": {"type": "number"}, + "input_cost_per_pixel": {"type": "number"}, + "input_cost_per_query": {"type": "number"}, + "input_cost_per_request": {"type": "number"}, + "cost_per_second": {"type": "number"}, + "input_cost_per_second": {"type": "number"}, + "input_cost_per_text_message": {"type": "number"}, + "input_cost_per_token": {"type": "number"}, + "input_cost_per_token_above_128k_tokens": {"type": "number"}, + "input_cost_per_audio_token_batches": {"type": "number"}, + "input_cost_per_image_token_batches": {"type": "number"}, + "input_cost_per_token_batches": {"type": "number"}, + "input_cost_per_video_token_batches": {"type": "number"}, + "input_cost_per_token_cache_hit": {"type": "number"}, + "input_cost_per_video_per_second": {"type": "number"}, + "input_cost_per_video_per_second_above_8s_interval": {"type": "number"}, + "input_cost_per_video_per_second_above_15s_interval": {"type": "number"}, + "input_cost_per_video_per_second_above_128k_tokens": {"type": "number"}, + "input_dbu_cost_per_token": {"type": "number"}, + "annotation_cost_per_page": {"type": "number"}, + "annotation_cost_per_page_batches": {"type": "number"}, + "ocr_cost_per_page": {"type": "number"}, + "ocr_cost_per_page_batches": {"type": "number"}, + "ocr_cost_per_credit": {"type": "number"}, + "code_interpreter_cost_per_session": {"type": "number"}, + "inference_geo": {"type": "string"}, + "litellm_provider": {"type": "string"}, + "max_input_tokens": {"type": "number"}, + "max_output_tokens": {"type": "number"}, + "max_tokens": {"type": "number"}, + "metadata": {"type": "object"}, + "provider_specific_entry": {"type": "object"}, + "mode": { + "type": "string", + "enum": [ + "audio_speech", + "audio_transcription", + "chat", + "completion", + "container", + "image_edit", + "embedding", + "evaluation", + "extraction", + "guardrail", + "image_generation", + "video_generation", + "moderation", + "rerank", + "realtime", + "responses", + "ocr", + "search", + "vector_store", + ], + }, + "output_cost_per_audio_token": {"type": "number"}, + "output_cost_per_character": {"type": "number"}, + "output_cost_per_character_above_128k_tokens": {"type": "number"}, + "output_cost_per_image": {"type": "number"}, + "output_cost_per_image_512": {"type": "number"}, + "output_cost_per_image_1024": {"type": "number"}, + "output_cost_per_image_1536": {"type": "number"}, + "output_cost_per_image_0.5K": {"type": "number"}, + "output_cost_per_image_1K": {"type": "number"}, + "output_cost_per_image_2K": {"type": "number"}, + "output_cost_per_image_4K": {"type": "number"}, + "output_cost_per_image_token": {"type": "number"}, + "output_cost_per_image_token_batches": {"type": "number"}, + "output_cost_per_video_token": {"type": "number"}, + "output_cost_per_pixel": {"type": "number"}, + "output_cost_per_second": {"type": "number"}, + "output_cost_per_second_480p": {"type": "number"}, + "output_cost_per_second_720p": {"type": "number"}, + "output_cost_per_second_768p": {"type": "number"}, + "output_cost_per_second_2k": {"type": "number"}, + "output_cost_per_second_1080p": {"type": "number"}, + "output_cost_per_second_4k": {"type": "number"}, + "output_cost_per_token": {"type": "number"}, + "output_cost_per_token_above_32k_tokens": {"type": "number"}, + "output_cost_per_token_above_100k_tokens": {"type": "number"}, + "output_cost_per_token_above_100k_tokens_batches": {"type": "number"}, + "output_cost_per_token_above_128k_tokens": {"type": "number"}, + "output_cost_per_token_above_200k_tokens": {"type": "number"}, + "output_cost_per_token_above_200k_tokens_batches": {"type": "number"}, + "output_cost_per_token_above_256k_tokens": {"type": "number"}, + "output_cost_per_token_above_272k_tokens": {"type": "number"}, + "output_cost_per_token_above_512k_tokens": {"type": "number"}, + "output_cost_per_token_batches": {"type": "number"}, + "output_cost_per_reasoning_token": {"type": "number"}, + "output_cost_per_video_per_second": {"type": "number"}, + "output_db_cost_per_token": {"type": "number"}, + "output_dbu_cost_per_token": {"type": "number"}, + "output_vector_size": {"type": "number"}, + "rpd": {"type": "number"}, + "rpm": {"type": "number"}, + "source": {"type": "string"}, + "comment": {"type": "string"}, + "supports_assistant_prefill": {"type": "boolean"}, + "supports_anthropic_compaction": {"type": "boolean"}, + "supports_audio_input": {"type": "boolean"}, + "supports_audio_output": {"type": "boolean"}, + "gemini_native_audio": {"type": "boolean"}, + "gemini_audio_only_live": {"type": "boolean"}, + "supports_embedding_image_input": {"type": "boolean"}, + "supports_forced_tool_use": {"type": "boolean"}, + "supports_function_calling": {"type": "boolean"}, + "supports_image_input": {"type": "boolean"}, + "supports_nova_canvas_image_edit": {"type": "boolean"}, + "supports_parallel_function_calling": {"type": "boolean"}, + "supports_parallel_tool_use_config": {"type": "boolean"}, + "supports_pdf_input": {"type": "boolean"}, + "prompt_cache_min_tokens": {"type": "number"}, + "supports_prompt_cache_breakpoint": {"type": "boolean"}, + "supports_thinking_cache_preservation": {"type": "boolean"}, + "supports_prompt_caching": {"type": "boolean"}, + "supports_response_schema": {"type": "boolean"}, + "supports_system_messages": {"type": "boolean"}, + "supports_tool_choice": {"type": "boolean"}, + "supports_tool_search": {"type": "boolean"}, + "supports_video_input": {"type": "boolean"}, + "supports_box_questions": {"type": "boolean"}, + "supports_vision": {"type": "boolean"}, + "supports_web_search": {"type": "boolean"}, + "token_count_framing": {"type": "integer"}, + "token_count_model": { + "type": "object", + "properties": {"model": {"type": "string"}, "region": {"type": "string"}}, + "required": ["model"], + }, + "token_count_tokenizer": {"type": "string", "enum": ["kimi_k3"]}, + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": {"type": "boolean"}, + "supports_bedrock_runtime_chat_completions_response_format": {"type": "boolean"}, + "supports_bedrock_runtime_chat_completions_inline_reasoning": {"type": "boolean"}, + "supports_bedrock_converse_tool_result_images": {"type": "boolean"}, + "supports_url_context": {"type": "boolean"}, + "supports_multimodal": {"type": "boolean"}, + "uses_embed_content": {"type": "boolean"}, + "supports_reasoning": {"type": "boolean"}, + "supports_minimal_reasoning_effort": {"type": "boolean"}, + "supports_low_reasoning_effort": {"type": "boolean"}, + "supports_none_reasoning_effort": {"type": "boolean"}, + "supports_xhigh_reasoning_effort": {"type": "boolean"}, + "supports_max_reasoning_effort": {"type": "boolean"}, + "reasoning_effort_levels": { + "type": "array", + "items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]}, + }, + "default_reasoning_effort": { + "type": "string", + "enum": ["none", "minimal", "low", "medium", "high", "xhigh"], + }, + "supports_adaptive_thinking": {"type": "boolean"}, + "supports_anthropic_thinking_payload": {"type": "boolean"}, + "supports_legacy_thinking": {"type": "boolean"}, + "thinking_always_on": {"type": "boolean"}, + "supports_mid_conversation_system": {"type": "boolean"}, + "supports_sampling_params": {"type": "boolean"}, + "supports_output_config": {"type": "boolean"}, + "supports_speed": {"type": "boolean"}, + "supports_fast_mode": {"type": "boolean"}, + "supported_audio_formats": { + "type": "array", + "items": { + "type": "string", + "enum": ["mp3", "wav"], + }, + }, + "vertex_ai_audio_api": { + "type": "string", + "enum": ["lyria_predict", "lyria_interactions"], + }, + "bedrock_output_config_effort_ceiling": { + "type": "string", + "enum": ["low", "medium", "high", "max", "xhigh"], + }, + "bedrock_converse_supports_strict_tools": {"type": "boolean"}, + "supports_regex_lookaround": {"type": "boolean"}, + "tpm": {"type": "number"}, + "supported_endpoints": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "/v1/responses", + "/v1/embeddings", + "/v1/chat/completions", + "/v1/completions", + "/v1/messages", + "/v1/images/generations", + "/v1/realtime", + "/v1/realtime/transcription_sessions", + "/v1/images/variations", + "/v1/images/edits", + "/v1/batch", + "/v1beta/interactions", + "/v1beta/models", + "/v1/audio/transcriptions", + "/v1/audio/speech", + "/v1/ocr", + "/v1/videos", + "/vertex_ai/live", + "/v1/listen", + "/v1/systemone", + "/v1/decisions", + "/v1beta/interactions", + ], + }, + }, + "supported_regions": { + "type": "array", + "items": { + "type": "string", + }, + }, + "guardrail_cost_per_unit": { + "type": "object", + "additionalProperties": {"type": "number"}, + }, + "search_context_cost_per_query": { + "type": "object", + "properties": { + "search_context_size_low": {"type": "number"}, + "search_context_size_medium": {"type": "number"}, + "search_context_size_high": {"type": "number"}, + }, + "additionalProperties": False, + }, + "web_search_billing_unit": { + "type": "string", + "enum": ["per_prompt", "per_query"], + }, + "search_free_requests": { + "type": "object", + "properties": { + "count": {"type": "integer", "minimum": 1}, + "period": {"type": "string", "enum": ["day", "month"]}, + "shared_group": {"type": "string"}, + }, + "required": ["count", "period", "shared_group"], + "additionalProperties": False, + }, + "citation_cost_per_token": {"type": "number"}, + "supported_modalities": { + "type": "array", + "items": { + "type": "string", + "enum": ["text", "audio", "image", "video"], + }, + }, + "supported_output_modalities": { + "type": "array", + "items": { + "type": "string", + "enum": ["text", "image", "audio", "code", "video"], + }, + }, + "supports_native_streaming": {"type": "boolean"}, + "supports_image_size": {"type": "boolean"}, + "supports_native_structured_output": {"type": "boolean"}, + "supports_bedrock_native_structured_output": {"type": "boolean"}, + "use_openai_responses_path": {"type": "boolean"}, + "off_peak_pricing": { + "type": "object", + "properties": { + "hours_utc": { + "oneOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}], + }, + "windows": { + "type": "array", + "items": { + "type": "object", + "properties": { + "hours_utc": { + "oneOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}], + }, + "weekdays": { + "type": "array", + "items": {"oneOf": [{"type": "integer"}, {"type": "string"}]}, + }, + }, + "required": ["hours_utc"], + "additionalProperties": False, + }, + }, + "weekday_timezone": {"type": "string"}, + "input_cost_per_token": {"type": "number"}, + "output_cost_per_token": {"type": "number"}, + "output_cost_per_reasoning_token": {"type": "number"}, + "cache_read_input_token_cost": {"type": "number"}, + "cache_creation_input_token_cost": {"type": "number"}, + }, + "additionalProperties": False, + }, + "tiered_pricing": { + "type": "array", + "items": { + "type": "object", + "properties": { + "range": { + "type": "array", + "items": {"type": "number"}, + "minItems": 2, + "maxItems": 2, + }, + "input_cost_per_token": {"type": "number"}, + "output_cost_per_token": {"type": "number"}, + "cache_read_input_token_cost": {"type": "number"}, + "cache_creation_input_token_cost": {"type": "number"}, + "output_cost_per_reasoning_token": {"type": "number"}, + "max_results_range": { + "type": "array", + "items": {"type": "number"}, + "minItems": 2, + "maxItems": 2, + }, + "input_cost_per_query": {"type": "number"}, + }, + "additionalProperties": False, + }, + }, + }, + "additionalProperties": False, + }, +} + + def test_aaamodel_prices_and_context_window_json_is_valid(): """ Validates the `model_prices_and_context_window.json` file. @@ -930,447 +1380,6 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): If this test fails after you update the json, you need to update the schema or correct the change you made. """ - INTENDED_SCHEMA = { - "type": "object", - "additionalProperties": { - "type": "object", - "properties": { - "supports_computer_use": {"type": "boolean"}, - "cache_creation_input_audio_token_cost": {"type": "number"}, - "cache_creation_input_token_cost": {"type": "number"}, - "cache_creation_input_token_cost_above_1hr": {"type": "number"}, - "cache_creation_input_token_cost_above_12hr": {"type": "number"}, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_flex": {"type": "number"}, - "cache_creation_input_token_cost_above_12hr_priority": {"type": "number"}, - "cache_creation_input_token_cost_above_32k_tokens": {"type": "number"}, - "cache_creation_input_token_cost_above_100k_tokens": {"type": "number"}, - "cache_creation_input_token_cost_above_100k_tokens_batches": {"type": "number"}, - "cache_creation_input_token_cost_above_128k_tokens": {"type": "number"}, - "cache_creation_input_token_cost_above_200k_tokens": {"type": "number"}, - "cache_creation_input_token_cost_above_256k_tokens": {"type": "number"}, - "cache_creation_input_token_cost_above_272k_tokens": {"type": "number"}, - "cache_creation_input_token_cost_above_272k_tokens_flex": {"type": "number"}, - "cache_creation_input_token_cost_above_272k_tokens_ultrafast": {"type": "number"}, - "cache_creation_input_token_cost_above_272k_tokens_priority": {"type": "number"}, - "cache_creation_input_token_cost_above_200k_tokens_batches": {"type": "number"}, - "cache_creation_input_token_cost_above_272k_tokens_batches": {"type": "number"}, - "cache_creation_input_token_cost_batches": {"type": "number"}, - "cache_creation_input_token_cost_decisions": {"type": "number"}, - "cache_creation_input_token_cost_above_272k_tokens_decisions": {"type": "number"}, - "cache_read_input_token_cost_decisions": {"type": "number"}, - "cache_read_input_token_cost_above_272k_tokens_decisions": {"type": "number"}, - "input_cost_per_token_decisions": {"type": "number"}, - "input_cost_per_token_above_272k_tokens_decisions": {"type": "number"}, - "output_cost_per_token_decisions": {"type": "number"}, - "output_cost_per_token_above_272k_tokens_decisions": {"type": "number"}, - "cache_creation_input_token_cost_flex": {"type": "number"}, - "cache_creation_input_token_cost_priority": {"type": "number"}, - "cache_creation_input_token_cost_ultrafast": {"type": "number"}, - "cache_read_input_token_cost": {"type": "number"}, - "explicit_cache_read_input_token_cost": {"type": "number"}, - "cache_read_input_token_cost_above_32k_tokens": {"type": "number"}, - "cache_read_input_token_cost_above_100k_tokens": {"type": "number"}, - "cache_read_input_token_cost_above_100k_tokens_batches": {"type": "number"}, - "cache_read_input_token_cost_above_128k_tokens": {"type": "number"}, - "cache_read_input_token_cost_above_200k_tokens": {"type": "number"}, - "cache_read_input_token_cost_above_200k_tokens_batches": {"type": "number"}, - "cache_read_input_token_cost_above_256k_tokens": {"type": "number"}, - "cache_read_input_token_cost_above_272k_tokens": {"type": "number"}, - "cache_read_input_token_cost_above_272k_tokens_flex": {"type": "number"}, - "cache_read_input_token_cost_above_272k_tokens_ultrafast": {"type": "number"}, - "cache_read_input_token_cost_above_512k_tokens": {"type": "number"}, - "input_cost_per_token_above_272k_tokens_ultrafast": {"type": "number"}, - "cache_read_input_token_cost_batches": {"type": "number"}, - "cache_read_input_token_cost_above_272k_tokens_batches": {"type": "number"}, - "cache_creation_input_token_cost_above_1hr_above_100k_tokens": {"type": "number"}, - "cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"}, - "cache_read_input_audio_token_cost": {"type": "number"}, - "cache_read_input_audio_token_cost_batches": {"type": "number"}, - "cache_read_input_audio_token_cost_flex": {"type": "number"}, - "cache_read_input_audio_token_cost_priority": {"type": "number"}, - "cache_read_input_image_token_cost": {"type": "number"}, - "audio_transcription_config": {"type": "string"}, - "deprecation_date": {"type": "string"}, - "input_cost_per_audio_per_second": {"type": "number"}, - "input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"}, - "google_maps_grounding_cost_per_query": {"type": "number"}, - "input_cost_per_audio_token": {"type": "number"}, - "input_cost_per_audio_token_flex": {"type": "number"}, - "input_cost_per_image_token": {"type": "number"}, - "input_cost_per_character": {"type": "number"}, - "input_cost_per_character_above_128k_tokens": {"type": "number"}, - "input_cost_per_image": {"type": "number"}, - "input_cost_per_image_above_128k_tokens": {"type": "number"}, - "input_cost_per_video_token": {"type": "number"}, - "input_cost_per_token_above_32k_tokens": {"type": "number"}, - "input_cost_per_token_above_100k_tokens": {"type": "number"}, - "input_cost_per_token_above_100k_tokens_batches": {"type": "number"}, - "input_cost_per_token_above_200k_tokens": {"type": "number"}, - "input_cost_per_token_above_200k_tokens_batches": {"type": "number"}, - "input_cost_per_token_above_256k_tokens": {"type": "number"}, - "input_cost_per_token_above_272k_tokens": {"type": "number"}, - "input_cost_per_token_above_512k_tokens": {"type": "number"}, - "cache_read_input_token_cost_flex": {"type": "number"}, - "cache_read_input_token_cost_priority": {"type": "number"}, - "cache_read_input_token_cost_balanced": {"type": "number"}, - "cache_read_input_token_cost_ultrafast": {"type": "number"}, - "cache_read_input_token_cost_above_200k_tokens_priority": {"type": "number"}, - "cache_read_input_token_cost_above_272k_tokens_priority": {"type": "number"}, - "input_cost_per_token_flex": {"type": "number"}, - "input_cost_per_token_priority": {"type": "number"}, - "input_cost_per_token_balanced": {"type": "number"}, - "input_cost_per_token_ultrafast": {"type": "number"}, - "input_cost_per_token_above_200k_tokens_priority": {"type": "number"}, - "input_cost_per_token_above_272k_tokens_priority": {"type": "number"}, - "input_cost_per_token_above_272k_tokens_batches": {"type": "number"}, - "input_cost_per_token_above_272k_tokens_flex": {"type": "number"}, - "input_cost_per_audio_token_priority": {"type": "number"}, - "output_cost_per_token_flex": {"type": "number"}, - "output_cost_per_token_priority": {"type": "number"}, - "output_cost_per_token_balanced": {"type": "number"}, - "output_cost_per_token_ultrafast": {"type": "number"}, - "output_cost_per_token_above_200k_tokens_priority": {"type": "number"}, - "output_cost_per_token_above_272k_tokens_priority": {"type": "number"}, - "output_cost_per_token_above_272k_tokens_ultrafast": {"type": "number"}, - "output_cost_per_token_above_272k_tokens_batches": {"type": "number"}, - "output_cost_per_token_above_272k_tokens_flex": {"type": "number"}, - "regional_endpoint_uplift_multiplier": {"type": "number"}, - "regional_processing_uplift_multiplier_eu": {"type": "number"}, - "regional_processing_uplift_multiplier_us": {"type": "number"}, - "input_cost_per_pixel": {"type": "number"}, - "input_cost_per_query": {"type": "number"}, - "input_cost_per_request": {"type": "number"}, - "cost_per_second": {"type": "number"}, - "input_cost_per_second": {"type": "number"}, - "input_cost_per_text_message": {"type": "number"}, - "input_cost_per_token": {"type": "number"}, - "input_cost_per_token_above_128k_tokens": {"type": "number"}, - "input_cost_per_audio_token_batches": {"type": "number"}, - "input_cost_per_image_token_batches": {"type": "number"}, - "input_cost_per_token_batches": {"type": "number"}, - "input_cost_per_video_token_batches": {"type": "number"}, - "input_cost_per_token_cache_hit": {"type": "number"}, - "input_cost_per_video_per_second": {"type": "number"}, - "input_cost_per_video_per_second_above_8s_interval": {"type": "number"}, - "input_cost_per_video_per_second_above_15s_interval": {"type": "number"}, - "input_cost_per_video_per_second_above_128k_tokens": {"type": "number"}, - "input_dbu_cost_per_token": {"type": "number"}, - "annotation_cost_per_page": {"type": "number"}, - "annotation_cost_per_page_batches": {"type": "number"}, - "ocr_cost_per_page": {"type": "number"}, - "ocr_cost_per_page_batches": {"type": "number"}, - "ocr_cost_per_credit": {"type": "number"}, - "code_interpreter_cost_per_session": {"type": "number"}, - "inference_geo": {"type": "string"}, - "litellm_provider": {"type": "string"}, - "max_input_tokens": {"type": "number"}, - "max_output_tokens": {"type": "number"}, - "max_tokens": {"type": "number"}, - "metadata": {"type": "object"}, - "provider_specific_entry": {"type": "object"}, - "mode": { - "type": "string", - "enum": [ - "audio_speech", - "audio_transcription", - "chat", - "completion", - "container", - "image_edit", - "embedding", - "evaluation", - "extraction", - "guardrail", - "image_generation", - "video_generation", - "moderation", - "rerank", - "realtime", - "responses", - "ocr", - "search", - "vector_store", - ], - }, - "output_cost_per_audio_token": {"type": "number"}, - "output_cost_per_character": {"type": "number"}, - "output_cost_per_character_above_128k_tokens": {"type": "number"}, - "output_cost_per_image": {"type": "number"}, - "output_cost_per_image_512": {"type": "number"}, - "output_cost_per_image_1024": {"type": "number"}, - "output_cost_per_image_1536": {"type": "number"}, - "output_cost_per_image_0.5K": {"type": "number"}, - "output_cost_per_image_1K": {"type": "number"}, - "output_cost_per_image_2K": {"type": "number"}, - "output_cost_per_image_4K": {"type": "number"}, - "output_cost_per_image_token": {"type": "number"}, - "output_cost_per_image_token_batches": {"type": "number"}, - "output_cost_per_video_token": {"type": "number"}, - "output_cost_per_pixel": {"type": "number"}, - "output_cost_per_second": {"type": "number"}, - "output_cost_per_second_480p": {"type": "number"}, - "output_cost_per_second_720p": {"type": "number"}, - "output_cost_per_second_768p": {"type": "number"}, - "output_cost_per_second_2k": {"type": "number"}, - "output_cost_per_second_1080p": {"type": "number"}, - "output_cost_per_second_4k": {"type": "number"}, - "output_cost_per_token": {"type": "number"}, - "output_cost_per_token_above_32k_tokens": {"type": "number"}, - "output_cost_per_token_above_100k_tokens": {"type": "number"}, - "output_cost_per_token_above_100k_tokens_batches": {"type": "number"}, - "output_cost_per_token_above_128k_tokens": {"type": "number"}, - "output_cost_per_token_above_200k_tokens": {"type": "number"}, - "output_cost_per_token_above_200k_tokens_batches": {"type": "number"}, - "output_cost_per_token_above_256k_tokens": {"type": "number"}, - "output_cost_per_token_above_272k_tokens": {"type": "number"}, - "output_cost_per_token_above_512k_tokens": {"type": "number"}, - "output_cost_per_token_batches": {"type": "number"}, - "output_cost_per_reasoning_token": {"type": "number"}, - "output_cost_per_video_per_second": {"type": "number"}, - "output_db_cost_per_token": {"type": "number"}, - "output_dbu_cost_per_token": {"type": "number"}, - "output_vector_size": {"type": "number"}, - "rpd": {"type": "number"}, - "rpm": {"type": "number"}, - "source": {"type": "string"}, - "comment": {"type": "string"}, - "supports_assistant_prefill": {"type": "boolean"}, - "supports_anthropic_compaction": {"type": "boolean"}, - "supports_audio_input": {"type": "boolean"}, - "supports_audio_output": {"type": "boolean"}, - "gemini_native_audio": {"type": "boolean"}, - "gemini_audio_only_live": {"type": "boolean"}, - "supports_embedding_image_input": {"type": "boolean"}, - "supports_forced_tool_use": {"type": "boolean"}, - "supports_function_calling": {"type": "boolean"}, - "supports_image_input": {"type": "boolean"}, - "supports_nova_canvas_image_edit": {"type": "boolean"}, - "supports_parallel_function_calling": {"type": "boolean"}, - "supports_parallel_tool_use_config": {"type": "boolean"}, - "supports_pdf_input": {"type": "boolean"}, - "prompt_cache_min_tokens": {"type": "number"}, - "supports_prompt_cache_breakpoint": {"type": "boolean"}, - "supports_thinking_cache_preservation": {"type": "boolean"}, - "supports_prompt_caching": {"type": "boolean"}, - "supports_response_schema": {"type": "boolean"}, - "supports_system_messages": {"type": "boolean"}, - "supports_tool_choice": {"type": "boolean"}, - "supports_tool_search": {"type": "boolean"}, - "supports_video_input": {"type": "boolean"}, - "supports_box_questions": {"type": "boolean"}, - "supports_vision": {"type": "boolean"}, - "supports_web_search": {"type": "boolean"}, - "token_count_framing": {"type": "integer"}, - "token_count_model": { - "type": "object", - "properties": {"model": {"type": "string"}, "region": {"type": "string"}}, - "required": ["model"], - }, - "token_count_tokenizer": {"type": "string", "enum": ["kimi_k3"]}, - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": {"type": "boolean"}, - "supports_bedrock_runtime_chat_completions_response_format": {"type": "boolean"}, - "supports_bedrock_runtime_chat_completions_inline_reasoning": {"type": "boolean"}, - "supports_bedrock_converse_tool_result_images": {"type": "boolean"}, - "supports_url_context": {"type": "boolean"}, - "supports_multimodal": {"type": "boolean"}, - "uses_embed_content": {"type": "boolean"}, - "supports_reasoning": {"type": "boolean"}, - "supports_minimal_reasoning_effort": {"type": "boolean"}, - "supports_low_reasoning_effort": {"type": "boolean"}, - "supports_none_reasoning_effort": {"type": "boolean"}, - "supports_xhigh_reasoning_effort": {"type": "boolean"}, - "supports_max_reasoning_effort": {"type": "boolean"}, - "reasoning_effort_levels": { - "type": "array", - "items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]}, - }, - "default_reasoning_effort": { - "type": "string", - "enum": ["none", "minimal", "low", "medium", "high", "xhigh"], - }, - "supports_adaptive_thinking": {"type": "boolean"}, - "supports_anthropic_thinking_payload": {"type": "boolean"}, - "supports_legacy_thinking": {"type": "boolean"}, - "thinking_always_on": {"type": "boolean"}, - "supports_mid_conversation_system": {"type": "boolean"}, - "supports_sampling_params": {"type": "boolean"}, - "supports_output_config": {"type": "boolean"}, - "supports_speed": {"type": "boolean"}, - "supports_fast_mode": {"type": "boolean"}, - "supported_audio_formats": { - "type": "array", - "items": { - "type": "string", - "enum": ["mp3", "wav"], - }, - }, - "vertex_ai_audio_api": { - "type": "string", - "enum": ["lyria_predict", "lyria_interactions"], - }, - "bedrock_output_config_effort_ceiling": { - "type": "string", - "enum": ["low", "medium", "high", "max", "xhigh"], - }, - "bedrock_converse_supports_strict_tools": {"type": "boolean"}, - "supports_regex_lookaround": {"type": "boolean"}, - "tpm": {"type": "number"}, - "supported_endpoints": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "/v1/responses", - "/v1/embeddings", - "/v1/chat/completions", - "/v1/completions", - "/v1/messages", - "/v1/images/generations", - "/v1/realtime", - "/v1/realtime/transcription_sessions", - "/v1/images/variations", - "/v1/images/edits", - "/v1/batch", - "/v1beta/interactions", - "/v1beta/models", - "/v1/audio/transcriptions", - "/v1/audio/speech", - "/v1/ocr", - "/v1/videos", - "/vertex_ai/live", - "/v1/listen", - "/v1/systemone", - "/v1/decisions", - "/v1beta/interactions", - ], - }, - }, - "supported_regions": { - "type": "array", - "items": { - "type": "string", - }, - }, - "guardrail_cost_per_unit": { - "type": "object", - "additionalProperties": {"type": "number"}, - }, - "search_context_cost_per_query": { - "type": "object", - "properties": { - "search_context_size_low": {"type": "number"}, - "search_context_size_medium": {"type": "number"}, - "search_context_size_high": {"type": "number"}, - }, - "additionalProperties": False, - }, - "web_search_billing_unit": { - "type": "string", - "enum": ["per_prompt", "per_query"], - }, - "search_free_requests": { - "type": "object", - "properties": { - "count": {"type": "integer", "minimum": 1}, - "period": {"type": "string", "enum": ["day", "month"]}, - "shared_group": {"type": "string"}, - }, - "required": ["count", "period", "shared_group"], - "additionalProperties": False, - }, - "citation_cost_per_token": {"type": "number"}, - "supported_modalities": { - "type": "array", - "items": { - "type": "string", - "enum": ["text", "audio", "image", "video"], - }, - }, - "supported_output_modalities": { - "type": "array", - "items": { - "type": "string", - "enum": ["text", "image", "audio", "code", "video"], - }, - }, - "supports_native_streaming": {"type": "boolean"}, - "supports_image_size": {"type": "boolean"}, - "supports_native_structured_output": {"type": "boolean"}, - "supports_bedrock_native_structured_output": {"type": "boolean"}, - "use_openai_responses_path": {"type": "boolean"}, - "off_peak_pricing": { - "type": "object", - "properties": { - "hours_utc": { - "oneOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}], - }, - "windows": { - "type": "array", - "items": { - "type": "object", - "properties": { - "hours_utc": { - "oneOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}], - }, - "weekdays": { - "type": "array", - "items": {"oneOf": [{"type": "integer"}, {"type": "string"}]}, - }, - }, - "required": ["hours_utc"], - "additionalProperties": False, - }, - }, - "weekday_timezone": {"type": "string"}, - "input_cost_per_token": {"type": "number"}, - "output_cost_per_token": {"type": "number"}, - "output_cost_per_reasoning_token": {"type": "number"}, - "cache_read_input_token_cost": {"type": "number"}, - "cache_creation_input_token_cost": {"type": "number"}, - }, - "additionalProperties": False, - }, - "tiered_pricing": { - "type": "array", - "items": { - "type": "object", - "properties": { - "range": { - "type": "array", - "items": {"type": "number"}, - "minItems": 2, - "maxItems": 2, - }, - "input_cost_per_token": {"type": "number"}, - "output_cost_per_token": {"type": "number"}, - "cache_read_input_token_cost": {"type": "number"}, - "cache_creation_input_token_cost": {"type": "number"}, - "output_cost_per_reasoning_token": {"type": "number"}, - "max_results_range": { - "type": "array", - "items": {"type": "number"}, - "minItems": 2, - "maxItems": 2, - }, - "input_cost_per_query": {"type": "number"}, - }, - "additionalProperties": False, - }, - }, - }, - "additionalProperties": False, - }, - } - prod_json = os.path.join(os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json") with open(prod_json, "r") as model_prices_file: actual_json = json.load(model_prices_file) @@ -1379,7 +1388,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): actual_json.pop("fallback_generalizations", None) # reserved meta key, not a model entry # Validate schema - validate(actual_json, INTENDED_SCHEMA) + validate(actual_json, _COST_MAP_ROW_SCHEMA) # Validate cost values # Define exceptions for models that are allowed to have costs > 1 @@ -1401,6 +1410,21 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): raise AssertionError(error_message) +def test_cost_map_schema_accepts_12hr_cache_write_rates_for_batch_and_ultrafast_tiers() -> None: + row: Final = { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_creation_input_token_cost_above_12hr_batches": 3e-6, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 4e-6, + "cache_creation_input_token_cost_above_12hr_ultrafast": 5e-6, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": 6e-6, + } + + assert Draft7Validator(_COST_MAP_ROW_SCHEMA).is_valid({"synthetic-12hr-cache-write-model": row}) + + def test_max_tokens_consistency(): """ Test that max_tokens == max_output_tokens for all models. diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index e81fa946833..262e8b6deae 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35223,14 +35223,22 @@ export interface components { cache_creation_input_token_cost_above_12hr?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */ cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Batches */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */ cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */ cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Ultrafast */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Batches */ + cache_creation_input_token_cost_above_12hr_batches?: number | null; /** Cache Creation Input Token Cost Above 12Hr Flex */ cache_creation_input_token_cost_above_12hr_flex?: number | null; /** Cache Creation Input Token Cost Above 12Hr Priority */ cache_creation_input_token_cost_above_12hr_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Ultrafast */ + cache_creation_input_token_cost_above_12hr_ultrafast?: number | null; /** Cache Creation Input Token Cost Above 1Hr */ cache_creation_input_token_cost_above_1hr?: number | null; /** Cache Creation Input Token Cost Above 1Hr Above 100K Tokens */ @@ -50253,14 +50261,22 @@ export interface components { cache_creation_input_token_cost_above_12hr?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */ cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Batches */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */ cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */ cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Ultrafast */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Batches */ + cache_creation_input_token_cost_above_12hr_batches?: number | null; /** Cache Creation Input Token Cost Above 12Hr Flex */ cache_creation_input_token_cost_above_12hr_flex?: number | null; /** Cache Creation Input Token Cost Above 12Hr Priority */ cache_creation_input_token_cost_above_12hr_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Ultrafast */ + cache_creation_input_token_cost_above_12hr_ultrafast?: number | null; /** Cache Creation Input Token Cost Above 1Hr */ cache_creation_input_token_cost_above_1hr?: number | null; /** Cache Read Input Token Cost */ @@ -50558,14 +50574,22 @@ export interface components { cache_creation_input_token_cost_above_12hr?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */ cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Batches */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */ cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */ cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Ultrafast */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Batches */ + cache_creation_input_token_cost_above_12hr_batches?: number | null; /** Cache Creation Input Token Cost Above 12Hr Flex */ cache_creation_input_token_cost_above_12hr_flex?: number | null; /** Cache Creation Input Token Cost Above 12Hr Priority */ cache_creation_input_token_cost_above_12hr_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Ultrafast */ + cache_creation_input_token_cost_above_12hr_ultrafast?: number | null; /** Cache Creation Input Token Cost Above 1Hr */ cache_creation_input_token_cost_above_1hr?: number | null; /** Cache Creation Input Token Cost Above 1Hr Above 100K Tokens */ From 222389501066caa22bb070fa93ec620e345609ee Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Sat, 24 Oct 2026 17:29:48 -0700 Subject: [PATCH 039/155] fix(cloudflare): clarify chat and decisions endpoints (#47965) (#48265) --- litellm/llms/cloudflare/chat/transformation.py | 2 +- litellm/llms/cloudflare/decisions/transformation.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/llms/cloudflare/chat/transformation.py b/litellm/llms/cloudflare/chat/transformation.py index 0a76588f767..2bbfd2010cc 100644 --- a/litellm/llms/cloudflare/chat/transformation.py +++ b/litellm/llms/cloudflare/chat/transformation.py @@ -58,7 +58,7 @@ class CloudflareChatConfig(OpenAIGPTConfig): trimmed: Final = api_base.rstrip("/") if trimmed.endswith("/ai/run"): verbose_logger.warning( - "Cloudflare api_base ending in '/ai/run' is the legacy Workers AI path and no longer serves OpenAI-compatible requests; rewriting to the '/ai/v1' endpoint" + "Cloudflare chat completions require the OpenAI-compatible '/ai/v1' endpoint; rewriting api_base ending in '/ai/run' to '/ai/v1'. Decision models continue to use the native Workers AI '/ai/run/{model}' endpoint." ) return join_api_base(trimmed[: -len("/ai/run")], "ai/v1") return api_base diff --git a/litellm/llms/cloudflare/decisions/transformation.py b/litellm/llms/cloudflare/decisions/transformation.py index 7c776367886..7575e8bce21 100644 --- a/litellm/llms/cloudflare/decisions/transformation.py +++ b/litellm/llms/cloudflare/decisions/transformation.py @@ -20,6 +20,8 @@ class CloudflareDecisionsConfig(BaseDecisionsConfig): inlines_remote_images = True def get_default_api_base(self) -> str | None: + # Decision models (Clef) require native /ai/run/{model} requests with + # state and questions; /ai/v1/chat/completions only accepts chat requests. return workers_ai_default_api_base() def missing_api_base_message(self, custom_llm_provider: str) -> str: From cb65a21439eab88a6498780591a7f894427f6fe3 Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Sat, 24 Oct 2026 17:37:38 -0700 Subject: [PATCH 040/155] fix(decisions): preserve missing native usage (#48147) (#48261) --- litellm/cost_calculator.py | 2 ++ .../llms/base_llm/decisions/transformation.py | 10 ++++-- .../fireworks_ai/decisions/transformation.py | 2 ++ .../llms/openai/decisions/transformation.py | 13 ++++---- .../openrouter/decisions/transformation.py | 2 +- litellm/types/decisions.py | 2 +- .../providers/test_decisions_wire.py | 19 +++++++++++ tests/unit/decisions/test_main.py | 32 +++++++++++++++++++ 8 files changed, 71 insertions(+), 11 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 8b02f93fca4..8331c68fe24 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1551,6 +1551,8 @@ def completion_cost( """ try: call_type = infer_call_type(call_type, completion_response) or "completion" + if isinstance(completion_response, DecisionsResponse) and completion_response.usage is None: + raise ValueError("Cannot calculate Decisions cost: provider did not report usage") if pricing_provider is not None: custom_llm_provider = pricing_provider # rebind-ok: model_info.pricing_provider overrides the routing provider for cost-map lookups diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index a122aeb3033..14c6a2d2a66 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -376,12 +376,14 @@ def _ir_answer(question: DecisionsIRQuestion, answer: DecisionAnswer | None) -> def systemone_response_to_ir(response: DecisionsResponse, request: DecisionsIRRequest) -> DecisionsIRResponse: - usage: Final = response.usage or DecisionsUsage() + usage: Final = response.usage keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True) return DecisionsIRResponse( model=response.model, answers=tuple(_ir_answer(question, response.answers.get(key)) for key, question in keyed_questions), - usage=DecisionsIRUsage( + usage=None + if usage is None + else DecisionsIRUsage( input_tokens=usage.input_tokens, output_tokens=usage.output_tokens, cached_tokens=usage.cached_tokens, @@ -485,7 +487,9 @@ def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIR **response.extra, "model": response.model, "answers": {key: answer for key, answer in keyed_answers if answer is not None}, - "usage": DecisionsUsage.model_validate( + "usage": None + if response.usage is None + else DecisionsUsage.model_validate( { **response.usage.extra, "input_tokens": response.usage.input_tokens, diff --git a/litellm/llms/fireworks_ai/decisions/transformation.py b/litellm/llms/fireworks_ai/decisions/transformation.py index 87d402902be..bba4835060b 100644 --- a/litellm/llms/fireworks_ai/decisions/transformation.py +++ b/litellm/llms/fireworks_ai/decisions/transformation.py @@ -14,6 +14,8 @@ class FireworksAIDecisionsConfig(BaseDecisionsConfig): def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: response: Final = parse_systemone_response(self.unwrap_response(payload), request) + if response.usage is None: + return response cached: Final = response.usage.extra.get("cached_input_tokens") if isinstance(cached, bool) or not isinstance(cached, int): return response diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 56f348881de..25d403a0dff 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -311,6 +311,7 @@ def _openai_answer(question: DecisionsIRQuestion, answer: DecisionsIRAnswer) -> def ir_to_openai_response( response: DecisionsIRResponse, request: DecisionsIRRequest, requested_model: str ) -> OpenAIDecisionResponse: + usage: Final = response.usage or DecisionsIRUsage() return OpenAIDecisionResponse( model=response.model or requested_model, answers=tuple( @@ -318,14 +319,14 @@ def ir_to_openai_response( for question, answer in zip(request.questions, response.answers, strict=True) ), usage=OpenAIDecisionUsage( - input_tokens=response.usage.input_tokens, + input_tokens=usage.input_tokens, input_tokens_details=OpenAIDecisionInputTokensDetails( - cached_tokens=response.usage.cached_tokens, - cache_write_tokens=response.usage.cache_write_tokens, + cached_tokens=usage.cached_tokens, + cache_write_tokens=usage.cache_write_tokens, ), - output_tokens=response.usage.output_tokens, - output_tokens_details=OpenAIDecisionOutputTokensDetails(reasoning_tokens=response.usage.reasoning_tokens), - total_tokens=response.usage.input_tokens + response.usage.output_tokens, + output_tokens=usage.output_tokens, + output_tokens_details=OpenAIDecisionOutputTokensDetails(reasoning_tokens=usage.reasoning_tokens), + total_tokens=usage.input_tokens + usage.output_tokens, ), ) diff --git a/litellm/llms/openrouter/decisions/transformation.py b/litellm/llms/openrouter/decisions/transformation.py index 0fa5c7ca7cb..30d33f09efd 100644 --- a/litellm/llms/openrouter/decisions/transformation.py +++ b/litellm/llms/openrouter/decisions/transformation.py @@ -13,7 +13,7 @@ class OpenRouterDecisionsConfig(BaseDecisionsConfig): return "https://openrouter.ai/api" def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: - cost: Final = response.usage.extra.get("cost") + cost: Final = response.usage.extra.get("cost") if response.usage is not None else None if not isinstance(cost, (int, float)): return None return float(cost) diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 5ad6e45e6d2..33b2c677d31 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -650,7 +650,7 @@ class DecisionsIRUsage: class DecisionsIRResponse: model: str | None answers: tuple[DecisionsIRAnswer, ...] - usage: DecisionsIRUsage + usage: DecisionsIRUsage | None extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) diff --git a/tests/integration/providers/test_decisions_wire.py b/tests/integration/providers/test_decisions_wire.py index ea29914ba65..89ca674ef31 100644 --- a/tests/integration/providers/test_decisions_wire.py +++ b/tests/integration/providers/test_decisions_wire.py @@ -660,3 +660,22 @@ def test_a_deployment_whose_provider_has_no_decisions_support_is_refused_naming_ for provider in _PROVIDERS: assert provider.name in response.text, response.text assert _upstream_calls(gateway, handle) == [] + + +def test_typesafe_gateway_preserves_missing_usage(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + body = {"model": "jev-1.13.0", "answers": {"exp": {"type": "noul", "noul": 0.8}}} + handle = _register(scenario, body) + model = _deployment(scenario, handle, _PROVIDERS[1]) + response = gateway.request( + "POST", + "/v1/systemone", + { + "model": model, + "state": {"finding": "x"}, + "questions": {"exp": {"type": "noul", "instructions": "Is it exploitable?"}}, + }, + ) + assert response.status_code == 200, response.text + assert response.json()["usage"] is None + assert response.json()["answers"] == body["answers"] diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 84e7a970195..e2385a2e215 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -3799,3 +3799,35 @@ def test_nace_upstream_failures_preserve_status_and_message( ) assert getattr(error.value, "status_code") == status assert upstream.call_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True]) +@pytest.mark.parametrize( + "usage", [None, {"input_tokens": 0, "output_tokens": 0}, {"input_tokens": 12, "output_tokens": 1}] +) +async def test_native_decisions_distinguishes_missing_usage_from_reported_zero( + is_async: bool, usage: dict[str, int] | None, respx_mock: respx.MockRouter +) -> None: + body = { + **{key: value for key, value in _RESPONSE.items() if key != "usage"}, + "model": "jev-1.13.0", + **({"usage": usage} if usage is not None else {}), + } + respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=body) + params = {"model": "typesafe/jev-1.13.0", "state": "review", "questions": _QUESTIONS, "api_key": "synthetic-key"} + response = await litellm.adecisions(**params) if is_async else litellm.decisions(**params) + + assert isinstance(response, DecisionsResponse) + if usage is None: + assert response.usage is None + assert response._hidden_params.get("response_cost") is None + with pytest.raises(ValueError, match="provider did not report usage"): + litellm.completion_cost(completion_response=response) + else: + assert response.usage is not None + assert response.usage.input_tokens == usage["input_tokens"] + assert response.usage.output_tokens == usage["output_tokens"] + expected = litellm.completion_cost(completion_response=response) + assert response._hidden_params["response_cost"] == pytest.approx(expected) + assert (expected == 0) == (usage["input_tokens"] == usage["output_tokens"] == 0) From 84754cca0a560f2488f7f844d43d5f6583f07d32 Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Sat, 24 Oct 2026 17:52:01 -0700 Subject: [PATCH 041/155] fix(groq): allow streamed reasoning replies in history (#47931) (#48268) --- litellm/llms/groq/chat/transformation.py | 4 +- .../chat/test_groq_chat_transformation.py | 66 +++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 349a4b4ab87..23f68335ea9 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -11,7 +11,7 @@ from pydantic import BaseModel, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.litellm_core_utils.prompt_templates.common_utils import strip_litellm_response_only_fields +from litellm.litellm_core_utils.prompt_templates.common_utils import strip_litellm_internal_message_fields from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, ) @@ -154,7 +154,7 @@ class GroqChatConfig(OpenAILikeChatConfig): for k, v in _message.items(): if v is not None: new_message[k] = v - messages[idx] = strip_litellm_response_only_fields(new_message) + messages[idx] = strip_litellm_internal_message_fields(new_message) if is_async: return super()._transform_messages(messages=messages, model=model, is_async=True) diff --git a/tests/unit/llms/groq/chat/test_groq_chat_transformation.py b/tests/unit/llms/groq/chat/test_groq_chat_transformation.py index 95611e344dc..7aa8c65215e 100644 --- a/tests/unit/llms/groq/chat/test_groq_chat_transformation.py +++ b/tests/unit/llms/groq/chat/test_groq_chat_transformation.py @@ -1,4 +1,5 @@ import logging +from typing import Final from unittest.mock import patch import httpx @@ -523,3 +524,68 @@ def test_tool_search_reply_sent_to_groq_drops_litellm_response_fields(): } ], } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True]) +@pytest.mark.parametrize("with_tool_call", [False, True]) +async def test_streamed_reasoning_reply_can_be_replayed(is_async: bool, with_tool_call: bool) -> None: + tool_calls: Final = [ + { + "index": 0, + "id": "call_weather", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city":"Paris"}'}, + } + ] + handler: Final = GroqChatCompletionStreamingHandler(streaming_response=iter(()), sync_stream=True) + chunk: Final = handler.chunk_parser( + { + "id": "chatcmpl-reasoning", + "object": "chat.completion.chunk", + "created": 1769511767, + "model": "openai/gpt-oss-20b", + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "reasoning": "I should check the weather.", + "content": "" if with_tool_call else "Hello there!", + **({"tool_calls": tool_calls} if with_tool_call else {}), + }, + "finish_reason": "tool_calls" if with_tool_call else "stop", + } + ], + } + ) + reply: Final = litellm.stream_chunk_builder([chunk]).choices[0].message + assert reply.reasoning_content == "I should check the weather." + history: Final = [reply.model_dump(exclude_none=True)] + config: Final = GroqChatConfig() + + if is_async: + replayed = await config._transform_messages(messages=history, model="openai/gpt-oss-20b", is_async=True) + else: + replayed = config._transform_messages(messages=history, model="openai/gpt-oss-20b", is_async=False) + + assert replayed == [ + { + "role": "assistant", + "content": "" if with_tool_call else "Hello there!", + **( + { + "tool_calls": [ + { + "id": "call_weather", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city":"Paris"}'}, + } + ] + } + if with_tool_call + else {} + ), + } + ] + assert reply.reasoning_content == "I should check the weather." From 24a7bf6af728272eca2ba3277a2f3cdfa7e60863 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 17:57:09 -0700 Subject: [PATCH 042/155] fix(proxy): reconcile team member total_spend with current cycle spend (#48260) * fix(proxy): reconcile team member total_spend with current cycle spend Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): read /team/list as a list and drain the scripted provider Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(repositories): compose the membership reset writer and drop SQL text asserts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: type the cascade repair test and flatten memberships without a stacked comprehension Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: keep the reset job test file's existing typing untouched Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: joshua Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/common_utils/reset_budget_job.py | 4 +- litellm/proxy/db/db_spend_update_writer.py | 4 +- .../management_endpoints/team_endpoints.py | 38 ++- litellm/repositories/prisma_protocols.py | 2 + litellm/repositories/unit_of_work.py | 41 ++- ..._team_member_total_spend_reconciliation.py | 273 ++++++++++++++++++ .../common_utils/test_reset_budget_job.py | 33 +++ .../proxy/db/test_db_spend_update_writer.py | 5 +- .../test_team_endpoints.py | 104 ++++++- tests/unit/repositories/test_unit_of_work.py | 21 +- .../src/components/team/MyUserTab.test.tsx | 25 ++ .../src/components/team/MyUserTab.tsx | 18 +- .../components/team/TeamMemberTab.test.tsx | 42 ++- .../src/components/team/TeamMemberTab.tsx | 19 +- .../src/utils/budgetUtils.test.ts | 25 +- ui/litellm-dashboard/src/utils/budgetUtils.ts | 38 +++ 16 files changed, 655 insertions(+), 37 deletions(-) create mode 100644 tests/integration/spend/test_team_member_total_spend_reconciliation.py diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index ccf2232e379..59fd00cec43 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -65,6 +65,7 @@ from litellm.repositories.table_repositories import ( from litellm.repositories.team_repository import TeamRepository from litellm.repositories.unit_of_work import ( LinkedSpendResetWrites, + SpendResetWrites, budget_cascade_unit_of_work, enduser_spend_reset_unit_of_work, spend_reset_unit_of_work, @@ -235,7 +236,7 @@ def _enduser_invalidation_where(budget_ids: Sequence[str]) -> dict[str, object]: def _queue_budget_linked_resets( - writes: LinkedSpendResetWrites, + writes: SpendResetWrites, cascade: "_BudgetCascade", extra: Mapping[str, object] = MappingProxyType({}), ) -> None: @@ -908,6 +909,7 @@ class ResetBudgetJob: db_span("reset_budget_cascade", "LiteLLM_BudgetTable"), budget_cascade_unit_of_work(self._new_batch) as uow, ): + uow.team_memberships.queue_total_spend_repair(cascade.budget_ids) _queue_budget_linked_resets(uow.team_memberships, cascade) _queue_budget_linked_resets(uow.keys, cascade, extra=_LINKED_KEYS_WHERE) _queue_budget_linked_resets(uow.organizations, cascade, extra=_SPENT_ROWS_WHERE) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 24ab09283f1..8ac2597c99b 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -472,6 +472,8 @@ _TEAM_ADVISORY_LOCK_SQL: Final = "SELECT pg_advisory_xact_lock(hashtext($1)) IS # recreates the member. The rows travel as one JSON document, not as a numeric array: Prisma # types a raw array parameter from the first batch a connection sees, so after an all-$0 batch # (integers) every later fractional batch on that connection failed with "improper binary format". +# total_spend is floored at spend before adding: rows written before total_spend existed (migration +# 20260421135425, default 0) carry their history only in spend, so a plain increment keeps them behind forever. _TEAM_MEMBER_SPEND_SQL: Final = """ INSERT INTO "LiteLLM_TeamMembership" (user_id, team_id, spend, total_spend) SELECT member.user_id, member.team_id, member.cost, member.cost @@ -487,7 +489,7 @@ WHERE EXISTS ( ) ON CONFLICT (user_id, team_id) DO UPDATE SET spend = "LiteLLM_TeamMembership".spend + EXCLUDED.spend, - total_spend = "LiteLLM_TeamMembership".total_spend + EXCLUDED.total_spend + total_spend = GREATEST("LiteLLM_TeamMembership".total_spend, "LiteLLM_TeamMembership".spend) + EXCLUDED.total_spend """ diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 95d4d03a7f0..1fa18e9b2b8 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -1013,7 +1013,7 @@ async def get_all_team_memberships( returned_tm: Final[list[LiteLLM_TeamMembership]] = [] for tm in team_memberships: - returned_tm.append(LiteLLM_TeamMembership.model_validate(tm.model_dump())) + returned_tm.append(with_total_spend_floor(LiteLLM_TeamMembership.model_validate(tm.model_dump()))) return returned_tm @@ -4202,6 +4202,23 @@ def _check_not_resetting_own_spend(user_id: str, user_api_key_dict: UserAPIKeyAu _raise_reset_spend_error(status.HTTP_403_FORBIDDEN, "Cannot reset your own spend. Ask a proxy admin.") +# One statement so the validated spend cannot go stale between the read and the write: the WHERE re-checks +# reset_to <= spend, and total_spend is floored at spend before spend drops so the history survives the reset. +_RESET_TEAM_MEMBER_SPEND_SQL: Final = """ +UPDATE "LiteLLM_TeamMembership" +SET total_spend = GREATEST(total_spend, spend), spend = $3::float8 +WHERE user_id = $1 AND team_id = $2 AND spend >= $3::float8 +""" + + +def with_total_spend_floor(membership: LiteLLM_TeamMembership) -> LiteLLM_TeamMembership: + """Rows written before total_spend existed (migration 20260421135425) hold their history only in spend.""" + spend: Final = membership.spend or 0.0 + if (membership.total_spend or 0.0) >= spend: + return membership + return membership.model_copy(update={"total_spend": spend}) + + def _raise_reset_spend_error(status_code: int, message: str) -> NoReturn: detail: Final = {"error": message} raise HTTPException(status_code=status_code, detail=detail) @@ -4286,10 +4303,12 @@ async def reset_team_member_spend_fn( current_spend: Final = membership.spend or 0.0 reset_to: Final = _validate_team_member_reset_spend_value(data.reset_to, membership) - await _team_membership_db(prisma_client).update( - where=membership_where, - data={"spend": reset_to}, - ) + reset_rows: Final = await _execute_raw(prisma_client)(_RESET_TEAM_MEMBER_SPEND_SQL, user_id, team_id, reset_to) + if reset_rows == 0: + _raise_reset_spend_error( + status.HTTP_409_CONFLICT, + f"Member spend changed since it was read; reset_to ({reset_to}) must be <= current spend. Retry.", + ) await invalidate_team_member_spend_state( user_id=user_id, @@ -5548,16 +5567,17 @@ async def team_member_me( litellm_budget_table=None, ) + floored: Final = with_total_spend_floor(membership) return TeamMemberInfoResponse( user_id=caller_user_id, team_id=team_id, team_alias=team_table.team_alias, role=member_role, user_email=user_email, - spend=membership.spend, - total_spend=membership.total_spend, - budget_id=membership.budget_id, - litellm_budget_table=membership.litellm_budget_table, + spend=floored.spend, + total_spend=floored.total_spend, + budget_id=floored.budget_id, + litellm_budget_table=floored.litellm_budget_table, ) diff --git a/litellm/repositories/prisma_protocols.py b/litellm/repositories/prisma_protocols.py index c42301a9316..63cfcbc7b45 100644 --- a/litellm/repositories/prisma_protocols.py +++ b/litellm/repositories/prisma_protocols.py @@ -171,4 +171,6 @@ class PrismaBatch(Protocol): @property def litellm_projecttable(self) -> BatchTable: ... + def execute_raw(self, query: str, *args: object) -> None: ... + async def commit(self) -> None: ... diff --git a/litellm/repositories/unit_of_work.py b/litellm/repositories/unit_of_work.py index 95f4f5e81be..f6aad883479 100644 --- a/litellm/repositories/unit_of_work.py +++ b/litellm/repositories/unit_of_work.py @@ -15,11 +15,11 @@ instead of full-model writes, which trip ``prisma.errors.DataError`` on rows carrying fields the update input type rejects (see #27730). """ -from collections.abc import AsyncGenerator, Callable, Mapping +from collections.abc import AsyncGenerator, Callable, Mapping, Sequence from contextlib import asynccontextmanager from dataclasses import dataclass from datetime import datetime -from typing import Final +from typing import Final, Protocol, runtime_checkable from litellm.repositories.prisma_protocols import BatchTable, PrismaBatch @@ -62,6 +62,13 @@ class TeamSpendResetWrites: ) +@runtime_checkable +class SpendResetWrites(Protocol): + def queue_spend_zero(self, where: Mapping[str, object]) -> None: ... + + def queue_spend_decrement(self, where: Mapping[str, object], amount: float) -> None: ... + + @dataclass(frozen=True, slots=True) class LinkedSpendResetWrites: table: BatchTable @@ -78,6 +85,30 @@ class LinkedSpendResetWrites: ) +# Membership rows written before total_spend existed (migration 20260421135425, default 0) hold their +# history only in spend. Zeroing spend would erase it, so the floor lands in the same batch, ahead of the reset. +_TEAM_MEMBERSHIP_TOTAL_SPEND_REPAIR_SQL: Final = """ +UPDATE "LiteLLM_TeamMembership" +SET total_spend = spend +WHERE budget_id = ANY($1::text[]) AND spend > total_spend +""" + + +@dataclass(frozen=True, slots=True) +class TeamMembershipSpendResetWrites: + resets: LinkedSpendResetWrites + batch: PrismaBatch + + def queue_spend_zero(self, where: Mapping[str, object]) -> None: + self.resets.queue_spend_zero(where) + + def queue_spend_decrement(self, where: Mapping[str, object], amount: float) -> None: + self.resets.queue_spend_decrement(where, amount) + + def queue_total_spend_repair(self, budget_ids: Sequence[str]) -> None: + self.batch.execute_raw(_TEAM_MEMBERSHIP_TOTAL_SPEND_REPAIR_SQL, list(budget_ids)) + + @dataclass(frozen=True, slots=True) class BudgetWindowWrites: table: BatchTable @@ -106,7 +137,7 @@ class BudgetCascadeUnitOfWork: ``enduser_spend_reset_unit_of_work`` because one tier can gate millions. """ - team_memberships: LinkedSpendResetWrites + team_memberships: TeamMembershipSpendResetWrites keys: LinkedSpendResetWrites organizations: LinkedSpendResetWrites tags: LinkedSpendResetWrites @@ -132,7 +163,9 @@ async def budget_cascade_unit_of_work( ) -> AsyncGenerator[BudgetCascadeUnitOfWork, None]: batch = new_batch() yield BudgetCascadeUnitOfWork( - team_memberships=LinkedSpendResetWrites(table=batch.litellm_teammembership), + team_memberships=TeamMembershipSpendResetWrites( + resets=LinkedSpendResetWrites(table=batch.litellm_teammembership), batch=batch + ), keys=LinkedSpendResetWrites(table=batch.litellm_verificationtoken), organizations=LinkedSpendResetWrites(table=batch.litellm_organizationtable), tags=LinkedSpendResetWrites(table=batch.litellm_tagtable), diff --git a/tests/integration/spend/test_team_member_total_spend_reconciliation.py b/tests/integration/spend/test_team_member_total_spend_reconciliation.py new file mode 100644 index 00000000000..f5ccae0d12b --- /dev/null +++ b/tests/integration/spend/test_team_member_total_spend_reconciliation.py @@ -0,0 +1,273 @@ +from __future__ import annotations + +import asyncio +import json +import os +import uuid +from concurrent.futures import ThreadPoolExecutor +from typing import Final + +import httpx +import pytest +from pydantic import JsonValue, TypeAdapter + +import litellm +from litellm.caching.dual_cache import DualCache +from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob +from litellm.proxy.utils import PrismaClient, ProxyLogging +from tests.integration._support.client import Gateway, Scenario, eventually, list_value, object_value, string_value +from tests.integration._support.database import read_rows, write_rows +from tests.integration._support.provider import PROVIDER_URL, SharedProvider +from tests.integration._support.wire import Reply + +_CALL_COST: Final = 0.02 +JSON_VALUE: Final = TypeAdapter(JsonValue) +_LEGACY_SPEND: Final = 612.4 +_LEGACY_TOTAL: Final = 148.1 +_HEALTHY_SPEND: Final = 12.25 +_HEALTHY_TOTAL: Final = 97.8 + + +def _completion() -> Reply: + return Reply( + body=json.dumps( + { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "scripted"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + ).encode() + ) + + +def _priced_model(scenario: Scenario) -> str: + return scenario.model(api_base=f"{PROVIDER_URL}/v1", input_cost_per_token=0.001, output_cost_per_token=0.002) + + +def _member_with_budget(gateway: Gateway, scenario: Scenario, team_id: str, *, max_budget: float) -> str: + user_id: Final = scenario.user(user_role="internal_user") + gateway.post( + "/team/member_add", + {"team_id": team_id, "member": {"role": "user", "user_id": user_id}, "max_budget_in_team": max_budget}, + ) + return user_id + + +def _seed_membership(user_id: str, team_id: str, *, spend: float, total_spend: float) -> None: + write_rows( + 'UPDATE "LiteLLM_TeamMembership" SET spend = %s::float8, total_spend = %s::float8 ' + "WHERE user_id = %s AND team_id = %s", + (str(spend), str(total_spend), user_id, team_id), + ) + + +def _membership(user_id: str, team_id: str) -> tuple[float, float]: + rows: Final = read_rows( + 'SELECT spend, total_spend FROM "LiteLLM_TeamMembership" WHERE user_id = %s AND team_id = %s', + (user_id, team_id), + ) + assert len(rows) == 1, rows + return float(str(rows[0]["spend"])), float(str(rows[0]["total_spend"])) + + +def _membership_reaches(user_id: str, team_id: str, *, total_spend: float) -> tuple[float, float]: + return eventually( + lambda: _membership(user_id, team_id), + lambda row: row[1] == pytest.approx(total_spend), + seconds=70, + ) + + +def _served(gateway: Gateway, provider: SharedProvider, model: str, key: str) -> None: + provider.expect(_completion()) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": f"reconcile {uuid.uuid4().hex}"}]}, + key=key, + ) + assert response.status_code == 200, response.text + assert len(provider.received()) == 1 + + +def _reset_spend(gateway: Gateway, team_id: str, user_id: str, reset_to: float) -> httpx.Response: + return gateway.request("POST", f"/team/{team_id}/member/{user_id}/reset_spend", {"reset_to": reset_to}) + + +def _membership_figures(row: JsonValue) -> tuple[str, tuple[float, float]]: + membership: Final = object_value(row) + return string_value(membership["user_id"]), ( + float(str(membership["spend"])), + float(str(membership["total_spend"])), + ) + + +def _reported_memberships(gateway: Gateway, path: str, params: dict[str, str]) -> dict[str, tuple[float, float]]: + """/team/info answers with one team object, /team/list with a list of them.""" + response: Final = gateway.request("GET", path, params=params) + assert response.status_code == 200, f"GET {path}: {response.status_code} {response.text}" + body: Final = JSON_VALUE.validate_json(response.content) + teams: Final = [object_value(body)] if isinstance(body, dict) else [object_value(team) for team in list_value(body)] + rows: Final = [ + row for team in teams for row in list_value(team["team_memberships"]) + ] # comprehension-ok: flatten memberships across teams + return dict(_membership_figures(row) for row in rows) + + +def test_a_flush_lifts_a_legacy_total_to_the_cycle_spend_and_leaves_a_healthy_row_on_its_baseline( + gateway: Gateway, provider: SharedProvider +) -> None: + with gateway.scenario() as scenario: + model: Final = _priced_model(scenario) + team_id: Final = scenario.team(models=[model]) + legacy: Final = _member_with_budget(gateway, scenario, team_id, max_budget=10_000) + healthy: Final = _member_with_budget(gateway, scenario, team_id, max_budget=10_000) + _seed_membership(legacy, team_id, spend=_LEGACY_SPEND, total_spend=_LEGACY_TOTAL) + _seed_membership(healthy, team_id, spend=_HEALTHY_SPEND, total_spend=_HEALTHY_TOTAL) + legacy_key: Final = scenario.key(user_id=legacy, team_id=team_id, models=[model]) + healthy_key: Final = scenario.key(user_id=healthy, team_id=team_id, models=[model]) + + _served(gateway, provider, model, legacy_key) + _served(gateway, provider, model, healthy_key) + + legacy_row: Final = _membership_reaches( + legacy, team_id, total_spend=max(_LEGACY_SPEND, _LEGACY_TOTAL) + _CALL_COST + ) + assert legacy_row[0] == pytest.approx(_LEGACY_SPEND + _CALL_COST) + healthy_row: Final = _membership_reaches(healthy, team_id, total_spend=_HEALTHY_TOTAL + _CALL_COST) + assert healthy_row[0] == pytest.approx(_HEALTHY_SPEND + _CALL_COST) + + +def test_team_info_and_team_list_report_a_legacy_total_no_lower_than_the_cycle_spend(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + team_id: Final = scenario.team() + legacy: Final = _member_with_budget(gateway, scenario, team_id, max_budget=10_000) + healthy: Final = _member_with_budget(gateway, scenario, team_id, max_budget=10_000) + _seed_membership(legacy, team_id, spend=_LEGACY_SPEND, total_spend=_LEGACY_TOTAL) + _seed_membership(healthy, team_id, spend=_HEALTHY_SPEND, total_spend=_HEALTHY_TOTAL) + + info: Final = _reported_memberships(gateway, "/team/info", {"team_id": team_id}) + listed: Final = _reported_memberships(gateway, "/team/list", {"team_id": team_id}) + + for reported in (info, listed): + assert reported[legacy] == (pytest.approx(_LEGACY_SPEND), pytest.approx(_LEGACY_SPEND)) + assert reported[healthy] == (pytest.approx(_HEALTHY_SPEND), pytest.approx(_HEALTHY_TOTAL)) + assert _membership(legacy, team_id) == (pytest.approx(_LEGACY_SPEND), pytest.approx(_LEGACY_TOTAL)) + + +@pytest.mark.parametrize("reset_to", [0.0, 100.0], ids=["to_zero", "to_nonzero"]) +def test_an_admin_reset_keeps_the_legacy_cycle_spend_as_the_lifetime_floor(gateway: Gateway, reset_to: float) -> None: + with gateway.scenario() as scenario: + team_id: Final = scenario.team() + legacy: Final = _member_with_budget(gateway, scenario, team_id, max_budget=10_000) + _seed_membership(legacy, team_id, spend=_LEGACY_SPEND, total_spend=_LEGACY_TOTAL) + + response: Final = _reset_spend(gateway, team_id, legacy, reset_to) + + assert response.status_code == 200, response.text + assert _membership(legacy, team_id) == (pytest.approx(reset_to), pytest.approx(_LEGACY_SPEND)) + + +def test_concurrent_admin_resets_leave_one_winner_and_a_lifetime_floor_at_the_old_cycle_spend( + gateway: Gateway, +) -> None: + """Two admins reset the same member at once. Each UPDATE re-checks reset_to <= spend, so + the higher reset either lands first or is refused with 409 once the lower one has landed. + Whatever the order, the row ends at the lower value with the old cycle spend preserved.""" + with gateway.scenario() as scenario: + team_id: Final = scenario.team() + legacy: Final = _member_with_budget(gateway, scenario, team_id, max_budget=10_000) + _seed_membership(legacy, team_id, spend=_LEGACY_SPEND, total_spend=_LEGACY_TOTAL) + + with ThreadPoolExecutor(max_workers=2) as pool: + higher, lower = pool.map(lambda value: _reset_spend(gateway, team_id, legacy, value), (600.0, 500.0)) + + assert lower.status_code == 200, lower.text + assert higher.status_code in (200, 409), higher.text + assert _membership(legacy, team_id) == (pytest.approx(500.0), pytest.approx(_LEGACY_SPEND)) + + +def test_an_admin_reset_racing_a_flush_ends_with_the_lifetime_equal_to_old_cycle_plus_cost( + gateway: Gateway, provider: SharedProvider +) -> None: + with gateway.scenario() as scenario: + model: Final = _priced_model(scenario) + team_id: Final = scenario.team(models=[model]) + legacy: Final = _member_with_budget(gateway, scenario, team_id, max_budget=10_000) + _seed_membership(legacy, team_id, spend=_LEGACY_SPEND, total_spend=_LEGACY_TOTAL) + key: Final = scenario.key(user_id=legacy, team_id=team_id, models=[model]) + + _served(gateway, provider, model, key) + reset: Final = _reset_spend(gateway, team_id, legacy, 0.0) + assert reset.status_code == 200, reset.text + + spend, total = _membership_reaches(legacy, team_id, total_spend=_LEGACY_SPEND + _CALL_COST) + assert spend in (pytest.approx(0.0), pytest.approx(_CALL_COST)), (spend, total) + assert total >= spend + + +def _expire_member_budget(user_id: str, team_id: str, *, budget_duration: str) -> None: + write_rows( + "UPDATE \"LiteLLM_BudgetTable\" SET budget_duration = %s, budget_reset_at = NOW() - INTERVAL '1 hour' " + 'WHERE budget_id = (SELECT budget_id FROM "LiteLLM_TeamMembership" WHERE user_id = %s AND team_id = %s)', + (budget_duration, user_id, team_id), + ) + + +async def _run_scheduled_reset() -> None: + client: Final = PrismaClient( + database_url=os.environ["DATABASE_URL"], proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()) + ) + await client.connect() + try: + await ResetBudgetJob( + ProxyLogging(user_api_key_cache=DualCache()), client + ).reset_budget_for_litellm_budget_table() + finally: + await client.db.disconnect() + + +@pytest.mark.parametrize( + "spend, max_budget, carried", + [(8.0, 10.0, 0.0), (10.0, 10.0, 0.0), (15.0, 10.0, 5.0)], + ids=["below_cap", "at_cap", "above_cap_carries_overage"], +) +def test_a_scheduled_reset_lifts_the_legacy_total_before_zeroing_or_carrying_the_cycle_spend( + gateway: Gateway, monkeypatch: pytest.MonkeyPatch, spend: float, max_budget: float, carried: float +) -> None: + monkeypatch.setattr(litellm, "budget_rollover", True) + with gateway.scenario() as scenario: + team_id: Final = scenario.team() + legacy: Final = _member_with_budget(gateway, scenario, team_id, max_budget=max_budget) + _seed_membership(legacy, team_id, spend=spend, total_spend=3.0) + _expire_member_budget(legacy, team_id, budget_duration="1d") + + asyncio.run(_run_scheduled_reset()) + + assert _membership(legacy, team_id) == (pytest.approx(carried), pytest.approx(spend)) + asyncio.run(_run_scheduled_reset()) + assert _membership(legacy, team_id) == (pytest.approx(carried), pytest.approx(spend)) + + +def test_a_scheduled_reset_racing_a_flush_ends_with_the_lifetime_equal_to_old_cycle_plus_cost( + gateway: Gateway, provider: SharedProvider +) -> None: + with gateway.scenario() as scenario: + model: Final = _priced_model(scenario) + team_id: Final = scenario.team(models=[model]) + legacy: Final = _member_with_budget(gateway, scenario, team_id, max_budget=10_000) + _seed_membership(legacy, team_id, spend=_LEGACY_SPEND, total_spend=_LEGACY_TOTAL) + _expire_member_budget(legacy, team_id, budget_duration="1d") + key: Final = scenario.key(user_id=legacy, team_id=team_id, models=[model]) + + _served(gateway, provider, model, key) + asyncio.run(_run_scheduled_reset()) + + spend, total = _membership_reaches(legacy, team_id, total_spend=_LEGACY_SPEND + _CALL_COST) + assert spend in (pytest.approx(0.0), pytest.approx(_CALL_COST)), (spend, total) + assert total >= spend diff --git a/tests/unit/proxy/common_utils/test_reset_budget_job.py b/tests/unit/proxy/common_utils/test_reset_budget_job.py index 70f7774e691..183dd7919c1 100644 --- a/tests/unit/proxy/common_utils/test_reset_budget_job.py +++ b/tests/unit/proxy/common_utils/test_reset_budget_job.py @@ -113,6 +113,9 @@ class MockBatcher: self.litellm_projecttable = _Table("project", self) self.litellm_endusertable = _Table("enduser", self) + def execute_raw(self, query: str, *args: object) -> None: + self.calls.append({"table": "raw", "op": "execute_raw", "query": query, "args": args}) + async def commit(self): self.committed = True return self.calls @@ -2112,6 +2115,7 @@ def test_budget_cascade_writes_land_in_a_single_transaction(reset_budget_job, mo assert {(call["table"], call["op"]) for call in enduser_page.calls} == {("enduser", "update_many")} assert batcher.committed is True assert {(call["table"], call["op"]) for call in batcher.calls} == { + ("raw", "execute_raw"), ("team_membership", "update_many"), ("key", "update_many"), ("org", "update_many"), @@ -3393,6 +3397,35 @@ def test_budget_cascade_carries_overage_per_tier_when_rollover_enabled( counter_cache.in_memory_cache.delete_cache.assert_any_call(key="spend:team_member:member-1:team-1") +def test_budget_cascade_floors_membership_total_spend_before_zeroing_spend( + rollover_enabled: None, + reset_budget_job: ResetBudgetJob, + mock_prisma_client: MockPrismaClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Membership rows written before total_spend existed hold their history only in + spend. The cascade lifts total_spend to spend for the expiring tier's rows, in the + same batch and ahead of the zero/decrement, so the reset cannot erase it.""" + _make_counter_invalidation_job(monkeypatch) + budget: Final = _budget_row(budget_id="budget-legacy", budget_duration="7d", max_budget=10.0) + mock_prisma_client.data["budget"] = [budget] + membership: Final = type( + "Membership", + (), + {"user_id": "member-1", "team_id": "team-1", "spend": 15.0, "budget_id": "budget-legacy"}, + ) + mock_prisma_client.db.litellm_teammembership.set_find_many_results([membership]) + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + writes: Final = mock_prisma_client.db.batch_calls + repair_index: Final = next(i for i, call in enumerate(writes) if call["op"] == "execute_raw") + first_membership_reset: Final = next(i for i, call in enumerate(writes) if call["table"] == "team_membership") + assert repair_index < first_membership_reset, writes + assert writes[repair_index]["args"] == (["budget-legacy"],) + assert mock_prisma_client.db.batchers[-1].committed + + def test_budget_cascade_carries_enduser_overage_when_rollover_enabled( rollover_enabled, reset_budget_job, mock_prisma_client, monkeypatch ): diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index 6071569b806..a42b6404278 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -1128,7 +1128,10 @@ async def test_commit_spend_updates_to_db_writes_team_member_spend_in_one_roster assert "members_with_roles @> jsonb_build_array(jsonb_build_object('user_id', member.user_id))" in statement assert "ON CONFLICT (user_id, team_id) DO UPDATE" in statement assert 'spend = "LiteLLM_TeamMembership".spend + EXCLUDED.spend' in statement - assert 'total_spend = "LiteLLM_TeamMembership".total_spend + EXCLUDED.total_spend' in statement + assert ( + 'total_spend = GREATEST("LiteLLM_TeamMembership".total_spend, "LiteLLM_TeamMembership".spend)' + " + EXCLUDED.total_spend" + ) in statement @pytest.mark.asyncio diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index 4feb7fb5bc8..f727448f3ac 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -12561,6 +12561,36 @@ async def test_team_member_me_returns_caller_membership(mock_db_client): assert call_kwargs["team_id"] == team_id +@pytest.mark.asyncio +@pytest.mark.parametrize( + "spend, stored_total, reported_total", + [(612.4, 148.1, 612.4), (12.25, 97.8, 97.8)], + ids=["legacy_row_floors_to_cycle_spend", "healthy_row_keeps_its_total"], +) +async def test_team_member_me_reports_total_spend_no_lower_than_cycle_spend( + mock_db_client, spend, stored_total, reported_total +): + from fastapi import Request + + from litellm.proxy.management_endpoints.team_endpoints import team_member_me + + team_id = "team-me-floor" + caller_id = "alice@example.com" + team = _build_team_for_me(team_id, [{"user_id": caller_id, "user_email": None, "role": "user"}]) + membership = _build_membership_for_me(caller_id, team_id, spend=spend).model_copy( + update={"total_spend": stored_total} + ) + p_team, p_membership, p_user = _patch_member_me_helpers(team=team, membership=membership, user=None) + with p_team, p_membership, p_user: + response = await team_member_me( + http_request=MagicMock(spec=Request), + team_id=team_id, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id=caller_id), + ) + + assert (response.spend, response.total_spend) == (spend, reported_total) + + @pytest.mark.asyncio async def test_team_member_me_matches_email_only_member(mock_db_client): """ @@ -16608,7 +16638,7 @@ async def test_reset_team_member_spend_fn_success(monkeypatch): litellm_budget_table=LiteLLM_BudgetTable(budget_id="b1", max_budget=50.0), ) mock_prisma_client.db.litellm_teammembership.find_unique = AsyncMock(return_value=membership_row) - mock_prisma_client.db.litellm_teammembership.update = AsyncMock(return_value=membership_row) + mock_prisma_client.db.execute_raw = AsyncMock(return_value=1) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", real_cache) @@ -16631,15 +16661,76 @@ async def test_reset_team_member_spend_fn_success(monkeypatch): assert response["spend"] == 0.0 assert response["previous_spend"] == 10.0 assert response["max_budget"] == 50.0 - mock_prisma_client.db.litellm_teammembership.update.assert_awaited_once_with( - where={"user_id_team_id": {"user_id": "member-1", "team_id": "team-1"}}, - data={"spend": 0.0}, - ) + statement, reset_user_id, reset_team_id, reset_to = mock_prisma_client.db.execute_raw.await_args.args + assert (reset_user_id, reset_team_id, reset_to) == ("member-1", "team-1", 0.0) + assert "total_spend = GREATEST(total_spend, spend)" in statement + assert "spend >= $3" in statement assert await real_cache.async_get_cache(key="team-1_member-1") is None assert await real_cache.async_get_cache(key="team_membership:member-1:team-1") is None assert real_spend_counter_cache.in_memory_cache.get_cache(key="spend:team_member:member-1:team-1") == 0.0 +@pytest.mark.asyncio +async def test_reset_team_member_spend_fn_stale_spend_returns_409_and_keeps_caches(monkeypatch): + """The reset validates reset_to against a spend it read a moment ago. If the row moved + underneath it (a flush landed, another admin reset first) the guarded UPDATE matches no + row, and the endpoint must say so instead of reporting a reset that never happened.""" + from litellm.caching.dual_cache import DualCache + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + mock_prisma_client = MagicMock() + real_cache = UserApiKeyCache() + await real_cache.async_set_cache(key="team_membership:member-1:team-1", value="cached-membership") + real_spend_counter_cache = DualCache() + real_spend_counter_cache.in_memory_cache.set_cache(key="spend:team_member:member-1:team-1", value=999.0) + + membership_row = LiteLLM_TeamMembership(user_id="member-1", team_id="team-1", spend=10.0) + mock_prisma_client.db.litellm_teammembership.find_unique = AsyncMock(return_value=membership_row) + mock_prisma_client.db.execute_raw = AsyncMock(return_value=0) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", real_cache) + monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", MagicMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.spend_counter_cache", real_spend_counter_cache) + + with patch( # test-quality-ok: no live DB here; matches this file's established convention for endpoint-logic unit tests + "litellm.proxy.management_endpoints.team_endpoints.get_team_object", + AsyncMock(return_value=LiteLLM_TeamTable(team_id="team-1")), + ), pytest.raises(HTTPException) as exc_info: + await reset_team_member_spend_fn( + team_id="team-1", + user_id="member-1", + data=ResetSpendRequest(reset_to=5.0), + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin", user_id="admin-user" + ), + ) + + assert exc_info.value.status_code == 409 + assert await real_cache.async_get_cache(key="team_membership:member-1:team-1") == "cached-membership" + assert real_spend_counter_cache.in_memory_cache.get_cache(key="spend:team_member:member-1:team-1") == 999.0 + + +@pytest.mark.asyncio +async def test_get_all_team_memberships_floors_total_spend_at_current_cycle_spend(): + """Rows written before total_spend existed carry their history only in spend. /team/info + and /team/list both read through this helper, so a legacy row reports its lifetime spend + as at least the current cycle, while a healthy row keeps its own total.""" + from litellm.proxy.management_endpoints.team_endpoints import get_all_team_memberships + + legacy_row = MagicMock() + legacy_row.model_dump = lambda: {"user_id": "legacy", "team_id": "team-1", "spend": 612.4, "total_spend": 148.1} + healthy_row = MagicMock() + healthy_row.model_dump = lambda: {"user_id": "healthy", "team_id": "team-1", "spend": 12.25, "total_spend": 97.8} + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_teammembership.find_many = AsyncMock(return_value=[legacy_row, healthy_row]) + + legacy, healthy = await get_all_team_memberships(mock_prisma_client, ["team-1"]) + + assert (legacy.spend, legacy.total_spend) == (612.4, 612.4) + assert (healthy.spend, healthy.total_spend) == (12.25, 97.8) + + @pytest.mark.asyncio async def test_reset_team_member_spend_fn_membership_not_found(monkeypatch): mock_prisma_client = MagicMock() @@ -16755,7 +16846,7 @@ async def test_reset_team_member_spend_fn_proxy_admin_can_reset_own_spend(monkey membership_row = LiteLLM_TeamMembership(user_id="admin-user", team_id="team-1", spend=10.0) mock_prisma_client.db.litellm_teammembership.find_unique = AsyncMock(return_value=membership_row) - mock_prisma_client.db.litellm_teammembership.update = AsyncMock(return_value=membership_row) + mock_prisma_client.db.execute_raw = AsyncMock(return_value=1) with patch( # test-quality-ok: no live DB here; matches this file's established convention for endpoint-logic unit tests "litellm.proxy.management_endpoints.team_endpoints.get_team_object", @@ -17994,6 +18085,7 @@ async def test_update_team_team_admin_resending_stored_member_defaults_is_not_an disable_audit_logging_for_mocked_team, ): import contextlib + from litellm.models.budget import LiteLLM_BudgetTable with contextlib.ExitStack() as stack: diff --git a/tests/unit/repositories/test_unit_of_work.py b/tests/unit/repositories/test_unit_of_work.py index a08d44b9b4d..082a29b7001 100644 --- a/tests/unit/repositories/test_unit_of_work.py +++ b/tests/unit/repositories/test_unit_of_work.py @@ -6,6 +6,7 @@ import pytest from litellm.repositories.unit_of_work import ( LinkedSpendResetWrites, + SpendResetWrites, budget_cascade_unit_of_work, enduser_spend_reset_unit_of_work, spend_reset_unit_of_work, @@ -39,6 +40,9 @@ class FakeBatch: self.litellm_projecttable = FakeBatchTable("litellm_projecttable", self.calls) self.litellm_endusertable = FakeBatchTable("litellm_endusertable", self.calls) + def execute_raw(self, query: str, *args: object) -> None: + self.calls.append(("execute_raw", {"query": query}, {"args": args})) + async def commit(self) -> None: self.commit_count += 1 @@ -166,7 +170,7 @@ async def test_every_cascade_dependent_writes_to_its_own_table_on_the_one_batch( async with budget_cascade_unit_of_work(_new_batch) as uow: writes = [getattr(uow, field.name) for field in fields(uow)] for write in writes: - if isinstance(write, LinkedSpendResetWrites): + if isinstance(write, SpendResetWrites): write.queue_spend_zero(where={"budget_id": "budget-1"}) else: write.queue_window_advance(budget_id="budget-1", budget_reset_at=reset_at) @@ -192,3 +196,18 @@ async def test_budget_cascade_raising_inside_block_skips_commit(): await _blow_up_mid_transaction() assert batch.commit_count == 0 + + +async def test_membership_total_spend_repair_is_queued_on_the_cascade_batch_ahead_of_the_reset(): + batch = FakeBatch() + + async with budget_cascade_unit_of_work(lambda: batch) as uow: + uow.team_memberships.queue_total_spend_repair(["budget-1", "budget-2"]) + uow.team_memberships.queue_spend_zero(where={"budget_id": "budget-1"}) + assert batch.commit_count == 0 + + assert batch.commit_count == 1 + repair, reset = batch.calls + assert repair[0] == "execute_raw" + assert repair[2] == {"args": (["budget-1", "budget-2"],)} + assert reset == ("litellm_teammembership.update_many", {"budget_id": "budget-1"}, {"spend": 0}) diff --git a/ui/litellm-dashboard/src/components/team/MyUserTab.test.tsx b/ui/litellm-dashboard/src/components/team/MyUserTab.test.tsx index d9ba3d82230..24ce626ecb7 100644 --- a/ui/litellm-dashboard/src/components/team/MyUserTab.test.tsx +++ b/ui/litellm-dashboard/src/components/team/MyUserTab.test.tsx @@ -45,5 +45,30 @@ describe("MyUserTab", () => { expect(screen.getByText("member@example.com")).toBeInTheDocument(); expect(screen.getByText("model-one")).toBeInTheDocument(); expect(screen.getByText("TPM: 1,000")).toBeInTheDocument(); + expect(screen.getByText("Never resets")).toBeInTheDocument(); + }); + + it("shows the next reset time when the member budget has a reset window", () => { + vi.mocked(useMyTeamMember).mockReturnValue({ + data: { + user_id: "user-1", + user_email: "member@example.com", + team_id: "team-1", + role: "user", + spend: 12.5, + total_spend: 30, + litellm_budget_table: { + max_budget: 100, + budget_duration: "30d", + budget_reset_at: new Date(2026, 6, 15, 12, 0, 0).toISOString(), + }, + }, + isLoading: false, + error: null, + } as ReturnType); + + renderWithProviders(); + + expect(screen.getByText("Resets Jul 15, 2026 12:00")).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/team/MyUserTab.tsx b/ui/litellm-dashboard/src/components/team/MyUserTab.tsx index 9aae6a25dc2..bf1a2c2c242 100644 --- a/ui/litellm-dashboard/src/components/team/MyUserTab.tsx +++ b/ui/litellm-dashboard/src/components/team/MyUserTab.tsx @@ -1,4 +1,4 @@ -import { formatBudgetReset } from "@/utils/budgetUtils"; +import { describeBudgetResetSchedule } from "@/utils/budgetUtils"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { SimpleTooltip } from "@/components/ui/tooltip"; import { Badge } from "@/components/ui/badge"; @@ -67,7 +67,7 @@ export default function MyUserTab({ teamId }: MyUserTabProps) { const totalSpend = data.total_spend ?? 0; const tpmLimit = budgetTable?.tpm_limit ?? null; const rpmLimit = budgetTable?.rpm_limit ?? null; - const budgetReset = formatBudgetReset(budgetTable?.budget_reset_at); + const budgetReset = describeBudgetResetSchedule(budgetTable?.budget_duration, budgetTable?.budget_reset_at); const allowedModels = budgetTable?.allowed_models ?? null; return ( @@ -93,17 +93,16 @@ export default function MyUserTab({ teamId }: MyUserTabProps) {
- {labelWithTooltip( - "Current Cycle Spend (USD)", - "Spend for the current budget cycle. Resets to $0 when the budget window rolls over.", - )} + {labelWithTooltip("Current Cycle Spend (USD)", budgetReset.tooltip)}

${formatNumber(spend, 4)}

of {maxBudget === null ? "Unlimited" : `$${formatNumber(maxBudget, 4)}`}
- {budgetReset &&
Resets {budgetReset}
} +
+ {budgetReset.resetAt ? `Resets ${budgetReset.label}` : budgetReset.label} +
@@ -120,7 +119,10 @@ export default function MyUserTab({ teamId }: MyUserTabProps) { - {labelWithTooltip("Total Spend (USD)", "Cumulative spend across all budget cycles within this team.")} + {labelWithTooltip( + "Total Spend (USD)", + "Cumulative spend across budget cycles within this team. This is a lower bound: lifetime tracking began 2026-04-21, so cycles completed before then are not included. It is never less than Current Cycle Spend.", + )}

${formatNumber(totalSpend, 4)}

 diff --git a/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx b/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx index cd51bcedb7b..38a8e4f97d5 100644 --- a/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx @@ -313,7 +313,47 @@ describe("TeamMembersComponent", () => { />, ); - expect(screen.getByText("Jul 15, 2026")).toBeInTheDocument(); + expect(screen.getByText("Jul 15, 2026 12:00")).toBeInTheDocument(); + }); + + it("shows Never resets for a member whose budget has no reset window", () => { + const teamData = createMockTeamData(); + teamData.team_memberships[0].litellm_budget_table!.budget_reset_at = null; + renderWithProviders( + , + ); + + expect(screen.queryByText("Jul 15, 2026 12:00")).not.toBeInTheDocument(); + expect(screen.getAllByText("Never resets").length).toBeGreaterThan(0); + }); + + it("shows Unknown for a member whose budget has a duration but no scheduled reset yet", () => { + const teamData = createMockTeamData(); + teamData.team_memberships[0].litellm_budget_table!.budget_duration = "30d"; + teamData.team_memberships[0].litellm_budget_table!.budget_reset_at = null; + renderWithProviders( + , + ); + + expect(screen.getByText("Unknown")).toBeInTheDocument(); }); it("shows each member the values from its own membership row", () => { diff --git a/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx b/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx index ddfd251cf82..319f8c973f6 100644 --- a/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx @@ -10,8 +10,9 @@ import MemberTable from "@/components/common_components/MemberTable"; import type { StoredModelMaxBudget } from "@/components/key_team_helpers/modelMaxBudgetPayload"; import { Member } from "@/components/networking"; import { parseErrorMessage } from "@/components/shared/errorUtils"; -import { DateCell, MoneyCell } from "@/components/shared/table_cells"; +import { MoneyCell } from "@/components/shared/table_cells"; import { toast } from "@/lib/toast"; +import { describeBudgetResetSchedule } from "@/utils/budgetUtils"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { isProxyAdminRole, isUserTeamAdminForSingleTeam } from "@/utils/roles"; import { CircleHelp } from "lucide-react"; @@ -152,6 +153,16 @@ export default function TeamMemberTab({ return findMembership(userId)?.litellm_budget_table?.budget_reset_at ?? null; }; + const renderBudgetReset = (userId: string | null) => { + const budget = findMembership(userId)?.litellm_budget_table; + const schedule = describeBudgetResetSchedule(budget?.budget_duration ?? null, budget?.budget_reset_at ?? null); + return ( + + {schedule.label} + + ); + }; + const extraColumns: NonNullable["extraColumns"]> = [ { title: ( @@ -190,7 +201,7 @@ export default function TeamMemberTab({ title: ( Current Cycle Spend (USD) - + @@ -204,7 +215,7 @@ export default function TeamMemberTab({ title: ( Total Spend (USD) - + @@ -255,7 +266,7 @@ export default function TeamMemberTab({ title: "Budget Reset", key: "budget_reset", sortValue: (record: Member) => getUserBudgetReset(record.user_id), - render: (record: Member) => , + render: (record: Member) => renderBudgetReset(record.user_id), }, { title: ( diff --git a/ui/litellm-dashboard/src/utils/budgetUtils.test.ts b/ui/litellm-dashboard/src/utils/budgetUtils.test.ts index a50fecabe83..a3b5f4da00a 100644 --- a/ui/litellm-dashboard/src/utils/budgetUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/budgetUtils.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it } from "vitest"; -import { formatBudgetReset, formatBudgetResetUtc } from "./budgetUtils"; +import { describeBudgetResetSchedule, formatBudgetReset, formatBudgetResetUtc } from "./budgetUtils"; const originalTimezone: string | undefined = process.env.TZ; @@ -21,3 +21,26 @@ describe("formatBudgetResetUtc", () => { expect(formatBudgetReset(resetAt)).toBe("Oct 19, 2026"); }); }); + +describe("describeBudgetResetSchedule", () => { + it("reports Never resets when the budget has no duration and no scheduled reset", () => { + const schedule = describeBudgetResetSchedule(null, null); + expect(schedule.label).toBe("Never resets"); + expect(schedule.resetAt).toBeNull(); + }); + + it("reports Unknown when a duration is set but no reset has been scheduled yet", () => { + const schedule = describeBudgetResetSchedule("30d", null); + expect(schedule.label).toBe("Unknown"); + expect(schedule.tooltip).toContain("every 30d"); + }); + + it("shows the next reset with its time zone and the rollover carryover note", () => { + const resetAt = new Date(2026, 6, 15, 12, 0, 0); + const schedule = describeBudgetResetSchedule("30d", resetAt.toISOString()); + expect(schedule.label).toBe("Jul 15, 2026 12:00"); + expect(schedule.resetAt).toBe(resetAt.toISOString()); + expect(schedule.tooltip).toContain(Intl.DateTimeFormat().resolvedOptions().timeZone); + expect(schedule.tooltip).toContain("carries into the next cycle"); + }); +}); diff --git a/ui/litellm-dashboard/src/utils/budgetUtils.ts b/ui/litellm-dashboard/src/utils/budgetUtils.ts index dfc57f2fa66..171ce5818f7 100644 --- a/ui/litellm-dashboard/src/utils/budgetUtils.ts +++ b/ui/litellm-dashboard/src/utils/budgetUtils.ts @@ -16,3 +16,41 @@ export function formatBudgetResetUtc(iso: string | null | undefined): string | n if (!resetDate.isValid()) return null; return resetDate.format("MMM D, YYYY"); } + +export interface BudgetResetSchedule { + label: string; + resetAt: string | null; + tooltip: string; +} + +const ROLLOVER_NOTE = "Spend above a rollover cap at reset carries into the next cycle."; + +export function describeBudgetResetSchedule( + budgetDuration: string | null | undefined, + budgetResetAt: string | null | undefined, +): BudgetResetSchedule { + const resetDate = budgetResetAt ? dayjs(budgetResetAt) : null; + if (resetDate?.isValid()) { + const cadence = budgetDuration ? `every ${budgetDuration}` : "on a schedule"; + const timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone; + const label = resetDate.format("MMM D, YYYY HH:mm"); + return { + label, + resetAt: resetDate.toISOString(), + tooltip: `Current Cycle Spend resets ${cadence}. Next reset ${label} (${timeZone}). ${ROLLOVER_NOTE}`, + }; + } + if (!budgetDuration) { + return { + label: "Never resets", + resetAt: null, + tooltip: + "This budget has no reset window, so Current Cycle Spend accumulates for as long as the membership exists.", + }; + } + return { + label: "Unknown", + resetAt: null, + tooltip: `Resets every ${budgetDuration}, but the next reset time has not been scheduled yet. ${ROLLOVER_NOTE}`, + }; +} From 2e52e0fb2a84b03f000b2c6498578036306710bd Mon Sep 17 00:00:00 2001 From: Hazelhof <110658790+Hazelhof@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:02:46 +1100 Subject: [PATCH 043/155] fix(deepseek): pass reasoning_effort through to the API (#38736) * fix(deepseek): pass reasoning_effort through to the API DeepSeekChatConfig.map_openai_params mapped reasoning_effort to a binary thinking {type: enabled/disabled} toggle and discarded the effort value, so DeepSeek always ran default high effort. High-effort reasoning can run away and exhaust max_tokens during reasoning, returning empty content. Preserve the reasoning_effort value into the request body when thinking is enabled so callers can bound effort (e.g. reasoning_effort="low"). Backward-compatible. See https://api-docs.deepseek.com/guides/thinking_mode * test(deepseek): reasoning_effort passthrough coverage * fix(deepseek): exclude reasoning_effort=none sentinel from passthrough "none" is the thinking-OFF sentinel, not a real effort value. Passing it alongside explicit thinking:enabled would send contradictory instructions to DeepSeek. Only forward real effort values (low/medium/high/xhigh/max). Addresses review feedback on the reasoning_effort passthrough change. * refactor(deepseek): name supported effort values and type mapped params * fix(deepseek): keep mapping types local to the adapter * fix(deepseek): remove cast and type mapped return --------- Co-authored-by: moe-berri --- litellm/llms/deepseek/chat/transformation.py | 11 +++- .../chat/test_deepseek_chat_transformation.py | 56 +++++++++++++++++-- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index a9a20101bb1..ecaa0a2ef16 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -17,6 +17,8 @@ from litellm.utils import supports_reasoning, supports_vision from ...openai.chat.gpt_transformation import OpenAIGPTConfig +SUPPORTED_REASONING_EFFORTS: Final = ("minimal", "low", "medium", "high", "xhigh", "max", "ultra") + class DeepSeekChatConfig(OpenAIGPTConfig): def get_supported_openai_params(self, model: str) -> list: @@ -33,7 +35,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig): optional_params: dict, model: str, drop_params: bool, - ) -> dict: + ) -> dict[str, object]: """ Map OpenAI params to DeepSeek params. @@ -60,6 +62,13 @@ class DeepSeekChatConfig(OpenAIGPTConfig): elif reasoning_effort is not None: optional_params["thinking"] = {"type": "disabled" if reasoning_effort == "none" else "enabled"} + if ( + isinstance(reasoning_effort, str) + and reasoning_effort in SUPPORTED_REASONING_EFFORTS + and optional_params["thinking"] == {"type": "enabled"} + ): + return {**optional_params, "reasoning_effort": reasoning_effort} + return optional_params def _fill_reasoning_content(self, messages: list[AllMessageValues]) -> list[AllMessageValues]: diff --git a/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py b/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py index dd99ad8bf91..3fbb12881da 100644 --- a/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py +++ b/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py @@ -511,6 +511,54 @@ class TestDeepSeekVisionMultimodalContent: assert any(block.get("type") == "image_url" for block in content) +@pytest.mark.usefixtures("_deepseek_httpx_transport") +@pytest.mark.parametrize("stream", (False, True)) +@pytest.mark.parametrize("model", ("deepseek-flash", "deepseek-v4-pro", "deepseek-chat", "deepseek-reasoner")) +@pytest.mark.parametrize("effort", ("minimal", "low", "medium", "high", "xhigh", "max", "ultra")) +@pytest.mark.parametrize("thinking", (None, "enabled")) +@pytest.mark.respx(assert_all_called=True) +async def test_deepseek_forwards_native_effort( + respx_mock: respx.MockRouter, model: str, effort: str, thinking: str | None, stream: bool +) -> None: + route: Final = respx_mock.post(DEEPSEEK_CHAT_COMPLETIONS_URL).mock(return_value=_deepseek_response(stream)) + response: Final = await litellm.acompletion( + model=f"deepseek/{model}", + messages=[{"role": "user", "content": "Hello"}], + api_key=DEEPSEEK_API_KEY, + reasoning_effort=effort, + stream=stream, + **({"thinking": {"type": thinking}} if thinking is not None else {}), + ) + if stream: + async for _ in response: + pass + body: Final = json.loads(route.calls[0].request.content) + assert body["reasoning_effort"] == effort + assert body["thinking"] == {"type": "enabled"} + + +@pytest.mark.parametrize( + "thinking,effort,expected_thinking", + ( + (None, None, None), + (None, "none", "disabled"), + ("enabled", "none", "enabled"), + ("disabled", "max", "disabled"), + ("enabled", None, "enabled"), + (None, "default", "enabled"), + (None, "bogus", "enabled"), + (None, {"effort": "high"}, "enabled"), + ), +) +def test_deepseek_preserves_toggle_without_forwarding_unsupported_effort( + thinking: str | None, effort: str | dict[str, str] | None, expected_thinking: str | None +) -> None: + params: Final = {"reasoning_effort": effort, **({"thinking": {"type": thinking}} if thinking is not None else {})} + result: Final = DeepSeekChatConfig().map_openai_params(params, {}, "deepseek-flash", False) + assert "reasoning_effort" not in result + assert result.get("thinking") == ({"type": expected_thinking} if expected_thinking else None) + + class TestDeepSeekThinkingParams: """Test thinking and reasoning_effort parameter handling for DeepSeek.""" @@ -735,9 +783,7 @@ async def test_deepseek_provider_async_completion(stream): messages = [{"role": "user", "content": "Hello, world!"}] # Mock AsyncHTTPHandler.post method for async test - with patch( - "litellm.llms.custom_httpx.llm_http_handler.AsyncHTTPHandler.post" - ) as mock_post: + with patch("litellm.llms.custom_httpx.llm_http_handler.AsyncHTTPHandler.post") as mock_post: mock_response_data = litellm.ModelResponse( choices=[ litellm.Choices( @@ -775,9 +821,7 @@ async def test_deepseek_provider_async_completion(stream): # Check request body request_body = json.loads(call_args.kwargs["data"]) assert call_args.kwargs["url"] == "https://api.deepseek.com/beta/chat/completions" - assert ( - request_body["model"] == "deepseek-reasoner" - ) # Model name should be stripped of provider prefix + assert request_body["model"] == "deepseek-reasoner" # Model name should be stripped of provider prefix assert request_body["messages"] == messages assert request_body["stream"] == stream From de5dc5b238b6e972869d7dedc6ba3c4fb8d931fe Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sat, 24 Oct 2026 18:20:36 -0700 Subject: [PATCH 044/155] feat(mistral): add zai-glm-5-3-flash to the cost map (#48270) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 23 +++++++++++++++++++ model_prices_and_context_window.json | 23 +++++++++++++++++++ .../test_model_prices_and_context_window.py | 15 ++++-------- 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 98df2fc4204..9a7d88b8eda 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -42726,6 +42726,29 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "mistral/zai-glm-5-3-flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "mistral", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], + "source": "https://docs.mistral.ai/models/zai-glm-5-3-flash", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "mistral/zai-glm-5": { "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.4e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 98df2fc4204..9a7d88b8eda 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -42726,6 +42726,29 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "mistral/zai-glm-5-3-flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "mistral", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], + "source": "https://docs.mistral.ai/models/zai-glm-5-3-flash", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "mistral/zai-glm-5": { "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.4e-06, diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 9b9fec4c9a1..c0abb18ca8c 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -455,30 +455,23 @@ def is_active_priced_mistral_chat_row(name: str, entry: Mapping[str, object]) -> ) -def cache_read_is_tenth_of_input(entry: Mapping[str, object]) -> bool: +def cache_read_is_below_input(entry: Mapping[str, object]) -> bool: cache_read: Final = entry.get("cache_read_input_token_cost") input_cost: Final = entry.get("input_cost_per_token") - return ( - isinstance(cache_read, float) - and isinstance(input_cost, (int, float)) - and 0 < cache_read < input_cost - and cache_read == pytest.approx(input_cost / 10) - ) + return isinstance(cache_read, float) and isinstance(input_cost, (int, float)) and 0 < cache_read < input_cost @pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) def test_active_mistral_chat_rows_price_cache_reads_below_input(path: Path): """A Mistral chat row without a cache-read rate bills cached prompt tokens at zero, so every - active priced row must carry one, and it must be cheaper than a fresh input token. Mistral - bills cached tokens at 10% of the input price for every model (docs.mistral.ai/studio/ - conversations/advanced/prompt-caching, read 2026-09-18), so the ratio is checked as well.""" + active priced row must carry one, and it must be cheaper than a fresh input token.""" rows: Mapping[str, object] = json.loads(path.read_text()) drifted: Final = [ f"{name}: cache_read={entry.get('cache_read_input_token_cost')} input={entry.get('input_cost_per_token')}" for name, entry in rows.items() if isinstance(entry, dict) and is_active_priced_mistral_chat_row(name, entry) - and not cache_read_is_tenth_of_input(entry) + and not cache_read_is_below_input(entry) ] assert drifted == [] From d3b70c42ed1599727669ea7883c3971008a9113d Mon Sep 17 00:00:00 2001 From: Jeevan Babu Gotru <114808955+jeevan1098@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:15:46 -0500 Subject: [PATCH 045/155] fix(proxy): keep JSON integers outside the 64-bit range exact when parsing request bodies (#48213) Preserve oversized JSON integers in proxy requests and model rewrites Parse large bodies in a worker thread to avoid blocking request handling Fixes #47947 Co-authored-by: Jeevan Babu Gotru <114808955+jeevan1098@users.noreply.github.com> Co-authored-by: moe-berri --- .../proxy/common_utils/http_parsing_utils.py | 67 ++++++++++++-- .../common_utils/test_http_parsing_utils.py | 92 +++++++++++++++++++ 2 files changed, 153 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index e34d03a04c3..9a3aed0b67b 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -1,3 +1,4 @@ +import asyncio import json import re from collections.abc import Collection, Mapping @@ -161,10 +162,54 @@ def coerce_numeric_form_fields( } -def _parse_binary_body(body: bytes) -> dict: +_ORJSON_INT_MIN: Final = -(2**63) +_ORJSON_INT_MAX: Final = 2**64 - 1 +_JSON_PARSE_THREAD_THRESHOLD: Final = 64 * 1024 +_JSON_LONG_DIGITS: Final = re.compile(rb"[0-9]{19}") +_JSON_REQUEST_BODY: Final = TypeAdapter(Mapping[str, object]) +_JSON_BODY_BYTES: Final = TypeAdapter(bytes) +# Consume unterminated strings through EOF to avoid rescanning every escaped quote in malformed bodies. +_JSON_STRING_OR_NUMBER: Final = re.compile( + rb'"[^"\\]*(?:\\.[^"\\]*)*(?:"|\\?$)|(-?[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?)', re.DOTALL +) + + +def _has_int_outside_orjson_range(body: bytes) -> bool: + if _JSON_LONG_DIGITS.search(body) is None: + return False + for token in _JSON_STRING_OR_NUMBER.finditer(body): + if ( + (number := token.group(1)) is None + or len(number) < 19 + or any(marker in number for marker in (b".", b"e", b"E")) + ): + continue + if len(number) > 20 or not _ORJSON_INT_MIN <= int(number) <= _ORJSON_INT_MAX: + return True + return False + + +def _loads_json_body(body: bytes) -> Mapping[str, object]: + if not _has_int_outside_orjson_range(body): + return _JSON_REQUEST_BODY.validate_python(orjson.loads(body)) + try: + parsed: Final = _JSON_REQUEST_BODY.validate_python(json.loads(body)) + json.dumps(parsed, ensure_ascii=False, allow_nan=False).encode("utf-8") + except ValueError: + return _JSON_REQUEST_BODY.validate_python(orjson.loads(body)) + return parsed + + +async def _loads_json_body_async(body: bytes) -> Mapping[str, object]: + if len(body) > _JSON_PARSE_THREAD_THRESHOLD: + return await asyncio.to_thread(_loads_json_body, body) + return _loads_json_body(body) + + +async def _parse_binary_body(body: bytes) -> dict: """JSON sent under a binary content type still parses; real binary (protobuf) carries no params -> {}.""" try: - parsed: Final = _DECODED_JSON.validate_python(orjson.loads(body)) + parsed: Final = _DECODED_JSON.validate_python(await _loads_json_body_async(body)) if isinstance(parsed, dict): return parsed except orjson.JSONDecodeError: @@ -236,7 +281,7 @@ async def read_request_body(request: Request | None) -> dict: if _normalize_media_type(content_type) in _BINARY_CONTENT_TYPES: binary_body: Final = await request.body() _mark_body_received(len(binary_body)) - parsed_body = _parse_binary_body(binary_body) + parsed_body = await _parse_binary_body(binary_body) elif _is_form_content_type(content_type): try: form_data: Final = await request.form() @@ -272,7 +317,7 @@ async def read_request_body(request: Request | None) -> dict: parsed_body = {} else: try: - parsed_body = orjson.loads(body) + parsed_body = dict(await _loads_json_body_async(body)) except orjson.JSONDecodeError as e: # The surrogate-repair fallback below runs two full-body re.sub # passes, which block the event loop on multi-MB malformed bodies. @@ -403,6 +448,14 @@ def detach_request_parsed_body( return client_body +def _dumps_json_body(body: Mapping[str, object]) -> bytes: + try: + encoded: Final = _JSON_BODY_BYTES.validate_python(orjson.dumps(body)) + except TypeError: + return json.dumps(body, separators=(",", ":")).encode("utf-8") + return encoded + + def rewrite_request_model( request_data: dict[str, object], # mutable-ok: the request body is rewritten in place for every downstream reader request: Request | None, @@ -415,10 +468,12 @@ def rewrite_request_model( if request is None: return cached_body: Final = _safe_get_request_parsed_body(request=request) - body: Final = {**cached_body, "model": model} if cached_body is not None else request_data + body: Final = ( + {**_JSON_REQUEST_BODY.validate_python(cached_body), "model": model} if cached_body is not None else request_data + ) safe_set_request_parsed_body(request=request, parsed_body=body) request._json = body # pyright: ignore[reportPrivateUsage] # Starlette JSON cache - request._body = orjson.dumps(body) # pyright: ignore[reportPrivateUsage] # Starlette body cache + request._body = _dumps_json_body(body) # pyright: ignore[reportPrivateUsage] # Starlette body cache def safe_get_request_headers(request: Request | None) -> dict: diff --git a/tests/unit/proxy/common_utils/test_http_parsing_utils.py b/tests/unit/proxy/common_utils/test_http_parsing_utils.py index cd5820fa2fd..3ec95dfdcdc 100644 --- a/tests/unit/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/unit/proxy/common_utils/test_http_parsing_utils.py @@ -7,6 +7,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import orjson import pytest +from pydantic import TypeAdapter from fastapi.testclient import TestClient from starlette.datastructures import FormData from starlette.requests import Request @@ -104,6 +105,97 @@ async def test_read_request_body_marks_body_received_for_binary_and_form_bodies( ] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body", + [ + b'{"maximum": 12345678901234567890123}', + b'{"minimum": -9223372036854775809}', + b'{"maximum": 18446744073709551616}', + b'{"maximum": 18446744073709551615}', + b'{"minimum": -9223372036854775808}', + b'{"const": 9007199254740993}', + b'{"nested": {"list": [12345678901234567890123, {"deep": -9223372036854775809}]}}', + b'{"maximum": ' + b"9" * 300 + b"}", + b'{"maximum": ' + b"9" * 400 + b"}", + rb'{"text": "escaped \"quote\" and backslash \\", "maximum": 18446744073709551616}', + ], +) +@pytest.mark.parametrize("content_type", ["application/json", "application/x-protobuf"]) +async def test_read_request_body_keeps_large_integers_exact(body: bytes, content_type: str) -> None: + request: Final = _starlette_request(body, content_type) + parsed: Final = await read_request_body(request) + + assert parsed == json.loads(body) + assert json.dumps(parsed).encode() == json.dumps(json.loads(body)).encode() + assert await read_request_body(request) == parsed + + +@pytest.mark.asyncio +@pytest.mark.parametrize("padding", [0, 100_000]) +@pytest.mark.parametrize("content_type", ["application/json", "application/x-protobuf"]) +async def test_model_rewrite_keeps_large_integers_in_every_body_reader(padding: int, content_type: str) -> None: + body: Final = { + "model": "integer-alias", + "input": [12345678901234567890123, -9223372036854775809, {"nested": 10**400}], + "text": "escaped \"quote\" and backslash \\" + "x" * padding, + } + request: Final = _starlette_request(json.dumps(body).encode(), content_type) + parsed: Final = dict(TypeAdapter(Mapping[str, object]).validate_python(await read_request_body(request))) + + http_parsing_utils.rewrite_request_model(parsed, request, "integer-target") + + expected: Final = {**body, "model": "integer-target"} + assert parsed == expected + assert await read_request_body(request) == expected + assert await request.json() == expected + assert json.loads(await request.body()) == expected + + +@pytest.mark.asyncio +async def test_read_request_body_does_not_turn_floats_or_digit_strings_into_integers() -> None: + body: Final = ( + b'{"model": "glm-5.3", "content": "id 12345678901234567890123", ' + b'"score": 1.234567890123456789, "rate": 12345678901234567890e0, "count": 42}' + ) + parsed: Final = await read_request_body(_starlette_request(body, "application/json")) + + assert parsed == { + "model": "glm-5.3", + "content": "id 12345678901234567890123", + "score": 1.234567890123456789, + "rate": 12345678901234567890e0, + "count": 42, + } + assert isinstance(parsed["score"], float) + assert isinstance(parsed["rate"], float) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body", + [ + b'{"maximum": 18446744073709551616,}', + b'{"maximum": 18446744073709551616, "text": "\\ud800"}', + b'{"maximum": 18446744073709551616, "text": "\\udc00"}', + b'{"text": "' + b'\\"' * 100 + b"18446744073709551616\\", + ], +) +async def test_large_integer_does_not_bypass_invalid_json_rejection(body: bytes) -> None: + with pytest.raises(ProxyException) as error: + await read_request_body(_starlette_request(body, "application/json")) + + assert error.value.code == "400" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("number", [b"NaN", b"Infinity", b"-Infinity", b"1e400"]) +async def test_large_integer_does_not_make_nonfinite_binary_json_valid(number: bytes) -> None: + body: Final = b'{"maximum": 18446744073709551616, "other": ' + number + b"}" + + assert await read_request_body(_starlette_request(body, "application/x-protobuf")) == {} + + @pytest.mark.asyncio async def test_read_raw_json_body_returns_the_bytes_the_parsed_body_came_from(): body = b'{"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "hi"}]}' From 72f1ac4a2795e160d1044fb963041aec3111ff70 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 24 Oct 2026 20:57:49 -0700 Subject: [PATCH 046/155] revert(proxy): revert large-integer request parsing (#48289) Reverts d3b70c42ed1599727669ea7883c3971008a9113d because the JSON scanner causes excessive memory use and blocks concurrent requests on escape-heavy payloads --- .../proxy/common_utils/http_parsing_utils.py | 64 ++----------- .../common_utils/test_http_parsing_utils.py | 92 ------------------- 2 files changed, 6 insertions(+), 150 deletions(-) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 9a3aed0b67b..a38769ac41b 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -1,4 +1,3 @@ -import asyncio import json import re from collections.abc import Collection, Mapping @@ -32,6 +31,7 @@ _BINARY_CONTENT_TYPES: Final[frozenset[str]] = frozenset({"application/x-protobu _ANNOTATION_QUALIFIERS: Final[frozenset[object]] = frozenset({Annotated, NotRequired, ReadOnly, Required}) _DECODED_JSON: Final = TypeAdapter(object) +_JSON_REQUEST_BODY: Final = TypeAdapter(Mapping[str, object]) def resolve_inference_model( @@ -162,54 +162,10 @@ def coerce_numeric_form_fields( } -_ORJSON_INT_MIN: Final = -(2**63) -_ORJSON_INT_MAX: Final = 2**64 - 1 -_JSON_PARSE_THREAD_THRESHOLD: Final = 64 * 1024 -_JSON_LONG_DIGITS: Final = re.compile(rb"[0-9]{19}") -_JSON_REQUEST_BODY: Final = TypeAdapter(Mapping[str, object]) -_JSON_BODY_BYTES: Final = TypeAdapter(bytes) -# Consume unterminated strings through EOF to avoid rescanning every escaped quote in malformed bodies. -_JSON_STRING_OR_NUMBER: Final = re.compile( - rb'"[^"\\]*(?:\\.[^"\\]*)*(?:"|\\?$)|(-?[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?)', re.DOTALL -) - - -def _has_int_outside_orjson_range(body: bytes) -> bool: - if _JSON_LONG_DIGITS.search(body) is None: - return False - for token in _JSON_STRING_OR_NUMBER.finditer(body): - if ( - (number := token.group(1)) is None - or len(number) < 19 - or any(marker in number for marker in (b".", b"e", b"E")) - ): - continue - if len(number) > 20 or not _ORJSON_INT_MIN <= int(number) <= _ORJSON_INT_MAX: - return True - return False - - -def _loads_json_body(body: bytes) -> Mapping[str, object]: - if not _has_int_outside_orjson_range(body): - return _JSON_REQUEST_BODY.validate_python(orjson.loads(body)) - try: - parsed: Final = _JSON_REQUEST_BODY.validate_python(json.loads(body)) - json.dumps(parsed, ensure_ascii=False, allow_nan=False).encode("utf-8") - except ValueError: - return _JSON_REQUEST_BODY.validate_python(orjson.loads(body)) - return parsed - - -async def _loads_json_body_async(body: bytes) -> Mapping[str, object]: - if len(body) > _JSON_PARSE_THREAD_THRESHOLD: - return await asyncio.to_thread(_loads_json_body, body) - return _loads_json_body(body) - - -async def _parse_binary_body(body: bytes) -> dict: +def _parse_binary_body(body: bytes) -> dict: """JSON sent under a binary content type still parses; real binary (protobuf) carries no params -> {}.""" try: - parsed: Final = _DECODED_JSON.validate_python(await _loads_json_body_async(body)) + parsed: Final = _DECODED_JSON.validate_python(orjson.loads(body)) if isinstance(parsed, dict): return parsed except orjson.JSONDecodeError: @@ -281,7 +237,7 @@ async def read_request_body(request: Request | None) -> dict: if _normalize_media_type(content_type) in _BINARY_CONTENT_TYPES: binary_body: Final = await request.body() _mark_body_received(len(binary_body)) - parsed_body = await _parse_binary_body(binary_body) + parsed_body = _parse_binary_body(binary_body) elif _is_form_content_type(content_type): try: form_data: Final = await request.form() @@ -317,7 +273,7 @@ async def read_request_body(request: Request | None) -> dict: parsed_body = {} else: try: - parsed_body = dict(await _loads_json_body_async(body)) + parsed_body = dict(_JSON_REQUEST_BODY.validate_python(orjson.loads(body))) except orjson.JSONDecodeError as e: # The surrogate-repair fallback below runs two full-body re.sub # passes, which block the event loop on multi-MB malformed bodies. @@ -448,14 +404,6 @@ def detach_request_parsed_body( return client_body -def _dumps_json_body(body: Mapping[str, object]) -> bytes: - try: - encoded: Final = _JSON_BODY_BYTES.validate_python(orjson.dumps(body)) - except TypeError: - return json.dumps(body, separators=(",", ":")).encode("utf-8") - return encoded - - def rewrite_request_model( request_data: dict[str, object], # mutable-ok: the request body is rewritten in place for every downstream reader request: Request | None, @@ -473,7 +421,7 @@ def rewrite_request_model( ) safe_set_request_parsed_body(request=request, parsed_body=body) request._json = body # pyright: ignore[reportPrivateUsage] # Starlette JSON cache - request._body = _dumps_json_body(body) # pyright: ignore[reportPrivateUsage] # Starlette body cache + request._body = orjson.dumps(body) # pyright: ignore[reportPrivateUsage] # Starlette body cache def safe_get_request_headers(request: Request | None) -> dict: diff --git a/tests/unit/proxy/common_utils/test_http_parsing_utils.py b/tests/unit/proxy/common_utils/test_http_parsing_utils.py index 3ec95dfdcdc..cd5820fa2fd 100644 --- a/tests/unit/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/unit/proxy/common_utils/test_http_parsing_utils.py @@ -7,7 +7,6 @@ from unittest.mock import AsyncMock, MagicMock, patch import orjson import pytest -from pydantic import TypeAdapter from fastapi.testclient import TestClient from starlette.datastructures import FormData from starlette.requests import Request @@ -105,97 +104,6 @@ async def test_read_request_body_marks_body_received_for_binary_and_form_bodies( ] -@pytest.mark.asyncio -@pytest.mark.parametrize( - "body", - [ - b'{"maximum": 12345678901234567890123}', - b'{"minimum": -9223372036854775809}', - b'{"maximum": 18446744073709551616}', - b'{"maximum": 18446744073709551615}', - b'{"minimum": -9223372036854775808}', - b'{"const": 9007199254740993}', - b'{"nested": {"list": [12345678901234567890123, {"deep": -9223372036854775809}]}}', - b'{"maximum": ' + b"9" * 300 + b"}", - b'{"maximum": ' + b"9" * 400 + b"}", - rb'{"text": "escaped \"quote\" and backslash \\", "maximum": 18446744073709551616}', - ], -) -@pytest.mark.parametrize("content_type", ["application/json", "application/x-protobuf"]) -async def test_read_request_body_keeps_large_integers_exact(body: bytes, content_type: str) -> None: - request: Final = _starlette_request(body, content_type) - parsed: Final = await read_request_body(request) - - assert parsed == json.loads(body) - assert json.dumps(parsed).encode() == json.dumps(json.loads(body)).encode() - assert await read_request_body(request) == parsed - - -@pytest.mark.asyncio -@pytest.mark.parametrize("padding", [0, 100_000]) -@pytest.mark.parametrize("content_type", ["application/json", "application/x-protobuf"]) -async def test_model_rewrite_keeps_large_integers_in_every_body_reader(padding: int, content_type: str) -> None: - body: Final = { - "model": "integer-alias", - "input": [12345678901234567890123, -9223372036854775809, {"nested": 10**400}], - "text": "escaped \"quote\" and backslash \\" + "x" * padding, - } - request: Final = _starlette_request(json.dumps(body).encode(), content_type) - parsed: Final = dict(TypeAdapter(Mapping[str, object]).validate_python(await read_request_body(request))) - - http_parsing_utils.rewrite_request_model(parsed, request, "integer-target") - - expected: Final = {**body, "model": "integer-target"} - assert parsed == expected - assert await read_request_body(request) == expected - assert await request.json() == expected - assert json.loads(await request.body()) == expected - - -@pytest.mark.asyncio -async def test_read_request_body_does_not_turn_floats_or_digit_strings_into_integers() -> None: - body: Final = ( - b'{"model": "glm-5.3", "content": "id 12345678901234567890123", ' - b'"score": 1.234567890123456789, "rate": 12345678901234567890e0, "count": 42}' - ) - parsed: Final = await read_request_body(_starlette_request(body, "application/json")) - - assert parsed == { - "model": "glm-5.3", - "content": "id 12345678901234567890123", - "score": 1.234567890123456789, - "rate": 12345678901234567890e0, - "count": 42, - } - assert isinstance(parsed["score"], float) - assert isinstance(parsed["rate"], float) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "body", - [ - b'{"maximum": 18446744073709551616,}', - b'{"maximum": 18446744073709551616, "text": "\\ud800"}', - b'{"maximum": 18446744073709551616, "text": "\\udc00"}', - b'{"text": "' + b'\\"' * 100 + b"18446744073709551616\\", - ], -) -async def test_large_integer_does_not_bypass_invalid_json_rejection(body: bytes) -> None: - with pytest.raises(ProxyException) as error: - await read_request_body(_starlette_request(body, "application/json")) - - assert error.value.code == "400" - - -@pytest.mark.asyncio -@pytest.mark.parametrize("number", [b"NaN", b"Infinity", b"-Infinity", b"1e400"]) -async def test_large_integer_does_not_make_nonfinite_binary_json_valid(number: bytes) -> None: - body: Final = b'{"maximum": 18446744073709551616, "other": ' + number + b"}" - - assert await read_request_body(_starlette_request(body, "application/x-protobuf")) == {} - - @pytest.mark.asyncio async def test_read_raw_json_body_returns_the_bytes_the_parsed_body_came_from(): body = b'{"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "hi"}]}' From dddbaa803d6e9fcc862ea7a5412f771cb1ae040e Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 24 Oct 2026 22:33:45 -0700 Subject: [PATCH 047/155] build(docker): refresh base images and ship only runtime files in component images (#48279) * build(docker): move to newer wolfi-base and nginx base images Bump the pinned wolfi-base digest in every Wolfi Dockerfile to 360896ef, which ships brotli 1.2.1-r1 and ca-certificates-bundle 20260909-r3. The previous digest already carried libbrotli 1.2.0-r6 as an OpenSSL dependency, and `apk add --no-cache` never upgrades a package the base already has Move the UI image to nginx 1.31.8-alpine3.24, whose CA bundle is the 20261020 Mozilla set, and upgrade brotli-libs in its runtime stage since no nginx digest ships 1.2.1 yet * build(docker): copy only runtime files into the gateway, backend and migrations images The component runtime stages copied the builder's whole /app, which is the entire repository plus build output: six lockfiles, helm, terraform, docs, .env.example and the Rust build tree. Measured at runtime with an audit hook on a booted container, the gateway reads only gateway/, litellm/ and enterprise/ from /app, the backend only backend/, litellm/ and enterprise/, and the migrations job only litellm/ and run.py. The pyproject.toml read is prisma-python's optional [tool.prisma] lookup, which ours does not define Copy just those, plus .venv, the entrypoint script and schema.prisma for the gateway and backend. PYTHONPATH=/app and the in-Dockerfile asserts are unchanged --- Dockerfile | 4 ++-- backend/Dockerfile | 11 ++++++++--- docker/Dockerfile.database | 4 ++-- docker/Dockerfile.non_root | 4 ++-- gateway/Dockerfile | 11 ++++++++--- migrations/Dockerfile | 8 +++++--- ui/Dockerfile | 4 +++- 7 files changed, 30 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 713d7f8f5ab..4cef63c8d71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 diff --git a/backend/Dockerfile b/backend/Dockerfile index 46d65c432bb..1c22f914100 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin @@ -124,7 +124,12 @@ ENV HOME=/home/nonroot \ PYTHONUNBUFFERED=1 \ PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries -COPY --from=builder --chown=nonroot:nonroot /app /app +COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv +COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm +COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise +COPY --from=builder --chown=nonroot:nonroot /app/backend /app/backend +COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh +COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma COPY --from=builder /opt/prisma /opt/prisma RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 69d9c26b2a8..20f810686a3 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 54bf8c807aa..90bcc523d9e 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -1,8 +1,8 @@ # syntax=docker/dockerfile:1.7 # Base images -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 390a5f5666f..69068cfc8a5 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) ARG PGBOUNCER_VERSION=1.26.0 @@ -140,7 +140,12 @@ ENV HOME=/home/nonroot \ PYTHONUNBUFFERED=1 \ PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries -COPY --from=builder --chown=nonroot:nonroot /app /app +COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv +COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm +COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise +COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway +COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh +COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma COPY --from=builder /opt/prisma /opt/prisma COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer diff --git a/migrations/Dockerfile b/migrations/Dockerfile index 84ea67fcc5f..2b1fd029143 100644 --- a/migrations/Dockerfile +++ b/migrations/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin @@ -137,7 +137,9 @@ ENV HOME=/home/nonroot \ PRISMA_CLI_QUERY_ENGINE_TYPE=binary \ PRISMA_OFFLINE_MODE=true -COPY --from=builder --chown=nonroot:nonroot /app /app +COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv +COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm +COPY --from=builder --chown=nonroot:nonroot /app/run.py /app/run.py COPY --from=builder /opt/prisma /opt/prisma RUN chmod -R a+rX /opt/prisma && \ diff --git a/ui/Dockerfile b/ui/Dockerfile index 4b6b0c1581a..d78a155b874 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -3,7 +3,7 @@ # UI container — Next.js static export served by nginx. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 -ARG NGINX_VERSION=1.31.5-alpine3.24@sha256:34f40471dea485273c5e2a04dd5e97a682332ceb4a9adecd67de450dcb2fb390 +ARG NGINX_VERSION=1.31.8-alpine3.24@sha256:1ef7ca94d7879df82421428ce13bd1eaa412404953859cf7a5b6512c25e67a60 # ---------- builder ---------- FROM ${UI_BUILD_IMAGE} AS builder @@ -27,6 +27,8 @@ RUN npm run build # ---------- runtime ---------- FROM nginx:${NGINX_VERSION} AS runtime +RUN apk upgrade --no-cache brotli-libs + # Drop the upstream default :80 server; we own the config. RUN rm -f /etc/nginx/conf.d/default.conf From ebf829c2faf672caa01cf1019c3920c8d3704011 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 26 Oct 2026 09:00:11 -0700 Subject: [PATCH 048/155] fix: support Python 3.15 (#48359) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test-mcp-dependency-resolution.yml | 4 +- .github/workflows/test-merge-smoke.yml | 8 +- litellm/_uuid.py | 8 +- .../complexity_router/config.py | 13 +- packaging/litellm-core/pyproject.toml | 10 +- pyproject.toml | 23 +- .../router_strategy/test_complexity_router.py | 5 + tests/unit/test_uuid_helper.py | 28 +- uv.lock | 1627 +++++++++++++---- 9 files changed, 1292 insertions(+), 434 deletions(-) diff --git a/.github/workflows/test-mcp-dependency-resolution.yml b/.github/workflows/test-mcp-dependency-resolution.yml index 3f0d1260e49..20efb351985 100644 --- a/.github/workflows/test-mcp-dependency-resolution.yml +++ b/.github/workflows/test-mcp-dependency-resolution.yml @@ -45,7 +45,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 @@ -132,7 +132,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: diff --git a/.github/workflows/test-merge-smoke.yml b/.github/workflows/test-merge-smoke.yml index 0d1cc6596de..f3f09523491 100644 --- a/.github/workflows/test-merge-smoke.yml +++ b/.github/workflows/test-merge-smoke.yml @@ -33,7 +33,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] env: LITELLM_LOCAL_MODEL_COST_MAP: "True" steps: @@ -76,6 +76,12 @@ jobs: - name: Import and CLI checks run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py cli + - name: Routing and UUID compatibility + run: >- + $IN_NETNS .venv/bin/python -m pytest + tests/unit/router_strategy/test_complexity_router.py::TestCustomDimensions + tests/unit/test_uuid_helper.py -q + - name: Proxy startup check run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py proxy-startup --diagnostics-dir "${RUNNER_TEMP}/smoke-diagnostics" diff --git a/litellm/_uuid.py b/litellm/_uuid.py index e9578b7287f..55df233b553 100644 --- a/litellm/_uuid.py +++ b/litellm/_uuid.py @@ -1,10 +1,14 @@ """ Internal unified UUID helper. -Always uses fastuuid for performance. +Uses fastuuid where supported and the standard library on Python 3.15+. """ -import fastuuid as _uuid +import sys +import uuid as _uuid + +if sys.version_info < (3, 15): + import fastuuid as _uuid # Expose a module-like alias so callers can use: uuid.uuid4() uuid = _uuid diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 4a2c56daf00..ec960a19f79 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -7,12 +7,12 @@ All values are configurable via proxy config.yaml. import math import re -import warnings +import sys from collections.abc import Iterable, Mapping from dataclasses import dataclass from enum import Enum from types import MappingProxyType -from typing import Annotated, Final, Literal, NamedTuple +from typing import TYPE_CHECKING, Annotated, Final, Literal, NamedTuple from pydantic import ( ConfigDict, @@ -27,8 +27,13 @@ from pydantic import ( from litellm.types.llms.base import LiteLLMBaseModel -with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) +if TYPE_CHECKING: + import sre_constants + import sre_parse +elif sys.version_info >= (3, 11): + from re import _constants as sre_constants + from re import _parser as sre_parse +else: import sre_constants import sre_parse diff --git a/packaging/litellm-core/pyproject.toml b/packaging/litellm-core/pyproject.toml index 890fa0fd773..27681d9116c 100644 --- a/packaging/litellm-core/pyproject.toml +++ b/packaging/litellm-core/pyproject.toml @@ -3,14 +3,14 @@ name = "litellm-core" version = "1.105.0" description = "Library to easily interface with LLM API providers" readme = "README.md" -requires-python = ">=3.10, <3.15" +requires-python = ">=3.10, <3.16" license = "MIT" license-files = ["LICENSE"] authors = [ { name = "BerriAI" }, ] dependencies = [ - "fastuuid>=0.14.0,<1.0", + "fastuuid>=0.14.0,<1.0; python_version < '3.15'", "filelock>=3.16.1,<4.0", "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<4.0.0", @@ -20,14 +20,16 @@ dependencies = [ "packaging>=24.0", "importlib-metadata>=8.0.0,<9.0", "tiktoken>=0.8.0,<1.0; python_version < '3.14'", - "tiktoken>=0.12.0,<1.0; python_version >= '3.14'", + "tiktoken>=0.12.0,<1.0; python_version == '3.14'", + "tiktoken>=0.14.0,<1.0; python_version >= '3.15'", "click>=8.0.0,<9.0", "jinja2>=3.1.6,<4.0", "aiohttp>=3.14.2,<4.0", "anyio>=4.14.0,<5.0", "async-timeout>=4.0.3,<6.0; python_version < '3.11'", "pydantic>=2.11.0,<3.0.0; python_version < '3.14'", - "pydantic>=2.12.0,<3.0.0; python_version >= '3.14'", + "pydantic>=2.12.0,<3.0.0; python_version == '3.14'", + "pydantic>=2.14.0,<3.0.0; python_version >= '3.15'", "pydantic-settings>=2.14.1,<3.0", "jsonschema>=4.0.0,<5.0", "certifi>=2026.10.19", diff --git a/pyproject.toml b/pyproject.toml index 817c50df90b..b15eb042959 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "litellm" version = "1.108.0" description = "Library to easily interface with LLM API providers" readme = "README.md" -requires-python = ">=3.10, <3.15" +requires-python = ">=3.10, <3.16" license = "MIT" license-files = ["LICENSE"] authors = [ @@ -14,7 +14,7 @@ dependencies = [ # deps. Reproducibility for our Docker/CI comes from `uv.lock`. # When changing a floor, verify it installs + imports on every supported # Python with: `uv pip install --resolution=lowest-direct .` - "fastuuid>=0.14.0,<1.0", + "fastuuid>=0.14.0,<1.0; python_version < '3.15'", "filelock>=3.16.1,<4.0", "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<4.0.0", @@ -23,7 +23,8 @@ dependencies = [ "packaging>=24.0", "importlib-metadata>=8.0.0,<9.0", "tiktoken>=0.8.0,<1.0; python_version < '3.14'", - "tiktoken>=0.12.0,<1.0; python_version >= '3.14'", + "tiktoken>=0.12.0,<1.0; python_version == '3.14'", + "tiktoken>=0.14.0,<1.0; python_version >= '3.15'", "tokenizers>=0.21.0,<1.0", "huggingface-hub>=0.34.0,<2.0", "click>=8.0.0,<9.0", @@ -32,7 +33,8 @@ dependencies = [ "anyio>=4.14.0,<5.0", "async-timeout>=4.0.3,<6.0; python_version < '3.11'", "pydantic>=2.11.0,<3.0.0; python_version < '3.14'", - "pydantic>=2.12.0,<3.0.0; python_version >= '3.14'", + "pydantic>=2.12.0,<3.0.0; python_version == '3.14'", + "pydantic>=2.14.0,<3.0.0; python_version >= '3.15'", "pydantic-settings>=2.14.1,<3.0", "jsonschema>=4.0.0,<5.0", "boto3>=1.43.117,<2.0", @@ -55,17 +57,20 @@ Documentation = "https://docs.litellm.ai" proxy = [ "gunicorn>=23.0.0,<24.0", "uvicorn>=0.33.0,<1.0", - "granian>=2.7.4,<3.0", + "granian>=2.7.4,<3.0; python_version < '3.15'", + "granian>=2.8.0,<3.0; python_version >= '3.15'", "uvloop>=0.22.1,<1.0; sys_platform != 'win32'", "fastapi>=0.136.3,<1.0", "starlette>=1.0.1,<2.0", "backoff>=2.2.1,<3.0", "pyyaml>=6.0.3,<7.0", "rq>=2.7.0,<3.0", - "orjson>=3.11.6,<4.0", + "orjson>=3.11.6,<4.0; python_version < '3.15'", + "orjson>=3.12.0,<4.0; python_version >= '3.15'", # redis-py's C response parser. It arrives with redis (via rq) either way; naming # it here is what makes redis-py select _HiredisParser instead of the Python one. - "hiredis>=3.0.0,<4.0", + "hiredis>=3.0.0,<4.0; python_version < '3.15'", + "hiredis>=3.5.0,<4.0; python_version >= '3.15'", "apscheduler>=3.11.2,<4.0", "fastapi-sso>=0.19.0,<1.0", "PyJWT>=2.13.0,<3.0", @@ -210,8 +215,8 @@ dev = [ "tomli==2.4.1; python_version < '3.11'", "pytest-mock==3.15.1", "pytest-asyncio==1.3.0", - "psycopg==3.3.3", - "psycopg-binary==3.3.3", + "psycopg==3.3.6", + "psycopg-binary==3.3.6", "pytest-xdist==3.8.0", "requests-mock==1.12.1", "responses==0.26.0", diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index f956bb66e6c..9b4019bead6 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -1065,6 +1065,11 @@ class TestCustomDimensions: "create table widgets (id integer); ALTER TABLE widgets ADD label text;", id="regex", ), + pytest.param( + {"patterns": [r"\b(?Pcreate|alter)\s{1,4}?table\b"]}, + "CREATE TABLE widgets (id integer)", + id="named-group-alternation-lazy-repeat", + ), ], ) def test_custom_dimension_changes_only_matching_requests( diff --git a/tests/unit/test_uuid_helper.py b/tests/unit/test_uuid_helper.py index 589d3f96a17..f636942ba88 100644 --- a/tests/unit/test_uuid_helper.py +++ b/tests/unit/test_uuid_helper.py @@ -1,14 +1,20 @@ -import importlib +from typing import Final +from uuid import NAMESPACE_DNS, UUID, uuid5 as stdlib_uuid5 + +from litellm._uuid import uuid, uuid4 -def test_uses_fastuuid_and_uuid4_works(): - mod = importlib.import_module("litellm._uuid") - fastuuid_mod = importlib.import_module("fastuuid") - assert hasattr(mod, "uuid4") - assert hasattr(mod, "uuid") - assert mod.uuid is fastuuid_mod +def test_uuid4_returns_canonical_random_uuid() -> None: + for value in (uuid4(), uuid.uuid4()): + encoded: Final = str(value) + parsed: Final = UUID(encoded) + assert parsed.version == 4 + assert encoded == str(parsed) + assert value.bytes == parsed.bytes - # Ensure uuid4 returns something that looks like a UUID string - val = str(mod.uuid4()) - assert isinstance(val, str) - assert len(val) >= 8 + +def test_uuid5_accepts_byte_names_for_stable_ids() -> None: + namespace: Final = uuid.UUID(str(NAMESPACE_DNS)) + name: Final = "guardrail-\u00e9" + generated: Final = uuid.uuid5(namespace, name.encode("utf-8")) + assert str(generated) == str(stdlib_uuid5(NAMESPACE_DNS, name)) diff --git a/uv.lock b/uv.lock index 531082730bb..60bcb367e0f 100644 --- a/uv.lock +++ b/uv.lock @@ -1,8 +1,9 @@ version = 1 revision = 5 -requires-python = ">=3.10, <3.15" +requires-python = ">=3.10, <3.16" resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", @@ -58,7 +59,8 @@ dependencies = [ { name = "json-rpc" }, { name = "packaging" }, { name = "protobuf" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c7/7e/8ac10bbf8b15b16574355f39b17dbdf617a282c27b41c7ff2116e30336df/a2a_sdk-1.1.0.tar.gz", hash = "sha256:e8102dad1b36709dbdc3d19319e38e6dfa3b3a79c30416030eb2d482576be204", size = 375726, upload-time = "2026-05-29T09:34:43.015Z" } wheels = [ @@ -107,7 +109,7 @@ dependencies = [ { name = "frozenlist" }, { name = "multidict" }, { name = "propcache" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, { name = "yarl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6c/4c/bdccd81e9ee225b69c60e7766c9a5b05364f118f4d383713b89a682d772d/aiohttp-3.14.5.tar.gz", hash = "sha256:5558a7f5a05af9ecf744af91e5baefc436f93c9333e656c27ec253f9a6bbe178", size = 8078052, upload-time = "2026-10-11T01:05:12.408Z" } @@ -230,6 +232,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/20/eacbecfea3b5c3dcbfc9b023e3a5460f43e5dda16d06a2877ebe7184c3f3/aiohttp-3.14.5-cp314-cp314t-win32.whl", hash = "sha256:33f706574e32c6e694f352a856e05caf18f7f2c871b3e87b41c55ea452b409ab", size = 503973, upload-time = "2026-10-11T01:03:10.481Z" }, { url = "https://files.pythonhosted.org/packages/ff/78/18eec294f6c8c5dc845dcf6d730a0147d8d0f17e86138a7bdb85e43a30fa/aiohttp-3.14.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5ba14a839fbe87cf7c12a6b5661c05f324a296eb8363141edb3944ba63d4c9d3", size = 531190, upload-time = "2026-10-11T01:03:12.716Z" }, { url = "https://files.pythonhosted.org/packages/9d/39/e53f8169acc85271ebd12b5b32ad7f1541b35639ccbe0f49034c64785d10/aiohttp-3.14.5-cp314-cp314t-win_arm64.whl", hash = "sha256:1061b364556e8172e8d46b0b183adeeb73e8c42d30ebc745591e1bd89acad52e", size = 510360, upload-time = "2026-10-11T01:03:15.08Z" }, + { url = "https://files.pythonhosted.org/packages/f3/1c/06d89f58b2db3ee92dd377217659d587e06f973e57bf0a97a0d8a4586c0c/aiohttp-3.14.5-cp315-cp315-android_24_arm64_v8a.whl", hash = "sha256:788ecaa9c10533b786ce5ba70c4f2df78ad41819fd00a6c99d92b66f9a32e1da", size = 540674, upload-time = "2026-10-11T01:03:17.483Z" }, + { url = "https://files.pythonhosted.org/packages/18/39/5e822e038f496f0540ada91e27099d48f9e7f919b6c6deb4cf6db36cc706/aiohttp-3.14.5-cp315-cp315-android_24_x86_64.whl", hash = "sha256:5c76f1802bab718a68ac3cce447160605c734551f95c67ae90fa1132b215cb29", size = 554183, upload-time = "2026-10-11T01:03:19.71Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ff/0cf2619d902b5b160762422a7e5e02091295766a7fe4fcf5b9a655e386c1/aiohttp-3.14.5-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:a6d02b4c38de03d9c7617813433e6a0fb6b522797974177d69d9dad431900833", size = 523160, upload-time = "2026-10-11T01:03:22.193Z" }, + { url = "https://files.pythonhosted.org/packages/86/99/3553abfc53a40849dbaacc3f54c730ec58410809ffa2d05885ae56108f2d/aiohttp-3.14.5-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:20726f9782d5c2744c1c66255842d1d163bb3edcf768b8de25216bf47f7b6ccf", size = 530311, upload-time = "2026-10-11T01:03:24.435Z" }, + { url = "https://files.pythonhosted.org/packages/fe/a4/5d25f73754bc1e8f983ba704e86d641aea290c967f195f2aeebdaed2bd84/aiohttp-3.14.5-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:248d779ad720b49d4fb355720e60c9e5f444f95887bc16974fea48fc56c41789", size = 540274, upload-time = "2026-10-11T01:03:26.698Z" }, + { url = "https://files.pythonhosted.org/packages/29/a4/07eda5db2e3ee017d9590f36c12a94ec6f1f50516e8df78672373dfc7185/aiohttp-3.14.5-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:0a8ea271867e360ac985ae607f4a23ad9a38414b9aca1d49ec98839ae660e49f", size = 819239, upload-time = "2026-10-11T01:03:29.481Z" }, + { url = "https://files.pythonhosted.org/packages/cb/aa/a8723dd987a696dd48d4cf2f0088e589ce77caebff0b96f2a78c20424380/aiohttp-3.14.5-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:823c910f046f23f4c713b8d99a2242dc65f591cb45ee86418fa11762a3c2963c", size = 549832, upload-time = "2026-10-11T01:03:32.02Z" }, + { url = "https://files.pythonhosted.org/packages/29/5c/969a1b72692055fd2a419590c41847ec9144fefb97b75ff1cde5b6372891/aiohttp-3.14.5-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:9b42db919715e91eb76acf3bc492a9a7ccd8bd9adc6745c1412b689735269f14", size = 549234, upload-time = "2026-10-11T01:03:34.469Z" }, + { url = "https://files.pythonhosted.org/packages/38/05/8e3e07fd8a0d33d06955ff4e54a1cb92f4bce347ff55e441dc3e25a7b5e5/aiohttp-3.14.5-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3112585250b199296c26ca6e0131640b6a8d01bab8b232d2eb3763ed469de11", size = 1891314, upload-time = "2026-10-11T01:03:36.899Z" }, + { url = "https://files.pythonhosted.org/packages/b6/b3/05a79ce2e25f024e93de30c94f39dc6aa6e2bc1e9c531a5c4b18dc61b7a4/aiohttp-3.14.5-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c147451b4a58e7050f7f7394e6c467867c84161560001f9ad4fb2d1446743946", size = 1884372, upload-time = "2026-10-11T01:03:39.334Z" }, + { url = "https://files.pythonhosted.org/packages/94/52/0fd8af0717db109eea258191b326b5cb5847fb88928bfa6c862fd78b9ad3/aiohttp-3.14.5-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bf163cc701f3d4ac43ba7d97771bf5fd955220ef5500ef3ee847bc0ecfbf4ec1", size = 1930957, upload-time = "2026-10-11T01:03:42.172Z" }, + { url = "https://files.pythonhosted.org/packages/4e/b3/fa78733da88812bf9fb193913fb0ce1548f8b6912047633fdf88a758ff8c/aiohttp-3.14.5-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f8d40ce41991e9d56fab4f5dc4a51fe59bc3b5c77c27f4b148963064d00232e8", size = 2038886, upload-time = "2026-10-11T01:03:44.646Z" }, + { url = "https://files.pythonhosted.org/packages/cf/f5/2fcc5e30053a938286f17d0edf3f0850b8061b984256fa7c26850b9c8809/aiohttp-3.14.5-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:276a4fc00b1d9ae492b802763a789c5b86328b989c5ea169f2faa447d6a11c7c", size = 1894470, upload-time = "2026-10-11T01:03:47.304Z" }, + { url = "https://files.pythonhosted.org/packages/62/2a/f87feb42abe8e6a7c03814dbcb711540849a1e90aa392055f3183c643610/aiohttp-3.14.5-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:50983e3be33d8c0942ab88cec3905b10602f64c469b20153c48c5d4e558dd016", size = 1716273, upload-time = "2026-10-11T01:03:50.121Z" }, + { url = "https://files.pythonhosted.org/packages/81/b2/adf1f960dd977722ed1347d33da512a1624807114f57a3b91f5cc828e081/aiohttp-3.14.5-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:16c8abd5bca220a47efe667d26f8460124c81810787e79ee87b242677563d9dd", size = 1857513, upload-time = "2026-10-11T01:03:52.959Z" }, + { url = "https://files.pythonhosted.org/packages/d7/fd/ef8d910e641de4160026a513ace5888b7f92826bbc3fd90ced05d55a828e/aiohttp-3.14.5-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:0790ec66fa4013e83c53b9025a45d454723da1a2fce28b3208c9b32d08af162f", size = 1887704, upload-time = "2026-10-11T01:03:55.641Z" }, + { url = "https://files.pythonhosted.org/packages/7e/db/6c9142f941cba8d35be8e1fae6ea2bd390e754fc14076b8147aee1a4592f/aiohttp-3.14.5-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:cb11a971a3aea10f9b8373be628f1df932964fc6c6b174516d318a48c3ac4412", size = 1895684, upload-time = "2026-10-11T01:03:58.18Z" }, + { url = "https://files.pythonhosted.org/packages/e6/7c/6a6bd9a72e576d376c668333b00c51c6147aba4fb863ed6c94996d497eca/aiohttp-3.14.5-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:932ce7e694bbc29b2bf6f64f2343c27d148d4997c771d01bdade4639b6749ff4", size = 1706299, upload-time = "2026-10-11T01:04:00.8Z" }, + { url = "https://files.pythonhosted.org/packages/69/ec/d2cc494242f8c4d3d1cd70baf591742818a74386dc3b84af3195c5c4fced/aiohttp-3.14.5-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:a7d470cf7b206e6359fc77b1b860632fde400d5a2ed59cd0181b93a686bc81ee", size = 1916645, upload-time = "2026-10-11T01:04:03.892Z" }, + { url = "https://files.pythonhosted.org/packages/a3/6e/e852c53647e815db09a1b6b5ab634f54bd736412397e11940feef4d2c89a/aiohttp-3.14.5-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:6e1d8637cf73eebc92eba2e11d4cfff98a3b562f2505bd75bba766d908926e8d", size = 1875408, upload-time = "2026-10-11T01:04:06.922Z" }, + { url = "https://files.pythonhosted.org/packages/fe/f6/72ab6ef20c332399be593bac543d2c24a6d241e39d3540ce9f95a63e4bd2/aiohttp-3.14.5-cp315-cp315-win32.whl", hash = "sha256:fbdc5ec49f9ca3cd24955cf3520b10a4d4c901ba2572094c84274e9e7eb30534", size = 485590, upload-time = "2026-10-11T01:04:09.626Z" }, + { url = "https://files.pythonhosted.org/packages/06/eb/e9de75b8c6d2170c42c08ff303abf857ea8a6d9d9b6e99b5aba40f15e962/aiohttp-3.14.5-cp315-cp315-win_amd64.whl", hash = "sha256:a9d3983bd6ab7aa1cfd573544ae98df9b6cb6912a5185a198263e024a636861d", size = 511740, upload-time = "2026-10-11T01:04:12.277Z" }, + { url = "https://files.pythonhosted.org/packages/fc/25/455f3c2785eb0d50748cffd0abd07500815f419a9495b14610b7622d8d2d/aiohttp-3.14.5-cp315-cp315-win_arm64.whl", hash = "sha256:e29347c142cf6e99e0dff5e2995ead1d50fa3b51bf37a7c726a7ccfe5419745a", size = 495171, upload-time = "2026-10-11T01:04:14.667Z" }, + { url = "https://files.pythonhosted.org/packages/e7/6c/497f0a98782eebfcf0f02a7fbdef5428148bd426027140cbad494cf842b5/aiohttp-3.14.5-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:9bf1d5dcc15204d9ec8b8ea4c18fd66e6b80e5de1f4ecbafb3a2f2740f8039d4", size = 855638, upload-time = "2026-10-11T01:04:17.154Z" }, + { url = "https://files.pythonhosted.org/packages/9a/c5/55c0cef2572af9b1ee81608f7c0a1bf74e6c9151a73b04915d933f192335/aiohttp-3.14.5-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:14f04769cfefe4734016a856a83af36133cd17779cef9ae817f812b8ba9d6d51", size = 567455, upload-time = "2026-10-11T01:04:19.702Z" }, + { url = "https://files.pythonhosted.org/packages/4d/47/e1a0e39f4a2b881f6071225afaf94a6547001b5aefc1566734d73c685934/aiohttp-3.14.5-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:6e4251c0ba4624a68a2c11471a1ac54c3306876c21f0ae86de085cc9241c8905", size = 567836, upload-time = "2026-10-11T01:04:22.235Z" }, + { url = "https://files.pythonhosted.org/packages/c3/1d/817d85836f52b687160064a326e62e037dc42f1ad5b6b5188a70e5c134b5/aiohttp-3.14.5-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e4f5cf4dc72a71c4cfa9751b4950be22f733626670230d46e7d606592aa22d59", size = 2051409, upload-time = "2026-10-11T01:04:24.821Z" }, + { url = "https://files.pythonhosted.org/packages/f7/25/e8ea6fc212a9aabce982917346ce8ecab0929c74b60232a056dd11c99da0/aiohttp-3.14.5-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:dbf53ae2601b7fd5a93c3944deea3a78d40f495226d582c35ef7a433425ce2b2", size = 1936493, upload-time = "2026-10-11T01:04:27.456Z" }, + { url = "https://files.pythonhosted.org/packages/24/33/de0517f71f1a19feea4aff78a2ec4ec2d98634129ec30ead78fce2f8f81b/aiohttp-3.14.5-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:657291433bf4dd3142f3abac495764cd47d0c7c92087751e6666c6447e65fcef", size = 2021405, upload-time = "2026-10-11T01:04:30.276Z" }, + { url = "https://files.pythonhosted.org/packages/c4/11/ddaf2e7930543e9f0cad3a1c54e1af0c1bd2fae3973d568c4263d3b010e9/aiohttp-3.14.5-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3e51a27980c3788e6e6b3325d694fdd4898087fa8a86b2763af77b39353da41e", size = 2142017, upload-time = "2026-10-11T01:04:33.022Z" }, + { url = "https://files.pythonhosted.org/packages/1d/7b/58784353c06de8adc20f426daad3d85dd86fd55331c713a3f4b638c94573/aiohttp-3.14.5-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c7583cd3dfdc7dcc927835b4f6c7faae7ecc1ba3ca5879321621ae2e6f8e84", size = 2002482, upload-time = "2026-10-11T01:04:35.94Z" }, + { url = "https://files.pythonhosted.org/packages/b9/f0/417d9535caa9e165ffe6a53e2347a78107e7c87dcb0e10aca315c3af0344/aiohttp-3.14.5-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6fdcd6af7e2e51d1ba1b4bea16e97b074bcb7b5dd0246a9d8201341bb28085a0", size = 1791774, upload-time = "2026-10-11T01:04:38.733Z" }, + { url = "https://files.pythonhosted.org/packages/98/01/25e49c2e8a01b9f0e19ca0a8448ad50aa2bdf96c8cd41e92bd45af044784/aiohttp-3.14.5-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:c8859a013ae0de1074660992139a1a440df3e6b219b86cf0d3f11c2692bb4fe3", size = 1970358, upload-time = "2026-10-11T01:04:41.66Z" }, + { url = "https://files.pythonhosted.org/packages/2d/fc/c132fd3465b6c7e4ce0193154f602c3e6c46b680e4da7eb3bd0d8a40d1c7/aiohttp-3.14.5-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:8966ecac808dd5f473c9c4cefd10cd3ffda71c18a4d3493b7c7d2ae1803bf2cc", size = 1969192, upload-time = "2026-10-11T01:04:44.66Z" }, + { url = "https://files.pythonhosted.org/packages/95/4e/d58b45e7dba4eb607eca11fc0a4aa77ae0f39c11afa804635a61ce18cff4/aiohttp-3.14.5-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:3093b72c215bda16ce961a6d073f6e71d46e022962a9d5d457c5d4d421c78b57", size = 1963618, upload-time = "2026-10-11T01:04:47.505Z" }, + { url = "https://files.pythonhosted.org/packages/a2/d9/f6ac50946efb3490428ef52b56e62c1c6b7f9c6ff3ec6083535526c83e60/aiohttp-3.14.5-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:149fb56caf7acb67073126f675d0958d9c4b3125fcd3f6d4877df98aa8a97ce9", size = 1784817, upload-time = "2026-10-11T01:04:50.287Z" }, + { url = "https://files.pythonhosted.org/packages/d0/2f/f255eb63da788cd8a452fe350869c03266ccad7d884925f6963c87808f24/aiohttp-3.14.5-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:293d3ae7c6a0ed176a42e59a1b5fde825ead65c835360f734148e96729f928d2", size = 1995956, upload-time = "2026-10-11T01:04:53.213Z" }, + { url = "https://files.pythonhosted.org/packages/d9/9b/241aa3393eaafda0034470add1625f82a4c252901108723b283a9b0b32ca/aiohttp-3.14.5-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:3f2dcc00191fd563e9075181a14ec31d7dd63223ced7582cc70a15a499de0c79", size = 1967794, upload-time = "2026-10-11T01:04:55.958Z" }, + { url = "https://files.pythonhosted.org/packages/5f/7f/a68e689288c9e4bfcf8d0979f2b12b774bf01861985420df6150eba5448e/aiohttp-3.14.5-cp315-cp315t-win32.whl", hash = "sha256:7779cd97e61ebe583ec2f1c5616cdd038aa08a4453b1848c67842176d054948e", size = 503876, upload-time = "2026-10-11T01:04:58.978Z" }, + { url = "https://files.pythonhosted.org/packages/a4/78/49b0299da6d54de19fc6fdc6889d50233ac47d192a461624f4fc010fde83/aiohttp-3.14.5-cp315-cp315t-win_amd64.whl", hash = "sha256:0e6f16f5e49c4b8267988c05ab07760d7064cea57d077c3d068d04b0fbb992cb", size = 530957, upload-time = "2026-10-11T01:05:02.23Z" }, + { url = "https://files.pythonhosted.org/packages/21/d4/b0afc936aeb6d2f93157e3408b069ec5d7934429ae7d023de5e0953b1887/aiohttp-3.14.5-cp315-cp315t-win_arm64.whl", hash = "sha256:1aead151c3abbac6b32942e452020cb66d7efc099d253cc6c20f748e926c858b", size = 510408, upload-time = "2026-10-11T01:05:05.362Z" }, { url = "https://files.pythonhosted.org/packages/68/30/173960c42b05a6c59f7558e4b12a4b0d9ba376cf6aa9bde7f9e08a30ca8d/aiohttp-3.14.5-py3-none-any.whl", hash = "sha256:efc21a454892828368b11c2c780de0ff8bc991f73f6b99c6b66e56205470929b", size = 279517, upload-time = "2026-10-11T01:05:08.523Z" }, ] @@ -239,7 +282,7 @@ version = "0.17.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "sniffio" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" } }, { name = "wrapt" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/a7/809482759f40079f4c4328c7318bf569ae25d457f5017aad30a1b9aafedc/aiologic-0.17.0.tar.gz", hash = "sha256:65aa058e858c94cd208badb188e7f00b54dcabb3ba85b34f794db98074d108b9", size = 251625, upload-time = "2026-06-14T12:24:35.367Z" } @@ -253,7 +296,7 @@ version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "frozenlist" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } wheels = [ @@ -277,7 +320,8 @@ dependencies = [ { name = "mako" }, { name = "sqlalchemy" }, { name = "tomli", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/94/13/8b084e0f2efb0275a1d534838844926f798bd766566b1375174e2448cd31/alembic-1.18.4.tar.gz", hash = "sha256:cb6e1fd84b6174ab8dbb2329f86d631ba9559dd78df550b57804d607672cedbc", size = 2056725, upload-time = "2026-02-10T16:00:47.195Z" } wheels = [ @@ -312,9 +356,11 @@ dependencies = [ { name = "docstring-parser" }, { name = "httpx" }, { name = "jiter" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "sniffio" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/04/ea/0869d6df9ef83dcf393aeefc12dd81677d091c6ffc86f783e51cf44062f2/anthropic-0.84.0.tar.gz", hash = "sha256:72f5f90e5aebe62dca316cb013629cfa24996b0f5a4593b8c3d712bc03c43c37", size = 539457, upload-time = "2026-02-25T05:22:38.54Z" } wheels = [ @@ -333,7 +379,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, { name = "idna" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } wheels = [ @@ -405,7 +451,7 @@ name = "asgiref" version = "3.11.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/63/40/f03da1264ae8f7cfdbf9146542e5e7e8100a4c66ab48e791df9a03d3f6c0/asgiref-3.11.1.tar.gz", hash = "sha256:5f184dc43b7e763efe848065441eac62229c9f7b0475f41f80e207a114eda4ce", size = 38550, upload-time = "2026-02-03T13:30:14.33Z" } wheels = [ @@ -418,9 +464,11 @@ version = "0.52.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "pydantic-settings", marker = "python_full_version >= '3.14'" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "websockets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/57/c9/d111b0546635fee21eccaef00edcac8146b450c840dd65ca8d193921ac07/assemblyai-0.52.4.tar.gz", hash = "sha256:c31f777409ba030690840faec6840e36bf0223851becd008e5e61df199034546", size = 57906, upload-time = "2026-02-25T15:30:22.667Z" } @@ -433,7 +481,7 @@ name = "astroid" version = "4.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/07/63/0adf26577da5eff6eb7a177876c1cfa213856be9926a000f65c4add9692b/astroid-4.0.4.tar.gz", hash = "sha256:986fed8bcf79fb82c78b18a53352a0b287a73817d6dbcfba3162da36667c49a0", size = 406358, upload-time = "2026-02-07T23:35:07.509Z" } wheels = [ @@ -535,7 +583,7 @@ dependencies = [ { name = "aiofiles" }, { name = "aiohttp" }, { name = "colorlog" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" } }, { name = "python-dotenv" }, { name = "requests" }, { name = "requests-toolbelt" }, @@ -634,7 +682,8 @@ version = "1.41.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "requests" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a6/f3/b416179e408990df5db0d516283022dde0f5d0111d98c1a848e41853e81c/azure_core-1.41.0.tar.gz", hash = "sha256:f46ff5dfcd230f25cf1c19e8a34b8dc08a337b2503e268bb600a16c00db8ad5a", size = 381042, upload-time = "2026-05-07T23:30:54.302Z" } wheels = [ @@ -650,7 +699,8 @@ dependencies = [ { name = "cryptography" }, { name = "msal" }, { name = "msal-extensions" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c2/3a/439a32a5e23e45f6a91f0405949dc66cfe6834aba15a430aebfc063a81e7/azure_identity-1.25.2.tar.gz", hash = "sha256:030dbaa720266c796221c6cdbd1999b408c079032c919fef725fcc348a540fe9", size = 284709, upload-time = "2026-02-11T01:55:42.323Z" } wheels = [ @@ -664,7 +714,8 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "azure-core" }, { name = "isodate" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/97/e5/3074e581b6e8923c4a1f2e42192ea6f390bb52de3600c68baaaed529ef05/azure_keyvault_secrets-4.10.0.tar.gz", hash = "sha256:666fa42892f9cee749563e551a90f060435ab878977c95265173a8246d546a36", size = 129695, upload-time = "2025-06-16T22:52:20.986Z" } wheels = [ @@ -679,7 +730,8 @@ dependencies = [ { name = "azure-core" }, { name = "cryptography" }, { name = "isodate" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/71/24/072ba8e27b0e2d8fec401e9969b429d4f5fc4c8d4f0f05f4661e11f7234a/azure_storage_blob-12.28.0.tar.gz", hash = "sha256:e7d98ea108258d29aa0efbfd591b2e2075fa1722a2fae8699f0b3c9de11eff41", size = 604225, upload-time = "2026-01-06T23:48:57.282Z" } wheels = [ @@ -694,7 +746,8 @@ dependencies = [ { name = "azure-core" }, { name = "azure-storage-blob" }, { name = "isodate" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4d/4b/66e68e1cb5655d3f290541151dc97c82ae565e4146b66f72f3465ba6e617/azure_storage_file_datalake-12.20.0.tar.gz", hash = "sha256:8d6932b75470de935998755d980a3296412d8ff2a11631c1d99267ae65f11f03", size = 283945, upload-time = "2025-03-27T17:43:19.42Z" } wheels = [ @@ -854,7 +907,8 @@ version = "4.14.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "soupsieve" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c3/b0/1c6a16426d389813b48d95e26898aff79abbde42ad353958ad95cc8c9b21/beautifulsoup4-4.14.3.tar.gz", hash = "sha256:6292b1c5186d356bba669ef9f7f051757099565ad9ada5dd630bd9de5fa7fb86", size = 627737, upload-time = "2025-11-30T15:08:26.084Z" } wheels = [ @@ -1002,6 +1056,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/da/1ec2e9b4f500693e56e6391f128dd0d304d6d2b7f0d5090e80a802b722f1/brotli-1.2.1-cp314-cp314-win32.whl", hash = "sha256:0a30e5df5ba985b67f85e13dfd7e5a68e186d8d9b5cff7d9a8904832bb413e81", size = 338208, upload-time = "2026-10-22T13:58:40.927Z" }, { url = "https://files.pythonhosted.org/packages/24/5e/80cd38322263da5e92ea4935423f1d3cfa2509201673dce77b772c47b997/brotli-1.2.1-cp314-cp314-win_amd64.whl", hash = "sha256:972a2e19dbaab47d0aa77c1d07f71c930bdc53538f2b33decc905e92a4be5cbd", size = 376707, upload-time = "2026-10-22T13:58:42.025Z" }, { url = "https://files.pythonhosted.org/packages/c5/35/cb35c489fcd0a813bbb7500465af5a7bac3bee7d996f6a211bac9096ef2c/brotli-1.2.1-cp314-cp314-win_arm64.whl", hash = "sha256:a3dea7e1ac49e29306500a8627fa3eacabcc9e238d22495eea2d8e61de77065b", size = 390280, upload-time = "2026-10-22T13:58:43.256Z" }, + { url = "https://files.pythonhosted.org/packages/a2/72/85ea89c1088673a297e8b652cf0080668812144d5525db6512eadddf9d19/brotli-1.2.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:eea17e3cdc6f9aecffe15fe9a9eeba8e8c049a5d347de1561eed2240cdf35a79", size = 863002, upload-time = "2026-10-22T13:58:44.325Z" }, + { url = "https://files.pythonhosted.org/packages/81/6a/bd4db4ef0c4d083d8a2e5a1d0b20239b186f261e72b14a9a0ec91c078e92/brotli-1.2.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:761db18dc32b24d00fbedca1013089eb3470053cefa8492bd9c07b11102d72bf", size = 443263, upload-time = "2026-10-22T13:58:45.474Z" }, + { url = "https://files.pythonhosted.org/packages/79/1a/08635be453d523474fdfce81f588583a4e8049c75c04df849b5a278bf027/brotli-1.2.1-cp315-cp315-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:035f3bf861328d8b7fcb61f4a976f9913027158e5666116733d59ffc65f2745e", size = 1244379, upload-time = "2026-10-22T13:58:46.615Z" }, + { url = "https://files.pythonhosted.org/packages/3b/94/52f2facc5ae118a68c4f0e715e03c9f854507a359608804d7de2dfe8d603/brotli-1.2.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:aa32b51702a0b6e9fddc0d1dc66cb6ed6af112f1bba0cac2be9039546a862b5c", size = 1404133, upload-time = "2026-10-22T13:58:47.756Z" }, + { url = "https://files.pythonhosted.org/packages/1e/9b/768635c710b684574da55e12bcdbe36419fd8bf6cb97e01c72a4593f9747/brotli-1.2.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:810aea5958d6a2d92593fd27218df528644aeb72cc7f6d401e650752d6c98949", size = 1424875, upload-time = "2026-10-22T13:58:48.912Z" }, + { url = "https://files.pythonhosted.org/packages/46/39/7b7d53d7cea61f35f6b52ea149804523d01e532bf26017eed4351d82a7ad/brotli-1.2.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f00ff4b56caf151365c77988eb85a4a71eb26fe63047d4084d5614e06d65e3cf", size = 1425345, upload-time = "2026-10-22T13:58:50.253Z" }, + { url = "https://files.pythonhosted.org/packages/cb/5e/652c788a4da5c6415c7ae31382626ba870020c9280e0f9be1a6dfec80261/brotli-1.2.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:dafaeb38712f99e5c31ad7f02c366f1ddd3792921b94066015f8c49f197163b0", size = 1487925, upload-time = "2026-10-22T13:58:51.593Z" }, + { url = "https://files.pythonhosted.org/packages/85/c9/1b7753d02c106ff70b3d88056586ed7c2fe329d2aa5707cdaebe90372942/brotli-1.2.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:972e34c5cbbfe6c90358bfae38edae1a29d3456e920da5a4664c98f221ee4df8", size = 1276526, upload-time = "2026-10-22T13:58:53.142Z" }, + { url = "https://files.pythonhosted.org/packages/a5/5f/6ae3e85bfb7a3f4711cea5a1b92d4377fddf1319a666cda213cc83b6c040/brotli-1.2.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:59ca34c7a83d7a3b842247183c426f891a95a8f57db41b7aac92195c51e8a83a", size = 1594681, upload-time = "2026-10-22T13:58:54.571Z" }, + { url = "https://files.pythonhosted.org/packages/8a/52/2808406d83e5462c086d69dbebb892f4ed78d2eb4b6755ea4b9e58357e3b/brotli-1.2.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:90bd49511341dd7c40bb3acaa97cd8b9e5df4821a5714b842117e7069c7cc03e", size = 1486968, upload-time = "2026-10-22T13:58:56.008Z" }, + { url = "https://files.pythonhosted.org/packages/a9/53/35042a907c309af8defb423e58692bdc5e3e1fd3d03388892b7614270626/brotli-1.2.1-cp315-cp315-win32.whl", hash = "sha256:1880db6241276e191d3e34a6e4eea708c6629103c396f0ea21e75b0ce63a9a2c", size = 338212, upload-time = "2026-10-22T13:58:57.189Z" }, + { url = "https://files.pythonhosted.org/packages/38/6c/883a32c23603e278ae93d8df31f0573b67382de827e6e05a6e9ee0b51896/brotli-1.2.1-cp315-cp315-win_amd64.whl", hash = "sha256:09a2f112a638aca15a88f298571788795fc45d62f2bca7f3870c1152153ef399", size = 376707, upload-time = "2026-10-22T13:58:58.292Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d5/2c574da9faf130a59c4c685ba1378f2f470b57fb9fa5d3b0f8b4e90579a0/brotli-1.2.1-cp315-cp315-win_arm64.whl", hash = "sha256:91097d66ebb8ca8d23372081a33405a44e2239445614888e38929bd7f5879dae", size = 390278, upload-time = "2026-10-22T13:58:59.512Z" }, { url = "https://files.pythonhosted.org/packages/7f/eb/803fb4003fc74fb7202eade0832057729837c0929b690e7f691c1778f230/brotli-1.2.1-pp311-pypy311_pp80-macosx_10_15_x86_64.whl", hash = "sha256:93269f11303fc9a3e3990b57e0533f41949546156d9f0e535571f313032563cc", size = 416747, upload-time = "2026-10-22T13:59:59.612Z" }, { url = "https://files.pythonhosted.org/packages/60/fc/48f827dda08844d6313a97c6c34b7f0cffc8310bf4f2cd4d98fae6c198fb/brotli-1.2.1-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:87492842770cd772fce25e9cd279913eaea6c55a91a960b6302aedfa2ec7b152", size = 399671, upload-time = "2026-10-22T14:00:01.176Z" }, { url = "https://files.pythonhosted.org/packages/c3/2b/21aa1de7b7b5739f8d9b55c12100de794c42b10abd1a83da1a99072e2b46/brotli-1.2.1-pp311-pypy311_pp80-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f06dd6a30438692d48c0d54de4b659cd55a57c2c2f6591c969c98d0a5523ab07", size = 404664, upload-time = "2026-10-22T14:00:02.357Z" }, @@ -1289,7 +1356,7 @@ dependencies = [ { name = "jsonschema" }, { name = "mcp" }, { name = "sniffio" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7f/45/10f00a408b480926b5e340e3507562ede1e30d4ce5b73eea9ccc3468a9e6/claude_agent_sdk-0.2.165.tar.gz", hash = "sha256:1bfa8e7a6bb36e82de9a12324c9bcfa6eebd4f96ee5890f0c656d50445240ff8", size = 365169, upload-time = "2026-10-08T18:18:49.528Z" } wheels = [ @@ -1435,7 +1502,8 @@ name = "contourpy" version = "1.3.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", @@ -1655,7 +1723,7 @@ version = "50.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/de/41/6cbdcf9142d00fe82836fbb51e503e58088575cf7a0fe1dbff6695bf0840/cryptography-50.0.0.tar.gz", hash = "sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9", size = 880201, upload-time = "2026-07-31T14:25:10.11Z" } wheels = [ @@ -1712,7 +1780,7 @@ version = "0.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiologic" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/d9/e3/49afa1bc180e0d28008ec6bcdf82a4072d1c7a41032b5b759b60814ca4b0/culsans-0.11.0.tar.gz", hash = "sha256:0b43d0d05dce6106293d114c86e3fb4bfc63088cfe8ff08ed3fe36891447fe33", size = 107546, upload-time = "2025-12-31T23:15:38.196Z" } wheels = [ @@ -1811,6 +1879,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/8e/b97984d9f43e2f36f5317034711140c3bd1fbb5d96880b1c11b9e1e8c3d3/ddtrace-4.11.0-cp314-cp314-win32.whl", hash = "sha256:c267fe4efceb71019f5e6833fdb4fbdff18a51641bdca9fceabbe579984461e4", size = 5831072, upload-time = "2026-07-10T08:57:03.578Z" }, { url = "https://files.pythonhosted.org/packages/e8/6a/7fb18925588f645d6d777c90d388f9a640adf7b986be7457943717b74170/ddtrace-4.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:3bebf3cb12ac51c422ad1ed74873746b8398cce845075b234115c2f518a7a652", size = 6455126, upload-time = "2026-07-10T08:57:06.362Z" }, { url = "https://files.pythonhosted.org/packages/a0/b2/44d7f07ba94c70bd1827669d446a2e979f7efd6e4a409c469a59684d9a7b/ddtrace-4.11.0-cp314-cp314-win_arm64.whl", hash = "sha256:f596cb2da582600e091c8174ab4f8e44bf4f05c50c03cc450e0378c516e0c6e3", size = 6130096, upload-time = "2026-07-10T08:57:09.413Z" }, + { url = "https://files.pythonhosted.org/packages/99/22/dadad1a1213a846499f360344c4be0854635bbd9c5726d2902bc8e92b194/ddtrace-4.11.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9d9a7c4644e4ffc858e951eeab37ab86e9b717ef1bc2f2a47de69f1ec373bccb", size = 6454958, upload-time = "2026-07-10T08:57:12.714Z" }, + { url = "https://files.pythonhosted.org/packages/d4/45/4a5bb647d47f01d135c918ad33678d1dba5b0210cc688da5d2f69534830e/ddtrace-4.11.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b49433f0e85e16c34f578e01ec4e263ab5181d9636635add8d6194bd47dd163c", size = 6638706, upload-time = "2026-07-10T08:57:15.791Z" }, + { url = "https://files.pythonhosted.org/packages/80/2b/175096efe3784f36d383e09d7bfa1defdba85689008417008f122a3e8d0c/ddtrace-4.11.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2affbac04583808cba374976d07e59a4fe156d1de122f8def30550bf7717b745", size = 7466814, upload-time = "2026-07-10T08:57:18.873Z" }, + { url = "https://files.pythonhosted.org/packages/80/b3/8fb55c9183ed5a162525ae308e0aaa8237ceb2a3ba78d65a8f1336c16b21/ddtrace-4.11.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:51a28ab8072c741e104d8c6f47d990eb5989a359b7ea4d91bab7c5c1f5c1a03c", size = 7707994, upload-time = "2026-07-10T08:57:21.918Z" }, ] [[package]] @@ -1929,7 +2001,8 @@ name = "docutils" version = "0.22.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", @@ -1966,7 +2039,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1996,7 +2069,8 @@ name = "expression" version = "5.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/43/c7/bb061623b5815566bda69f5e9d156e38a97ebb383b8db3d2dedb26415466/expression-5.6.0.tar.gz", hash = "sha256:454f6fe138347194a43c7f878d958efe9b84b9cc770e462010c7a52e18058065", size = 59147, upload-time = "2025-02-19T09:37:37.432Z" } wheels = [ @@ -2010,7 +2084,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "redis" }, { name = "sortedcontainers" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/11/40/fd09efa66205eb32253d2b2ebc63537281384d2040f0a88bcd2289e120e4/fakeredis-2.34.1.tar.gz", hash = "sha256:4ff55606982972eecce3ab410e03d746c11fe5deda6381d913641fbd8865ea9b", size = 177315, upload-time = "2026-02-25T13:17:51.315Z" } wheels = [ @@ -2023,10 +2097,13 @@ version = "0.136.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "starlette" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, + { name = "typing-inspection", version = "0.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-inspection", version = "0.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/81/2d/ff8d91d7b564d464629a0fd50a4489c97fcb836ac230bf3a7269232a9b1f/fastapi-0.136.3.tar.gz", hash = "sha256:e487fae93ad408e6f47641ee4dfe389864fd7bec92e547ea8498fc13f43e83ab", size = 396410, upload-time = "2026-05-23T18:53:15.192Z" } wheels = [ @@ -2053,7 +2130,8 @@ dependencies = [ { name = "fastapi" }, { name = "httpx" }, { name = "oauthlib" }, - { name = "pydantic", extra = ["email"] }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, extra = ["email"], marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, extra = ["email"], marker = "python_full_version >= '3.15'" }, { name = "pyjwt" }, ] sdist = { url = "https://files.pythonhosted.org/packages/74/fc/644bc8f82fc887fffcf9a3eab8eb3dea06ee9ea160ef20441455ed7a0001/fastapi_sso-0.19.0.tar.gz", hash = "sha256:629f00581f72ea7e57f7b8775f8d2c425629c428c194359a2b4ebaa6bcb8e12b", size = 17278, upload-time = "2025-12-17T15:18:06.721Z" } @@ -2546,7 +2624,8 @@ name = "google-api-core" version = "2.25.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", ] dependencies = [ { name = "google-auth" }, @@ -2659,8 +2738,10 @@ dependencies = [ { name = "packaging" }, { name = "proto-plus" }, { name = "protobuf" }, - { name = "pydantic" }, - { name = "typing-extensions" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d4/be/31ce7fd658ddebafbe5583977ddee536b2bacc491ad10b5a067388aec66f/google_cloud_aiplatform-1.133.0.tar.gz", hash = "sha256:3a6540711956dd178daaab3c2c05db476e46d94ac25912b8cf4f59b00b058ae0", size = 9921309, upload-time = "2026-01-08T22:11:25.079Z" } wheels = [ @@ -2774,7 +2855,8 @@ name = "google-cloud-storage" version = "3.4.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", ] dependencies = [ { name = "google-api-core", version = "2.25.2", source = { registry = "https://pypi.org/simple" } }, @@ -2855,10 +2937,12 @@ dependencies = [ { name = "anyio" }, { name = "google-auth" }, { name = "httpx" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "requests" }, { name = "tenacity" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "websockets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/11/b321935a5b58a82f3f65f1bce560bfefae76a798ef5d2f6b5a9fa52ad27b/google_genai-1.37.0.tar.gz", hash = "sha256:1e9328aa9c0bde5fe2afd71694f9e6eaf77b59b458525d7a4a073117578189f4", size = 244696, upload-time = "2025-09-16T04:23:45.159Z" } @@ -2877,9 +2961,11 @@ dependencies = [ { name = "google-api-python-client" }, { name = "google-auth" }, { name = "protobuf" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "tqdm" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/97/0f/ef33b5bb71437966590c6297104c81051feae95d54b11ece08533ef937d3/google_generativeai-0.8.6-py3-none-any.whl", hash = "sha256:37a0eaaa95e5bbf888828e20a4a1b2c196cc9527d194706e58a68ff388aeb0fa", size = 155098, upload-time = "2025-12-16T17:53:58.61Z" }, @@ -2916,91 +3002,101 @@ grpc = [ [[package]] name = "granian" -version = "2.7.4" +version = "2.8.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/db/0c/27aa25280b6c1f323312e83088304da8a7f3e5c1e568d3a560365ec6fa67/granian-2.7.4.tar.gz", hash = "sha256:1dc0530d7ae6b0ae43aafafe771ac0b8c38af68bbd71ab355828817faf13aac1", size = 128212, upload-time = "2026-04-23T11:55:55.275Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e2/63/c2548102956296ee0f8671291dd5a6f3f3dc2fc7c4e62e0b86e3be6de571/granian-2.8.5.tar.gz", hash = "sha256:032588689e2d524d633fcb088ee54fad4c5655d0af1a75058d5b4240be4f6864", size = 130933, upload-time = "2026-10-20T11:43:38.079Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1c/23/ccaa1c786aa528a672d6f25ce236156aeb06a63931109aa6f2d4d3c8a350/granian-2.7.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:1c2a13c5c119e34369f984d8414edb8ba3793d7c78c37bb795942648dda3eca1", size = 6354293, upload-time = "2026-04-23T11:53:32.922Z" }, - { url = "https://files.pythonhosted.org/packages/91/b7/2b0b0e2dc07cb17febbbf3b349c96f425a53d913ba76278091f821adfc22/granian-2.7.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:91963c4928a355d772f14075057ff721423bce70612a619edc2daf04dd258577", size = 6050464, upload-time = "2026-04-23T11:53:34.912Z" }, - { url = "https://files.pythonhosted.org/packages/a3/9c/1feb488abb85446ac7d12d05a93788a3a6b42810d64d5806ebc309c65fc9/granian-2.7.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9d068796cb7e8e0b7a4c8d51077701e37104a39cd103c655a5c232ad561fb07c", size = 6883201, upload-time = "2026-04-23T11:53:36.73Z" }, - { url = "https://files.pythonhosted.org/packages/c8/ac/176b47bb48689fe3575ccbf372d67dd52fb0390686f4d8b7174ab2538ce5/granian-2.7.4-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c932f5c292b643019c4dd410a352789dbb8cb2cb41ec5b373779a87375de398a", size = 6135933, upload-time = "2026-04-23T11:53:38.552Z" }, - { url = "https://files.pythonhosted.org/packages/25/5d/4d894789683bd074bbe0361df7e3e524a2b763e44d771e326a16d4ea014b/granian-2.7.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7a8f411408b0b65a07460e39cb53178e30a15ff5f0c77ed6aa31e1106590ea9", size = 6783971, upload-time = "2026-04-23T11:53:40.602Z" }, - { url = "https://files.pythonhosted.org/packages/88/e3/fe781d55306f1542b3db0fac13b70b39846b5e9a400fda9a3a2bf04a9af6/granian-2.7.4-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:6b7ab6a1a0c0d77ec1dd1145b7c8f3da5251ec7926c005da22f7415bf1b217a7", size = 6906647, upload-time = "2026-04-23T11:53:42.352Z" }, - { url = "https://files.pythonhosted.org/packages/76/11/c10af6940d8bb73cfb4eaa6781eb3f5c7880e1b69d9d87edc63292171538/granian-2.7.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6036316f781f7ad1412d7aa10b49c5a25e69fae3f67ed766b0923ebb43aa5118", size = 6930262, upload-time = "2026-04-23T11:53:43.855Z" }, - { url = "https://files.pythonhosted.org/packages/f4/3d/5d129792626ac990d3b14d484736b8eafa202966f1a541c5b665c30fc880/granian-2.7.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:acef581d94270a22763fba192fc8cef0df77dac125080ca27e6e847a5e59cd07", size = 7050428, upload-time = "2026-04-23T11:53:45.385Z" }, - { url = "https://files.pythonhosted.org/packages/4c/e0/9a9d9f9d0e68277d49829d53746b2e8b3761e7b79a666fd9a4f445587321/granian-2.7.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:187a85fe36561c74a1db94b858175824c3154ebe6d0aa61c97124427f5c5a5fa", size = 6899482, upload-time = "2026-04-23T11:53:47.372Z" }, - { url = "https://files.pythonhosted.org/packages/19/de/04de408275968d4bff8e6fed9d6abf406beccac27412a08c8daaab7c9534/granian-2.7.4-cp310-cp310-win_amd64.whl", hash = "sha256:c73c6099206288c903a305d975064fbb51f9d0c78d06c914b23dde56165105c9", size = 3995374, upload-time = "2026-04-23T11:53:48.751Z" }, - { url = "https://files.pythonhosted.org/packages/df/00/a7db7e3627992c59927f57d5447638be515e683e2c8037ab7845250270d2/granian-2.7.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:455c51baf51dd0c3d22004fc04f9afb0662cb84ab2b75b48e5d6bb8b3e4e3548", size = 6353285, upload-time = "2026-04-23T11:53:50.113Z" }, - { url = "https://files.pythonhosted.org/packages/b2/23/337ab1a0929cb0cfbdedc06879cff62d6c08cb725fa2d4e139c7e305fed3/granian-2.7.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2c54f3fe69790aa4b685372bcc8f382a8e9ba570b8ea4cb476e3b240a5a5a7c", size = 6050711, upload-time = "2026-04-23T11:53:51.887Z" }, - { url = "https://files.pythonhosted.org/packages/28/ba/3605834adaf5dc9ac3701b817bc9d42c73c89fb67815c7c87c7f64a9b6e1/granian-2.7.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f9549c44b325fe51ee4fc57308761f5178add4d531f1cc333b4a1eedf4a5b7af", size = 6882199, upload-time = "2026-04-23T11:53:53.298Z" }, - { url = "https://files.pythonhosted.org/packages/2e/1e/f11c9773dbf07ed326efc26a771b39ce97f7ee25608c16d69248db2da8da/granian-2.7.4-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0910390ea8f893cc4c3f38a28c923a321609358cf46d31aa7df5c3d3e58e8337", size = 6135800, upload-time = "2026-04-23T11:53:55.186Z" }, - { url = "https://files.pythonhosted.org/packages/f1/96/ca238b4f5d813643264abace48ca630efb1ab6d10409bd9e2c05c1d1ef12/granian-2.7.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0de44552990b3dacb87ea3f37ebbcce67881712c0b0db500013821b14df7e4e", size = 6784786, upload-time = "2026-04-23T11:53:56.706Z" }, - { url = "https://files.pythonhosted.org/packages/ae/85/2feeffc37fe3c8a0f3e932393bcc99c8972984fe95907b34b380284caf1c/granian-2.7.4-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:5c9c6d51a675d9b7084244e63157899dd1afe6f1a5ab014015bc86afd4871df5", size = 6906836, upload-time = "2026-04-23T11:53:58.991Z" }, - { url = "https://files.pythonhosted.org/packages/ae/b6/faed26e3abd741e1d261defc0e7e3b2ecb9a2189c557e829bb28c3281456/granian-2.7.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6be8c6ebbc53efea03284aef87de9b7367df3c9433f7df3b46c1edceaaa9d840", size = 6929985, upload-time = "2026-04-23T11:54:00.846Z" }, - { url = "https://files.pythonhosted.org/packages/25/5c/9f7a836177b5e94ad15da49046254e5b837e1d2c3d03981111d4af9a9d2b/granian-2.7.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:c10e056a6e76da640adb35f88d41ba40ae44065c5e04d4bc35f47c19a7f83a99", size = 7050668, upload-time = "2026-04-23T11:54:02.466Z" }, - { url = "https://files.pythonhosted.org/packages/08/88/19c01761d639b5e2c2eb2f12ff064c6765f32fa7f129c9f48162cdca0668/granian-2.7.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e60a3153456f8922ca73d3a427cc3bb594c021f70ec08ecded6581efe25f48c", size = 6899480, upload-time = "2026-04-23T11:54:03.948Z" }, - { url = "https://files.pythonhosted.org/packages/04/ec/9022f3c2728fcd99f9374ce633e254b201f4bf76fbe60031bbc137f1cf8a/granian-2.7.4-cp311-cp311-win_amd64.whl", hash = "sha256:e9cafbf391d16ea8b8a2e9f88501783fac8da75eb948620899062a17929c4a84", size = 3996087, upload-time = "2026-04-23T11:54:05.771Z" }, - { url = "https://files.pythonhosted.org/packages/8d/d9/148024fd3a8bd974bb5c68a0cb48d15df7763fd1364bf090ccc2d423028a/granian-2.7.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:2c2f40aaecf2ba3d8232e55181c8f6db7bc68d9112a419ab8d5f9e2f33f631f5", size = 6374067, upload-time = "2026-04-23T11:54:07.293Z" }, - { url = "https://files.pythonhosted.org/packages/fc/bb/c53b61a7cb67d33677d96913438eca3d79de1b1b7173a361fcdf2753ade7/granian-2.7.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a8111d5e74b27721e0fdda3edba7c154d44c41b469466857ca3c51b088e3846b", size = 6046338, upload-time = "2026-04-23T11:54:08.684Z" }, - { url = "https://files.pythonhosted.org/packages/29/8d/5c9dc91b9c9a05bf6ed0b795d30f4bb8f290d61502779a89ed2fd75f9fb6/granian-2.7.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:74adbb6c1920dbf4271b824135639318b2a20ff5e33bc35639a8e2928a777234", size = 7000585, upload-time = "2026-04-23T11:54:10.451Z" }, - { url = "https://files.pythonhosted.org/packages/d9/7c/c770593b24a472ab5265a44546f56079757efbf89f8e8b2229a8443e453b/granian-2.7.4-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b778d356b61e0389c823016ad2be50a634b80d3d28a33922f7ac39553e828ad", size = 6255544, upload-time = "2026-04-23T11:54:12.484Z" }, - { url = "https://files.pythonhosted.org/packages/15/46/796147587edb494a330294cb001cf68520ad8296a7da91d80ec672ac8615/granian-2.7.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3607b091c4ef225ee99150f3b02cb827de8d677b52fc75f0b28893244f7bab27", size = 6875124, upload-time = "2026-04-23T11:54:13.967Z" }, - { url = "https://files.pythonhosted.org/packages/c5/25/b867f624886e11053e7a6235244de26fd864a136e65d12295e728b3e5005/granian-2.7.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3d3cf4fe3cafd9b874d8b749c66c790cbf2b4225f2a7d9fb284c51b77a8e938d", size = 6982394, upload-time = "2026-04-23T11:54:15.733Z" }, - { url = "https://files.pythonhosted.org/packages/4f/e1/5746bfe202bd2f6a1506346463ce52dd015c2b5d03d07a53ecf0fddefa3f/granian-2.7.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:846c9cbfea8684ab13d21d66855ad06dc077fb95b5590e7f5040e79994d6429d", size = 6991457, upload-time = "2026-04-23T11:54:17.325Z" }, - { url = "https://files.pythonhosted.org/packages/e0/45/fc6992839d367b6ae8fa8d88b5e70ec293162c3a2e0e6b90fc426f228df2/granian-2.7.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:d34d97cfe4a7805ecb5b1b1684f3f197bb4baf019d2a9f18e34fd1d697a03a7f", size = 7148499, upload-time = "2026-04-23T11:54:19.234Z" }, - { url = "https://files.pythonhosted.org/packages/fe/12/16ffd64a1213858d4cf824767b398758be807dd1a6df5a303dc76994b6d6/granian-2.7.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f11336e4bcd8ef5c5143b075b5260e37e8431eb36d68564cc39416ca526c797f", size = 7006829, upload-time = "2026-04-23T11:54:20.804Z" }, - { url = "https://files.pythonhosted.org/packages/95/9a/f2fcda200f8739ddf25be72591b7a28897be0ffd952a76ec655e5f877144/granian-2.7.4-cp312-cp312-win_amd64.whl", hash = "sha256:9e0a4370773ec4a0e92a55a33fc700b60003e335480e5c7fe941f4bc3dda2e18", size = 4026771, upload-time = "2026-04-23T11:54:22.36Z" }, - { url = "https://files.pythonhosted.org/packages/0a/0f/fa7c63afedcb214edb96703cade360d946d5f1ca59ddb0b3d8e04587fb45/granian-2.7.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:d11da4a4527ba8dc28b5533d5e3241d8d9212e593195d27c6e72c8a422010af5", size = 6373513, upload-time = "2026-04-23T11:54:24.246Z" }, - { url = "https://files.pythonhosted.org/packages/be/39/3088ce32d940f7982102ea3bdc230090e34ac56dc0bce04f2d03b56ea435/granian-2.7.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:057a3db87e93eca1a11255dd13b45b5dd83f798a750fd87f02e14d54db5741b6", size = 6045232, upload-time = "2026-04-23T11:54:25.708Z" }, - { url = "https://files.pythonhosted.org/packages/ac/61/588f6b5397ea4f5bd9fc8de4b8cc092c555b8d95371c03d149b3bc419277/granian-2.7.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb63d64c686799cea850c0c328d21adf75e323991a20be04923afc729432d2b5", size = 7001059, upload-time = "2026-04-23T11:54:27.532Z" }, - { url = "https://files.pythonhosted.org/packages/58/63/2affbcecfe96f940744c2086ea3793935d5f6898207590a579c92fc8588f/granian-2.7.4-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f406648c47569e983f0c58bd0853bac30a2bcdc6227428255ee5cc65a8ee62b6", size = 6255487, upload-time = "2026-04-23T11:54:29.397Z" }, - { url = "https://files.pythonhosted.org/packages/87/ac/31f7155a467020e7640e91af15ca3a70b0e7da210de42e3d3344e5eba8d0/granian-2.7.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd56306eed06e293f4848c5ea997e1d019d1ad13b8252dde1f0bc773aca85ef", size = 6875068, upload-time = "2026-04-23T11:54:31.128Z" }, - { url = "https://files.pythonhosted.org/packages/99/22/402cc903e5c4e82bd363177392d4e1dcab8b27c1f7006c5316c37c597056/granian-2.7.4-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:732639e612e6b6e8d481f399f367e8c9bbb6f0e1b7b0aa74db340c574ee3dd98", size = 6982487, upload-time = "2026-04-23T11:54:32.704Z" }, - { url = "https://files.pythonhosted.org/packages/d3/92/3878f977bda82fc3a66fc7e95a54366a7b82edd53e6c9fdb3ec053693280/granian-2.7.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:47b8fdbfb369d52bb3fb884514a6a3a7e4d8e81c65fd26e5232985f2b46ebe0f", size = 6990683, upload-time = "2026-04-23T11:54:34.301Z" }, - { url = "https://files.pythonhosted.org/packages/ce/b3/a1239f3bc4e9034e07cb32403e6a6d26db01bba1c244dd654f6a76bf2612/granian-2.7.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:b679086082bfd7c1aa8c248ef673b715616a4ce58eec6fbeef8b83b30ac84283", size = 7148570, upload-time = "2026-04-23T11:54:36.494Z" }, - { url = "https://files.pythonhosted.org/packages/89/3c/fef781ea7356b21f671615dd0d53adc00fad81031a9ea506f80d1f46a43d/granian-2.7.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:a29191e949a99ffae2807abb7a864f7493f7a744e4fe2ddd2b5cd8db9b71378d", size = 7006976, upload-time = "2026-04-23T11:54:38.135Z" }, - { url = "https://files.pythonhosted.org/packages/56/54/ae2979fc45c06fbb37f595ee10eb6b138b6056202163b8e274d140d3f87b/granian-2.7.4-cp313-cp313-win_amd64.whl", hash = "sha256:07d26325cc69371ea2dc9d3a9cd0cc851c1c8e3dce40aca90e8c204547b5ba7e", size = 4027044, upload-time = "2026-04-23T11:54:39.957Z" }, - { url = "https://files.pythonhosted.org/packages/21/51/10344430e495bfa128dccc114957b33e712e971f91668788c08fe791df73/granian-2.7.4-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:4e093fe9511387313ad7ec9a76b0c78397cc584ef3dff47d46c336c5aee9cd8d", size = 6249290, upload-time = "2026-04-23T11:54:41.738Z" }, - { url = "https://files.pythonhosted.org/packages/ec/46/c7eda2e71a89a13e174598649f721c63ed3d908c0904b62621e8a433af0f/granian-2.7.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:227889f821526b8b60c5edf31b01fc987c4193bb0fc198c0998e0841e0cb719c", size = 5901799, upload-time = "2026-04-23T11:54:43.708Z" }, - { url = "https://files.pythonhosted.org/packages/72/d8/79e51f9f794389a9d6cab3d7c6b834b87d65fba72a43784eb5d2664a57a6/granian-2.7.4-cp313-cp313t-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2b28d4aec5a9f2758a48da1897649a01b70ee1c00f2c4649db574527a3d00943", size = 6037594, upload-time = "2026-04-23T11:54:45.595Z" }, - { url = "https://files.pythonhosted.org/packages/ce/d8/835873a407279435fa0c8e8ac52392d3ba5c9a652bb15c0036aa07d9c302/granian-2.7.4-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f708fea5024a40e0dfba1c17c1c4b09e02e00ac0ac9ac1e345b409f0c11b71e5", size = 6966672, upload-time = "2026-04-23T11:54:47.242Z" }, - { url = "https://files.pythonhosted.org/packages/92/5f/21eacdda27c38e4194de5f9bef36c4045058daf6d58533fadb7c54c70573/granian-2.7.4-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7006dfe9852cded794bc60008a168faf4dc2ecc18f1d74b5fde545685b699ec", size = 6563668, upload-time = "2026-04-23T11:54:49.751Z" }, - { url = "https://files.pythonhosted.org/packages/bd/06/9b19956d75277df44ee380e873a86b9890c431f2e2bcde32b3ba341f0efa/granian-2.7.4-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:77103af44034e30505fb5577b8214b0ad39cd6cbdc854ff980d4755faf93adaa", size = 6664285, upload-time = "2026-04-23T11:54:51.502Z" }, - { url = "https://files.pythonhosted.org/packages/85/33/740e0c9478be49c0778c4ea1773357680980e10e84b59bc19664033996dc/granian-2.7.4-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:b23194e1e0652297086224212605edb4998442511637e732d6009506277f8ff9", size = 6820367, upload-time = "2026-04-23T11:54:53.506Z" }, - { url = "https://files.pythonhosted.org/packages/c2/ad/3453fc1212268a01fee957122f2b1699af0efe50eca07ac570e11d1be12b/granian-2.7.4-cp313-cp313t-musllinux_1_1_armv7l.whl", hash = "sha256:f62941a4ffa1f1c2c5750cfc0b0ad96aa85d63b016125289779eef8888f5340d", size = 7132366, upload-time = "2026-04-23T11:54:55.123Z" }, - { url = "https://files.pythonhosted.org/packages/6c/ca/8479e4d2a02f210ce68b5dc73c77953ec1dfd3769bf725d06e6ec420d502/granian-2.7.4-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:ea6f97d2ade676f1bf49b79088fa4b5640b8b9804b7470218486df3d4be50046", size = 6842094, upload-time = "2026-04-23T11:54:56.665Z" }, - { url = "https://files.pythonhosted.org/packages/0d/96/71f95c73220726aee3e908b3ad2745c4c44fbfba508cb5ed615a9d4d367f/granian-2.7.4-cp313-cp313t-win_amd64.whl", hash = "sha256:759140ceef02ef72e57a184461927d72bcc2ddd3664c3cbbf4def7516f818041", size = 3974523, upload-time = "2026-04-23T11:54:58.541Z" }, - { url = "https://files.pythonhosted.org/packages/98/5d/a0c3d8778cd8aa68131974d34c439a38a00a32953e71e3b549759a5e3cdb/granian-2.7.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c19ebe797d7383cbb3497c599b8201af71f9fff6b18deaf9965d106f61588ab8", size = 6322736, upload-time = "2026-04-23T11:55:00.292Z" }, - { url = "https://files.pythonhosted.org/packages/5e/99/211da053030574f2402c750f3e3e5dc587f5192eac4888affe6ca8894a9f/granian-2.7.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4cee0bdba9179537669c2fa0afab2ce89327a372f1b2a82f280798da321c996c", size = 6052103, upload-time = "2026-04-23T11:55:02.797Z" }, - { url = "https://files.pythonhosted.org/packages/ab/9d/23ec1fd519a4c0db961b05d1821869ed6371cbaf8b3d3a0a85c04f89e6ca/granian-2.7.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a4bc5b54845bfb5f87537483f25c8f8e6003c3c1b4b0eadf6b93a432d0604265", size = 7000868, upload-time = "2026-04-23T11:55:04.826Z" }, - { url = "https://files.pythonhosted.org/packages/98/35/b8798c98c90d3293d9c85580ea6021f148d5ab73ab99d1f82a0e66f73131/granian-2.7.4-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b550fb98b89465c8192b6e506993de6bfb956838e715ffb58e944aec1afdae99", size = 6257266, upload-time = "2026-04-23T11:55:06.962Z" }, - { url = "https://files.pythonhosted.org/packages/6b/4f/5574db17193d90a5831120a0ce2a2dc64a711110ccb9af5a3630260c3597/granian-2.7.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7100a6a6d3835fec2a207fef536a259dd42d9efdb5c46933cf6f9d55d5bfaad", size = 6849667, upload-time = "2026-04-23T11:55:08.862Z" }, - { url = "https://files.pythonhosted.org/packages/66/a7/90b85cc6a31cbee772fc8ee731479429a64169e389444a5fdd685d44a342/granian-2.7.4-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:034ac1bfe8c19b5a7916d35a1ca426845db9ac11215f1b367566aec3b6530549", size = 6902612, upload-time = "2026-04-23T11:55:10.888Z" }, - { url = "https://files.pythonhosted.org/packages/06/6c/ba203ca40bd406db0412bca70281e44712f941bc6aafb59a628f4811d517/granian-2.7.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:baf1c390a25d3d9840204c39e7b801c909e99e896ae2713d898c46b563cbf962", size = 6927025, upload-time = "2026-04-23T11:55:12.663Z" }, - { url = "https://files.pythonhosted.org/packages/ee/52/77e2abfba54523943eea275ebbe733a6d186fe646304fe25f6d22b243d03/granian-2.7.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:3bb99778ae05c1118cd694717d025cc0b85f5ee81f60cbcb2a8783692798db96", size = 7146800, upload-time = "2026-04-23T11:55:14.459Z" }, - { url = "https://files.pythonhosted.org/packages/1d/66/7209201856b7de8d3c643ba87e11272c4d651c216d05ea3fcbdce0da4ab0/granian-2.7.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:13f0a39872afa81c6aaa8e29832371fd831373140f1f04de459ff862824f488b", size = 6999983, upload-time = "2026-04-23T11:55:16.236Z" }, - { url = "https://files.pythonhosted.org/packages/c3/45/bd1e521284714615996dcee48dad47d8b97ca2767a7e7cccd392f25fc176/granian-2.7.4-cp314-cp314-win_amd64.whl", hash = "sha256:97b5aeec98a9c6c0695bf8f068bd03aca83fc17c0d977a9c3a2e57bb5f10d47e", size = 3989433, upload-time = "2026-04-23T11:55:17.774Z" }, - { url = "https://files.pythonhosted.org/packages/45/a2/609f8f0dca7f596b5fb6e57b988b4b8f4d6579724b2720933c379d43301a/granian-2.7.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:a7b1aca6c654f0e61c9e493dd6d3ddb1698f47dc33ed04566a6635948b081b64", size = 6251034, upload-time = "2026-04-23T11:55:19.29Z" }, - { url = "https://files.pythonhosted.org/packages/4c/f5/2eefa8ff477cce7b119ed2fe97fc1f3b2d108397d4755e83a5198149f2c8/granian-2.7.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d4e0c8cc6850dec7180a26b6805b2c4cdbac4c1c48077fd7857a3cd8ff342d9d", size = 5912772, upload-time = "2026-04-23T11:55:21.581Z" }, - { url = "https://files.pythonhosted.org/packages/ae/40/9a5070badaed4ceecf4082855985840c320f7232b8c1ddc93e1732c63265/granian-2.7.4-cp314-cp314t-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:7e6b1f6e0fe873efa3393ef28803ff699a94254f2a7dc07422cc01d9849e2136", size = 6037318, upload-time = "2026-04-23T11:55:23.855Z" }, - { url = "https://files.pythonhosted.org/packages/95/52/1db412e63425cb12f5ca61877956583c6d12f21657b1a3e47eb3200e9c1b/granian-2.7.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dce110217825cff60f68da83280bc20471b10e004e720fa94b845e01925d8698", size = 6962778, upload-time = "2026-04-23T11:55:26.095Z" }, - { url = "https://files.pythonhosted.org/packages/b4/f2/fcca39f617bf70e29ef903bb7a4d037970c637023484f2112d9ed6882516/granian-2.7.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:058f9a4ebfc7b9c2577569c6ecfd333628d0d045de272afaa65ee9933849778c", size = 6566618, upload-time = "2026-04-23T11:55:28.233Z" }, - { url = "https://files.pythonhosted.org/packages/ee/20/0da1bb552746d74275017e1ffc7fc419dd1a33345f132f6f5a90f9f41142/granian-2.7.4-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:7c05f74fa5b5dcedc9f035a7c10b8afd90a3d941975a370f1e07c3f3095dd883", size = 6670850, upload-time = "2026-04-23T11:55:29.945Z" }, - { url = "https://files.pythonhosted.org/packages/11/2a/d0d9cdb10d2760e2f47bd4600c8eef02e326f8f7e253a80ce4ba384265e6/granian-2.7.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:8b992bbc667e3c74de4ad48ac8d735c7cddf3f709fc2097f7dd230ecc46fd7b3", size = 6824752, upload-time = "2026-04-23T11:55:32.066Z" }, - { url = "https://files.pythonhosted.org/packages/3d/79/0432f92f9df6e54394e4dd1c159c0d4814d255a2d2541fa9a5c187d19152/granian-2.7.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:df05e0f85712b3e90ddf28cb8be358664b1afa8cb8f09978141ca70052dca3a7", size = 7130809, upload-time = "2026-04-23T11:55:33.807Z" }, - { url = "https://files.pythonhosted.org/packages/19/03/11cc0e08f59f03a3cd6a1fe46d7632a0f8690ef945a495b1303140bb7541/granian-2.7.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:dbc620f35b67cf6b03d2b6a24b9b442d1bf52961eaebadb2c3ff214d3d0c8dc4", size = 6845920, upload-time = "2026-04-23T11:55:35.583Z" }, - { url = "https://files.pythonhosted.org/packages/b4/49/bcbaaeec0f68d3d1a3dd1fdd21e4a6963d303ae18027c42b2b53f87d6b89/granian-2.7.4-cp314-cp314t-win_amd64.whl", hash = "sha256:b9df8aead4d71562753788264db23d32db34147bb73294ddd90833bef1f4cf35", size = 3981107, upload-time = "2026-04-23T11:55:37.597Z" }, - { url = "https://files.pythonhosted.org/packages/51/18/577637bb861ab688db8eb5d698ad700133818debd7ae6f58c0574c43f70e/granian-2.7.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ce50300cf876f418ba0545f6e8c56d8c75038fc503add0fd1b58d9a3057d95ea", size = 6363314, upload-time = "2026-04-23T11:55:39.837Z" }, - { url = "https://files.pythonhosted.org/packages/48/41/11a6219baa10270f1a6a2a101cfa372e5d55a46a839a43b49a8d087fac09/granian-2.7.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:eb7f727f14d7d485a5df4078e7cc3038864b4e7c380865968e75e1e51e62457a", size = 6027259, upload-time = "2026-04-23T11:55:42.122Z" }, - { url = "https://files.pythonhosted.org/packages/bf/58/12b19b17fb79ee064a8a77a865a031bb49f4ea813789ad63186458ea02c9/granian-2.7.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abbab303b502a770355c13c93569e6c0c71ccc864ab41b59636720d5a643f6b3", size = 6760497, upload-time = "2026-04-23T11:55:44.116Z" }, - { url = "https://files.pythonhosted.org/packages/45/9f/572711f882423f599707aae577ccdbc1700cf0cc3ceb4e9500e00c6b8d14/granian-2.7.4-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:f0b0423fa33a1afb9730fbfb5700fef4dac16bf7a1b7a2a79d0349739c1b1f44", size = 6843897, upload-time = "2026-04-23T11:55:46.322Z" }, - { url = "https://files.pythonhosted.org/packages/31/f9/75d51721069a184cd00310c4b0b0d614a6370905c13a096ccee193432ba3/granian-2.7.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:efa0d4fc35ab42562747e4103124e1c4f21afab081c1591de6472174a3416802", size = 6876194, upload-time = "2026-04-23T11:55:48.231Z" }, - { url = "https://files.pythonhosted.org/packages/44/5e/fd81492529bc2b02dafc63c95d03c2c7faa26ac883ccd94aa93b21fc68c3/granian-2.7.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:03b5ce06df095b5db49bd4e976ac8d8419bb0e73dc160613fc3db5e5d5dcd1af", size = 7094066, upload-time = "2026-04-23T11:55:50.328Z" }, - { url = "https://files.pythonhosted.org/packages/c7/04/f2fa35dc2956edb9a5abaabc0840aed92b4121ce27adf684a1c75e3c70ac/granian-2.7.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:9247db25dd66f74766a6a9488f1279c9b40cf422c6d7a04010492fa1aa7c9019", size = 6892580, upload-time = "2026-04-23T11:55:52.084Z" }, - { url = "https://files.pythonhosted.org/packages/72/6f/f45fa86c36fcc34f6e613bb223b10fd36c6acd9f7aa43d4f65d0f1eff4cc/granian-2.7.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:efccd6818a1ac4cba7eededf5e2768f56d4a8c7c93bd5e3a8d7a901510976944", size = 3971242, upload-time = "2026-04-23T11:55:53.834Z" }, + { url = "https://files.pythonhosted.org/packages/9b/82/71f4ecf6fbe2eb7f9f32c2b73527c52ad1213d8d765876cf4a2175caa2eb/granian-2.8.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:98f09b0539e9dcce6ee534232b2d933ba97f27173675ef5a177d01380aa76371", size = 4592856, upload-time = "2026-10-20T11:41:32.556Z" }, + { url = "https://files.pythonhosted.org/packages/45/27/56b851450114a64e492f2a5ee3cbfcabbaaace58ccba00ab0e7dc101c4ff/granian-2.8.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cf900618c67163d7c881ed2cbc995fabd315a72618fd334d8101276fc50a5f68", size = 4311235, upload-time = "2026-10-20T11:41:34.606Z" }, + { url = "https://files.pythonhosted.org/packages/66/f1/64c4cb8de1aec462d99e9bb0f87709eee3a5c18951845d2a0d9d96f25d03/granian-2.8.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:88245684c009a66ab23c73ae1308763fc5e21a703cab75d4f9ee05ce1938836e", size = 5078378, upload-time = "2026-10-20T11:41:35.881Z" }, + { url = "https://files.pythonhosted.org/packages/e3/d9/8445f7a40c6a245fe03654f9f498ca63a987352517fa96e854684a88e890/granian-2.8.5-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d5c84021e3f548bae8b3efdc27935a374f5348fffb1f0b478ae62a9b533d2aed", size = 4592000, upload-time = "2026-10-20T11:41:37.161Z" }, + { url = "https://files.pythonhosted.org/packages/46/bf/7e6a90f84363700f18e347be11edf642bc2ac8c839bc965b683a62abffb8/granian-2.8.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46c54bc18a182cd57319076ef96d89c3be13605aaeafda1ecad999bd0449d201", size = 5047109, upload-time = "2026-10-20T11:41:38.627Z" }, + { url = "https://files.pythonhosted.org/packages/29/e1/656503580d5ca6f12872cd53ee452f9f222fb7a65468aaa39d7e3bad1602/granian-2.8.5-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:62f0d98695efa53b0f90ae211eff7c485009777378ad33af1ba46421173c239a", size = 5093769, upload-time = "2026-10-20T11:41:39.937Z" }, + { url = "https://files.pythonhosted.org/packages/99/92/40cf612d8ca16434ba3370a9a093d5a4d5040b52c0232ca960b0a719899b/granian-2.8.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:55b19896b5ddfff20a48bc07314d3f1136a04401e9e1b2a13dc5e35797f77f1a", size = 5125540, upload-time = "2026-10-20T11:41:41.565Z" }, + { url = "https://files.pythonhosted.org/packages/bd/16/50ecc5a575c38fb2d0bc2ffd2c35cd90dced4f0595b466782147788e80ea/granian-2.8.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:6baaace6a81a2068872bb60e550b9d9119c1143db7534d9cad8aa3e64d9c3059", size = 5136772, upload-time = "2026-10-20T11:41:42.992Z" }, + { url = "https://files.pythonhosted.org/packages/fc/08/6241869f4dcf7fed403331803fe0851b73b04da17f00bb18f461d4b39193/granian-2.8.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e531e9c22e527c9c8cfbf188fc446269011bb15370ec033b4f3b6decd52e215f", size = 5139950, upload-time = "2026-10-20T11:41:44.467Z" }, + { url = "https://files.pythonhosted.org/packages/e0/f2/8d9241cf97c6487330dc313dd60016ea199ca4056ce79cb8064ae050eae2/granian-2.8.5-cp310-cp310-win_amd64.whl", hash = "sha256:b54346f0e8a83b6d6b2ff23160fcbac17329540c45c3ebd1f968966d70ff3cd8", size = 3243248, upload-time = "2026-10-20T11:41:46.15Z" }, + { url = "https://files.pythonhosted.org/packages/0f/1d/c5eb0aa2ffcf6d18c34e2a0f77bc731b1f5c1159357315be036c5ac1eb30/granian-2.8.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3203ffa641bd6f16b2951afeec626178880b702d04e803a16dbbc38c1b25f859", size = 4602986, upload-time = "2026-10-20T11:41:47.423Z" }, + { url = "https://files.pythonhosted.org/packages/76/0d/731274bc814123f0ad1fac977202b65d2a703819a0b37c54315a4b7b78cd/granian-2.8.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dac7fe7e8263000f74387e07e3a178279d971039bd2b6d029129798ef9e2f59", size = 4311541, upload-time = "2026-10-20T11:41:48.658Z" }, + { url = "https://files.pythonhosted.org/packages/27/4f/d66f6701a770dedffe845581b9f03ff7ad7713a7ba5c38ad527fbdafbba4/granian-2.8.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:23d77fd0fe124fce9829ddbaa5b73c357580634331ca294f57bf37e9b8cfe834", size = 5151321, upload-time = "2026-10-20T11:41:49.913Z" }, + { url = "https://files.pythonhosted.org/packages/80/7c/67d0578d42bf97ecba89510a384da94dbdc147550d9f06e814ad14b08593/granian-2.8.5-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8435d25cf9427e954d6133dab0a07ff5fecab38e243ba28172ffcda3e7c2664", size = 4661179, upload-time = "2026-10-20T11:41:51.147Z" }, + { url = "https://files.pythonhosted.org/packages/21/09/0a59603a3733ab46bef1df03538813de11a510bfd97a324d5072c8370563/granian-2.8.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85fcdbdf012cdea0eb628b2858dac2e9137272c72d16b52d25b3cf014450f8c8", size = 5032423, upload-time = "2026-10-20T11:41:52.414Z" }, + { url = "https://files.pythonhosted.org/packages/50/89/a5883d04b5dad8f01fcb66803cadcf6fe900d1150cd715f5b08742e2e952/granian-2.8.5-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d733a133b381abfd4ab5035346071c81480e9ff2abedcb4375f10ce1c7c68794", size = 5052813, upload-time = "2026-10-20T11:41:53.829Z" }, + { url = "https://files.pythonhosted.org/packages/57/67/5151a391bc39fb2628e995332480e7efa5d83db69b499cb0aaac6832cf55/granian-2.8.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d9438061a319f5f621ba9c963baa760f62f907d95c21cdf10cda6434e75879f5", size = 5084084, upload-time = "2026-10-20T11:41:55.228Z" }, + { url = "https://files.pythonhosted.org/packages/20/ee/2de690e2dc0c6978f6d392f04c3c4d74c04fa7e780c669bce2332b868a4a/granian-2.8.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:ce17c1f040b8595ca65d6638a920590c8511e5c27365c7222653d0154480636e", size = 5204812, upload-time = "2026-10-20T11:41:57.051Z" }, + { url = "https://files.pythonhosted.org/packages/11/0c/009b9fc714dfb4d0402b8ed36bad04a2567d4b77cbba56712549550a335d/granian-2.8.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:06fa5fbbe2b1dcc66a3384bbe1d156730ff0d6128173f9fa35b999d9b6eca500", size = 5123155, upload-time = "2026-10-20T11:41:58.531Z" }, + { url = "https://files.pythonhosted.org/packages/dc/34/65d14a23c9f3d84228862451d3d84aaf5c06289fa5628b18dfd42616bbce/granian-2.8.5-cp311-cp311-win_amd64.whl", hash = "sha256:84db0de5ffd800bf912405daa08092ec19bab5aa05e1cc65236fc721c0aa0fb3", size = 3231999, upload-time = "2026-10-20T11:41:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/27/91/103c04ea8ef7d9d40cdecb96a07cae09bfe9e3857d39b8174fc5c1a771fd/granian-2.8.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3404a620364f3816f126863910e6dc86da833298608896e556c01c9bf52a63c2", size = 4600588, upload-time = "2026-10-20T11:42:01.027Z" }, + { url = "https://files.pythonhosted.org/packages/df/f2/52735de5caa6db560bfe91b60c97f5a1bf57a035ccd08004f21f18762830/granian-2.8.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:dcb0ecee7650794738fa347ce1aac5a7f790bf36dc0dace4194578335ad3fa60", size = 4310695, upload-time = "2026-10-20T11:42:02.305Z" }, + { url = "https://files.pythonhosted.org/packages/60/e3/274741428e960bf59b12c51cdca2340698e708f3bab4ec6fa16b0765d12a/granian-2.8.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29a5e1459736acd617631524db3c430df7b7fe2fd55b6f21536eb9d30a603d41", size = 5156401, upload-time = "2026-10-20T11:42:03.738Z" }, + { url = "https://files.pythonhosted.org/packages/77/53/c9c16f814597303e4373eb3cdfc8b0ca75582f6939a1a84b9109c98cbfeb/granian-2.8.5-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:217c38e88aa324ef73c9c5d823434b5c5ee7a7181b36c23b408d439905ece7a9", size = 4658203, upload-time = "2026-10-20T11:42:05.252Z" }, + { url = "https://files.pythonhosted.org/packages/26/7e/9a4aebcc1fe18d849b79cbf2aded21e8d6023436b98a53b1514955c5470a/granian-2.8.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66f0de2cb17109da7ca070a2898e4543083914c2c440ba4a134a53cf4ec526be", size = 5067921, upload-time = "2026-10-20T11:42:06.555Z" }, + { url = "https://files.pythonhosted.org/packages/1f/4b/d7bb8f98d739e62379d88131654ee68de441ccc5d6118f578784689257fe/granian-2.8.5-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:95bae2039b53bc532386a0b1327b52e73c16b3e11997a727024f9bd385242a2b", size = 5117522, upload-time = "2026-10-20T11:42:07.749Z" }, + { url = "https://files.pythonhosted.org/packages/59/3e/2c494c6afa6c4ae564c05d0611ce7ee108824812a74bb20e024e752a8f2c/granian-2.8.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1c72079469aba9856d4d030b21ff75a5c2d9a201cab930926237ab1646e5596f", size = 5141659, upload-time = "2026-10-20T11:42:08.96Z" }, + { url = "https://files.pythonhosted.org/packages/5a/16/047f981331408110441c5adbdf9a21acd34846e13ee6546e77c7359b760a/granian-2.8.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d263fe13e16a612a0149e6948f556e5ed8fb4fdec9890812743d116025fb442", size = 5210772, upload-time = "2026-10-20T11:42:10.818Z" }, + { url = "https://files.pythonhosted.org/packages/3f/21/40f73b96a0fe0bcdea7a65ede042a029a10c1490a513fbf8909087eab02c/granian-2.8.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:19902f90f272c089ecea998214c62f7eae846d454a9a82458fcb77b41adddd29", size = 5157211, upload-time = "2026-10-20T11:42:12.484Z" }, + { url = "https://files.pythonhosted.org/packages/14/a5/ac92e100d1f5141115ee796321d8f9d25d8f2c07198e5763b1ce53c457bc/granian-2.8.5-cp312-cp312-win_amd64.whl", hash = "sha256:efc2ae72102983a32cf6cc524a595fae3211937bdeb5bf93767b01dc6798f8ea", size = 3212050, upload-time = "2026-10-20T11:42:13.894Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9e/c831afbdc6306b4c05ea89619b3e51d3350d499ab3a3078f42b3e1481a6f/granian-2.8.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:c30bf8ad2a9547b8efe4cffa67adef5d3f065cac7b37ff7ca6f7df12ed07bfe8", size = 4600095, upload-time = "2026-10-20T11:42:15.316Z" }, + { url = "https://files.pythonhosted.org/packages/29/d0/66c795135b0dc79f0a1e14fdbfc9c1d01d88ef7ceafa651185ef6f9d3e87/granian-2.8.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4ea263ab8c64434080269a49dc706a1841f7939ceaf1ddec0f204edbab2d83fc", size = 4305333, upload-time = "2026-10-20T11:42:16.815Z" }, + { url = "https://files.pythonhosted.org/packages/1f/3d/1b32e5d497d413903c5c58dded3751b8588ea8282d4c61ca57d694408fb9/granian-2.8.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:048feae9762a044357ed7e31b9117f646ca985b5cd57383120f93b2d3da5bb63", size = 5114313, upload-time = "2026-10-20T11:42:18.291Z" }, + { url = "https://files.pythonhosted.org/packages/0d/94/063998b729003dab2a7a793a3deb4fec23581aa0a51f4a5e492a5fab194b/granian-2.8.5-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87eca4dbf606eb4a91f1c01f7cf7517c8290343cc083274a9e6d24d7b20577c1", size = 4599262, upload-time = "2026-10-20T11:42:19.529Z" }, + { url = "https://files.pythonhosted.org/packages/2c/00/c3db2af29473d37252a1629f9512c45479da3f4efe542a8ec07ebed24108/granian-2.8.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf44d5c35995f6009307d43c6b7c8310299f07f415d6cbecbacc5fdf8508c73f", size = 5065166, upload-time = "2026-10-20T11:42:20.995Z" }, + { url = "https://files.pythonhosted.org/packages/64/26/05d2bb66e8a6c07568a6ae334b7a5e9c674462ddc88cdcb9fc5a14e2b5cf/granian-2.8.5-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:30c85664d3a51397c97413f5c341e7031ea08614c016d9bfb2ee82bbd6029c12", size = 5114096, upload-time = "2026-10-20T11:42:22.261Z" }, + { url = "https://files.pythonhosted.org/packages/9a/ca/41e56be77bcd77b4bab356302dc6b7755227ab33645185131ae3a40b1331/granian-2.8.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:1be90ad793a14e367c3cc8a21aa82078b4a6e84df95700e3fd76f4207e558b3f", size = 5141240, upload-time = "2026-10-20T11:42:23.492Z" }, + { url = "https://files.pythonhosted.org/packages/ab/a8/cd89ab5a87ff71a3f6867fa9ec931a27810bb8e7da6b24d41288476a8f70/granian-2.8.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:9c84d73e53c774d8600d95c9eda1904ac72c3ce761407f4b41af20ce9a0b2859", size = 5172814, upload-time = "2026-10-20T11:42:24.719Z" }, + { url = "https://files.pythonhosted.org/packages/26/56/baf53317b674e5a8b8aeb1d188be42dae67982301da140b6214b448d2f7d/granian-2.8.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:f66fd2c88fc3f9bed832df754dc4c05d685d5254fec54f77abd813ac28433b2e", size = 5155624, upload-time = "2026-10-20T11:42:26.048Z" }, + { url = "https://files.pythonhosted.org/packages/9b/4b/d3fab8903e19053a1a877d12f61c9574f4eb22657ca8d16fa8dc8d66621f/granian-2.8.5-cp313-cp313-win_amd64.whl", hash = "sha256:34f5eba9ee539a45f83cfeeda5d21b84da165db1482c7be2525bca689eb37871", size = 3240628, upload-time = "2026-10-20T11:42:27.321Z" }, + { url = "https://files.pythonhosted.org/packages/94/6c/44290c79496b69cdc68c1438428cd8207de00a8b14bbf45aca59273f1b7d/granian-2.8.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:6a2de1f78f4b54193f37f6c89562b9fd372dc68812e0f4aa82499770631ef6a3", size = 4575722, upload-time = "2026-10-20T11:42:28.764Z" }, + { url = "https://files.pythonhosted.org/packages/09/09/5b1e9d5ee47ca8e248ed4d8d41f0290a312c1c2d9415f558ae0106a6691a/granian-2.8.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0ba7323572d6f0b6cc1a86fac8f6bc42d467fe0034c85429252e93b111c79a1f", size = 4292154, upload-time = "2026-10-20T11:42:30.142Z" }, + { url = "https://files.pythonhosted.org/packages/2c/75/8e8ab830eb7d3bbeb5147bdd5e05e7f307f46950647e0277973811ce511c/granian-2.8.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cdf207b3942b0c131f98cd01769b2e1e8f302891b220b39140c8d4a325d3af5d", size = 5111113, upload-time = "2026-10-20T11:42:31.924Z" }, + { url = "https://files.pythonhosted.org/packages/77/b4/1b3c164cdb8c16c130fa483ab15b18c4f068406e901b1bf30725ab2dca3b/granian-2.8.5-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a25ee45ab230a9983c6ba6401b1dbc7b3ee7de48793c52712c8ffb9662baefce", size = 4598387, upload-time = "2026-10-20T11:42:33.386Z" }, + { url = "https://files.pythonhosted.org/packages/c4/65/70d500aaf815be539d2761192b499fc7514a625eb92b1f55c4639db7d75c/granian-2.8.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8162358380d4b9398a2176779cefe553921fd5e433ccad7fe6d371e0845fe82", size = 5043379, upload-time = "2026-10-20T11:42:34.784Z" }, + { url = "https://files.pythonhosted.org/packages/1f/b0/863f62924357de8b486f22a7a5b11e7bda2dbebcc44ce2950af210f3d57b/granian-2.8.5-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:091fa616adfbd83460433a99df0858ff3e831e937e478d1e8170a0fb71c61431", size = 5104873, upload-time = "2026-10-20T11:42:36.078Z" }, + { url = "https://files.pythonhosted.org/packages/cc/19/4214986ccad3b6791b5b70ee8ee1545af4a12c8650726025280f69854655/granian-2.8.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:66f2ebe92dabfd986d99cb911c1027bacb35570119e244918e5beeef56dce1e3", size = 5133431, upload-time = "2026-10-20T11:42:37.506Z" }, + { url = "https://files.pythonhosted.org/packages/9d/68/fbeb8760e5854115d0573a998d2239d844f3681781f3d2ab1b732fc05224/granian-2.8.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:519d1461c2ea4086ff5643b8abaa78dadc5a3d0f2ab5ac4de1811f9980c3e153", size = 5168722, upload-time = "2026-10-20T11:42:38.779Z" }, + { url = "https://files.pythonhosted.org/packages/d1/72/063abeff9026aa62aa1412b327cb4b75b161dec5744e60b24eba6c98bbd7/granian-2.8.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:2049b3b0d591c754ac89e313f88e8dd35c4aa741193e39f2329eece4e447c04b", size = 5130161, upload-time = "2026-10-20T11:42:40.216Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/fd9cee9460989df1caa497bfebe4cbe7d98d2de013e8ca92594e8b15d4a0/granian-2.8.5-cp314-cp314-win_amd64.whl", hash = "sha256:14d92199bf255db7ce64a63a63f42d52392f287c097a0427e60436111a7853a7", size = 3233798, upload-time = "2026-10-20T11:42:41.451Z" }, + { url = "https://files.pythonhosted.org/packages/fa/cd/a56c529ae01d5ebf68971a4e741844274d1565ac0d2e05ef8f2cb76ff878/granian-2.8.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:9ae77ceee1148982c9d3f7ded52961355c81adae206e9621fe27c9a7c426a260", size = 4495468, upload-time = "2026-10-20T11:42:42.809Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ff/6c86c7cc796c440c383300aaf3c95736e5663a756a8164b4faa6ed0be2a9/granian-2.8.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3ab0046f99ccbb39198ecadac6df163ff312886b93630e1ca319a1fafecf6af4", size = 4176464, upload-time = "2026-10-20T11:42:44.035Z" }, + { url = "https://files.pythonhosted.org/packages/1f/d4/6211c9839cf93b675221d388e4eb14e1eb1f3826f694b9d4b872d2a5f0af/granian-2.8.5-cp314-cp314t-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:83ece3459a3652a6389ee663efdb39b5c5c27ebf349efdff0cbe4c6dcb7cbc9c", size = 4337940, upload-time = "2026-10-20T11:42:45.577Z" }, + { url = "https://files.pythonhosted.org/packages/05/fa/a4061b90996d5b6d8868d71b40a75f5d52618e4c8f59b5adf8d40bf420d9/granian-2.8.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2fd8a7956acfbd8b223e39103d63f80e6106d30fc9dc2f913eb3a638bc5c57", size = 5011790, upload-time = "2026-10-20T11:42:47.238Z" }, + { url = "https://files.pythonhosted.org/packages/27/b8/63070894c66028a35820d4d0c7a7ab976f74cc2c0e00cdbd1f60f79026ac/granian-2.8.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e67a0fbdca1f7e923d1627b584a1f0dab66b7a18227acfd583ecdaeeff67fa4e", size = 4854816, upload-time = "2026-10-20T11:42:48.526Z" }, + { url = "https://files.pythonhosted.org/packages/0e/d9/2f6e76dff34f81eaa4c45de9fbc86f5468f5a88e1224250c506a338bbf2a/granian-2.8.5-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:8efb3ab7e55f48c83695cc6cb196f56bc54874c45d2229e21c859135a4837788", size = 4884836, upload-time = "2026-10-20T11:42:50.035Z" }, + { url = "https://files.pythonhosted.org/packages/06/5c/a9459e7561b233420baec9dde2bdd1f76f9c1e026ebd51570b8f99013a6c/granian-2.8.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:91bdc54ddee28d5b809533937a086a2e1dc78e0b746b7d925785863036d815c6", size = 5082247, upload-time = "2026-10-20T11:42:51.343Z" }, + { url = "https://files.pythonhosted.org/packages/75/29/afa188c5f28b0d9ec09eda9016fa99e2a44ecd9ea399b3c3fa67f399300b/granian-2.8.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:9f9fbcfb98e19f649f70e12c432609c4d03eaf61e14b6e2093abde7042f9c533", size = 5053807, upload-time = "2026-10-20T11:42:52.739Z" }, + { url = "https://files.pythonhosted.org/packages/94/ff/b36c8988e56a2b3e7846fc63e5c93a15408d96786807427533fed87c49e6/granian-2.8.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:6c961cb4b17dc58bafa8a0c02a7be988c81f69fce76b5ecfe20c656fda0ede22", size = 5095841, upload-time = "2026-10-20T11:42:54.325Z" }, + { url = "https://files.pythonhosted.org/packages/3d/9d/c25f5eaa94f5e57238c23686531833bbd5c9dc2896068f8b95f8faf860e8/granian-2.8.5-cp314-cp314t-win_amd64.whl", hash = "sha256:3012b24c51882581c6717f77f907e9ea2e8178a9f5170a4eaabcb1b67a059adf", size = 3166239, upload-time = "2026-10-20T11:42:55.75Z" }, + { url = "https://files.pythonhosted.org/packages/98/7e/7fb5e711fa2e973691175cd7629ad3ad2268359489cc00c928ee7326ce30/granian-2.8.5-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:4d7d369665b628aca1e9aead5942e017f796ddf1d894aae775b61430a76b8c34", size = 4575846, upload-time = "2026-10-20T11:42:57.195Z" }, + { url = "https://files.pythonhosted.org/packages/75/92/30313d9f420871f17937f460356a37161f0749ef4bf00f1e55a206845644/granian-2.8.5-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:5efdd2240bfa3c73b28823b600a536435e668c319b563056169dc2ccc8455c00", size = 4289515, upload-time = "2026-10-20T11:42:58.922Z" }, + { url = "https://files.pythonhosted.org/packages/37/48/faebf1a75105cbb80fb5aaf10b0e847b5a3c95be4d1ae418574e0b9f471c/granian-2.8.5-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4643e7e06e41c5e6fd3f64120351b82f0355cac2fc55753b416d4ccfdcf3b096", size = 5107846, upload-time = "2026-10-20T11:43:00.087Z" }, + { url = "https://files.pythonhosted.org/packages/77/aa/74a3daefc65bae63aa69e83304ebc40b71f0cce697084f57ec08475062b4/granian-2.8.5-cp315-cp315-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f88a2287bd249e5412c48389eb8ef7abd402e224cae91a991c69038c05168a15", size = 4597929, upload-time = "2026-10-20T11:43:01.693Z" }, + { url = "https://files.pythonhosted.org/packages/82/7e/357ae2afd73f26f66afb10d5f56a05964034479202c6dcee0625acf11f0a/granian-2.8.5-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc4dd03483d79831d9e67c3490aa4d68e717bbf68b15fd84d75d05413b341764", size = 5043750, upload-time = "2026-10-20T11:43:02.975Z" }, + { url = "https://files.pythonhosted.org/packages/d1/ae/5a2323c84244954ea1f57cff8839b21dc160fc8867595ad02d14047c4e14/granian-2.8.5-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:1eefe87b1d4294e6c5f0876f5f497af6ecca6b1ebd79a4a841cbdc9469981781", size = 5105156, upload-time = "2026-10-20T11:43:04.628Z" }, + { url = "https://files.pythonhosted.org/packages/40/6d/842f6ba44ef4c8eb58487de3b80e712c2863e79053272ad6dc6936b4052f/granian-2.8.5-cp315-cp315-musllinux_1_1_aarch64.whl", hash = "sha256:bc75eabd606b76dbe31f2641d006152b27e8f317336faf18eb5ebe68e408b23f", size = 5135072, upload-time = "2026-10-20T11:43:05.938Z" }, + { url = "https://files.pythonhosted.org/packages/df/4c/413a653dff737322f37aa58bff198624a19a10bd949d269ae3851c98e296/granian-2.8.5-cp315-cp315-musllinux_1_1_armv7l.whl", hash = "sha256:9db4955fdddcbf975cb6f5aa97862679647ead741ecdd08963b5fc8b8d4a3ec5", size = 5170126, upload-time = "2026-10-20T11:43:07.332Z" }, + { url = "https://files.pythonhosted.org/packages/65/2a/d2faa15f0e8dfce88abaa607946219dad89ace7c0c0654e77157db65ed24/granian-2.8.5-cp315-cp315-musllinux_1_1_x86_64.whl", hash = "sha256:cf3ecf3a3244b9131ce0a847e50fa404b340a99b0904047d993eef0ed4cc5f4f", size = 5130094, upload-time = "2026-10-20T11:43:08.636Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f4/79b63eabb67cd36b43b98fa6d9a1a8ee95b8a01eadab64fbf9cfdfabd148/granian-2.8.5-cp315-cp315-win_amd64.whl", hash = "sha256:e23bcb230aaf0c15d8e8720f7950a7d4048d73e50032c17e5db6076af5dbbc69", size = 3233682, upload-time = "2026-10-20T11:43:10.701Z" }, + { url = "https://files.pythonhosted.org/packages/ee/35/6cb05558f4c3ceb75e8e5902450aa6ad40b11241684ef47cc9e2483584d0/granian-2.8.5-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:08b00072451bfc29cdd0dfef1066dbb8f9e9ca9d260b223f82c79d1168a194eb", size = 4494355, upload-time = "2026-10-20T11:43:12.011Z" }, + { url = "https://files.pythonhosted.org/packages/04/43/24df4c606be4d15dfca92a5ce3ee4b8f4fcf5327519af4d3c403a5b2fe4d/granian-2.8.5-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:b18d114a56029d464ff2f782387b906b180177462d9d2254f4f73dda829c4ad3", size = 4176714, upload-time = "2026-10-20T11:43:13.843Z" }, + { url = "https://files.pythonhosted.org/packages/99/ef/7ad359c036d88ae4801360bf5350e9976a9c662e2479964b96ba182463bf/granian-2.8.5-cp315-cp315t-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ea673c5c98fd182138ed21d0a05528a11bd469e201f557a7b59bb8062b5b37d6", size = 4340682, upload-time = "2026-10-20T11:43:15.085Z" }, + { url = "https://files.pythonhosted.org/packages/4e/2c/e23041ec7f44022fe9e5ffdc927008de62bc83c33eb8d3f3a6151a15a8bc/granian-2.8.5-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7aa3d26e71794cb7767b0909d92d76736b14aa0fdd451b94b6a0d4d0c875d403", size = 5012489, upload-time = "2026-10-20T11:43:16.573Z" }, + { url = "https://files.pythonhosted.org/packages/ce/95/7b9ba12477bd6c8fd34261c9586f303a9dc98928a15d8b05a0e50ced47a2/granian-2.8.5-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:62fe53469efaf497f84a822537470fe8702df8125b0247ee38415e4684ee1c91", size = 4854972, upload-time = "2026-10-20T11:43:17.962Z" }, + { url = "https://files.pythonhosted.org/packages/da/f4/b4d469cfdb0ca72e8990e4dd3183e668904f5c2e42cbda911c95bc502eb0/granian-2.8.5-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:fd39d21c95d49dd5be07b6e3031acef2590a0ef41218a38fca489700c7e05847", size = 4884122, upload-time = "2026-10-20T11:43:19.214Z" }, + { url = "https://files.pythonhosted.org/packages/55/c8/14db3b06e52ddf74b8b50e70b0ce7d8929aed24a828d35fc8bba7cf4cae3/granian-2.8.5-cp315-cp315t-musllinux_1_1_aarch64.whl", hash = "sha256:deb8d3bfb1ebefd8eaabbb2b25bb533f74a3e453bcaa419177e59d39682710e3", size = 5083706, upload-time = "2026-10-20T11:43:20.575Z" }, + { url = "https://files.pythonhosted.org/packages/d6/9e/4fcf222551e36358c66c6b971e0b2149ba8a0adb5ef99d0168033b84505d/granian-2.8.5-cp315-cp315t-musllinux_1_1_armv7l.whl", hash = "sha256:58713e4fd8b3ee5d51b5d1aacc2f800b6922dd5f19cd2609dbbcc47336bc6b7d", size = 5055522, upload-time = "2026-10-20T11:43:21.77Z" }, + { url = "https://files.pythonhosted.org/packages/c6/5c/6e1b5d5b498fae263145fe719d053810b38d86d754a92454d15a985274d2/granian-2.8.5-cp315-cp315t-musllinux_1_1_x86_64.whl", hash = "sha256:42aa50fcf6798f30bec90f9b5a22d3039be06932423b12ffa80b1ade2328a631", size = 5096223, upload-time = "2026-10-20T11:43:23.238Z" }, + { url = "https://files.pythonhosted.org/packages/72/ba/8f76547c8c52eed6ac2fe2b17e749b7fb8bef9bc23dcfd849897bc7ae7b1/granian-2.8.5-cp315-cp315t-win_amd64.whl", hash = "sha256:1fb135bb48bef82f9dcaef3462c4b49b24a6a423572c65e6b9db4647555b821e", size = 3166220, upload-time = "2026-10-20T11:43:24.509Z" }, + { url = "https://files.pythonhosted.org/packages/89/c9/751496c5332fe7b6d1775d7342f3856ba5a7b32b978c5a083d11a9025df0/granian-2.8.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c7787e67de0aa08cf7ca24fdcb775cd03001d0240212a1294f73bf09356d40ba", size = 4586917, upload-time = "2026-10-20T11:43:25.967Z" }, + { url = "https://files.pythonhosted.org/packages/dc/f1/d419875fe9fac243324dbc949e7b3dac63a70a54bb205f56a2d057e18765/granian-2.8.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:1915e09e4eb6cf3d5134633c5302e6525529b929d549a3861df011d4044f3706", size = 4257626, upload-time = "2026-10-20T11:43:27.845Z" }, + { url = "https://files.pythonhosted.org/packages/4c/4b/520fe7f10a527b72294e6714370943fcd9d003da7c940c4c9695ca36943a/granian-2.8.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41017d36448de63a171c76e9e728bdad0645ca1f8a4b2a62267c61dedf13c19f", size = 5028778, upload-time = "2026-10-20T11:43:29.286Z" }, + { url = "https://files.pythonhosted.org/packages/9a/c7/8c25aef4cde87871414eebd0ab081141af1645f0514a4179ded6d3555082/granian-2.8.5-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:93d8e1ebaa7af1599437a7e7bd627249374a4267ed4aad7ce08feabeea25bdbc", size = 5048930, upload-time = "2026-10-20T11:43:30.824Z" }, + { url = "https://files.pythonhosted.org/packages/d0/23/2ddcc44baabd39c32714cd3b868a1d27c6ae318a2bfac879a8241a8ac083/granian-2.8.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:8c052f5399c31eb5b08d6e67232cae69070341acb5e6d19b5aa0b45bed48f6c6", size = 5075940, upload-time = "2026-10-20T11:43:32.601Z" }, + { url = "https://files.pythonhosted.org/packages/16/24/24ae014a30c5eb98afbf88d75fe066a4cb803379e588247730966c9cdf7a/granian-2.8.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:fa4caf085a082fbc192bafe8f5bce37673b82258b56399520cf7203217eea48a", size = 5165526, upload-time = "2026-10-20T11:43:33.845Z" }, + { url = "https://files.pythonhosted.org/packages/d1/57/958e7f82de6947f95b6248c2ec2cc9979858f5f8a2ba0dd98665dfaa4c10/granian-2.8.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:a51bc4b6a6c7692037bfa2a55969714318bfb72cab1901280829333be21899e7", size = 5123288, upload-time = "2026-10-20T11:43:35.321Z" }, + { url = "https://files.pythonhosted.org/packages/b3/3e/60d2c5b2e696fe59b4922bdb96d69661f073c60eb52519790e378593af5a/granian-2.8.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c6803546881c0a2e4ad21088fabdb8ef0f802c3edf5d1606cbd8b7146b7345bc", size = 3222429, upload-time = "2026-10-20T11:43:36.661Z" }, ] [[package]] @@ -3011,7 +3107,8 @@ dependencies = [ { name = "graphql-core" }, { name = "graphql-relay" }, { name = "python-dateutil" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/f6/bf62ff950c317ed03e77f3f6ddd7e34aaa98fe89d79ebd660c55343d8054/graphene-3.4.3.tar.gz", hash = "sha256:2a3786948ce75fe7e078443d37f609cbe5bb36ad8d6b828740ad3b95ed1a0aaa", size = 44739, upload-time = "2024-11-09T20:44:25.757Z" } wheels = [ @@ -3124,7 +3221,8 @@ name = "grpcio" version = "1.84.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c0/3a/5c8e0eea2af3d30cb20d33ac325a98d9b799c069d63101ca4da20bcc776a/grpcio-1.84.1.tar.gz", hash = "sha256:446e7876b9b991fa124c1d414899fa7cf95f89a3e71237ea0161863dc0b2c09f", size = 13421202, upload-time = "2026-10-22T14:59:43.986Z" } wheels = [ @@ -3178,6 +3276,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/83/c87911b98ed766d7084d64053dc06f7855267f686b673dc0bb1ac8c07aee/grpcio-1.84.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f4b64099afaccdeeb987240af47aa1fc14f57285d3d2ff000c7327228624999a", size = 8088730, upload-time = "2026-10-22T14:59:13.431Z" }, { url = "https://files.pythonhosted.org/packages/88/bf/6b18690790c0c4d7062749785bc8e515881a8a4058f46cb2925c86adc9e0/grpcio-1.84.1-cp314-cp314-win32.whl", hash = "sha256:116b4796da0e7fca3fdf6edf422ea7521c25d8c4be64ab7e3b5e2a77c832b0ed", size = 4527749, upload-time = "2026-10-22T14:59:15.712Z" }, { url = "https://files.pythonhosted.org/packages/b8/d9/ee50fe67481254c45c9b709b31ba468e4c90925018f4507c424369a9abd5/grpcio-1.84.1-cp314-cp314-win_amd64.whl", hash = "sha256:3875f4c4caf0c54ac370b66322f0bf2d46e9d21414be36ea8b7f618badb3b87c", size = 5359432, upload-time = "2026-10-22T14:59:18.331Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c0/fc3341990f75cba2096d07ea5e7a8f78af8cb36f6b1fd78660b8c1220e11/grpcio-1.84.1-cp315-cp315-linux_armv7l.whl", hash = "sha256:775f98cb476625936061b91e3c1da229112b212a6b2a70fefc9c87e5aaf9fba6", size = 6393689, upload-time = "2026-10-22T14:59:20.402Z" }, + { url = "https://files.pythonhosted.org/packages/c9/f4/6b9a70760bbd8e927f31452ed2ba33cc3dc1a327f1b46c44f65ddb6dbf0a/grpcio-1.84.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:cfdc7530e5707ab6e5c80fc624f89e26ddf8f38d3e89e5226dd7e55c080e9b65", size = 12325454, upload-time = "2026-10-22T14:59:22.805Z" }, + { url = "https://files.pythonhosted.org/packages/ea/87/e44f1c73d4c003a10a8c9b348b44296e99ec5623b70fb441ad250f4b532b/grpcio-1.84.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c61a0d5c60506b2371444716b56d6c633d3c56a17c703566bcca39604c02c12c", size = 6984778, upload-time = "2026-10-22T14:59:25.385Z" }, + { url = "https://files.pythonhosted.org/packages/19/9f/efd7828266b9e5ff0764af8420d52add5acfc13382d073596f15d7dfc183/grpcio-1.84.1-cp315-cp315-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:79b3ee4683290b3cfabadc718a008c3474073b585bc450231a842c21c2ef1f27", size = 7073415, upload-time = "2026-10-22T14:59:27.503Z" }, + { url = "https://files.pythonhosted.org/packages/80/5e/de257fc887b5a4e955173fb28a76a39fdb69f12e0d99987ed723c937c419/grpcio-1.84.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8ae49802f09cc272546592b3656207de6f719e50a6a0d1654e613aaba6336ac", size = 7165478, upload-time = "2026-10-22T14:59:30.083Z" }, + { url = "https://files.pythonhosted.org/packages/75/b0/66840129d46b06a5159277206a1f706ff755d725fdd0652e7acf6961a930/grpcio-1.84.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1e25f45414b3a53bfc1fe43b980f9d0cae9e194a62541d5f6bef0846758162ac", size = 7707719, upload-time = "2026-10-22T14:59:32.284Z" }, + { url = "https://files.pythonhosted.org/packages/10/0d/92682af96b05c1d5e5ce1ea82a3969b221c9d58112a097ff637946e395c6/grpcio-1.84.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:de3c31abc0d052511744a817506e03534f14e32dadab8ec19f63f2d4facd05f3", size = 8079074, upload-time = "2026-10-22T14:59:34.507Z" }, + { url = "https://files.pythonhosted.org/packages/0c/69/1e426aa52c23640430eed1728c0f7c1167bb64eb7b0a31bb5ce9a500f73c/grpcio-1.84.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:6ce0835704f30d4954aa0a68352fc9cc30242cc85f39d7b6cd20c0790e05925b", size = 8118408, upload-time = "2026-10-22T14:59:37.082Z" }, + { url = "https://files.pythonhosted.org/packages/72/66/037c29a3668a32f2dbd287ceac43286797ffe14c95e7234db75c42ea10dc/grpcio-1.84.1-cp315-cp315-win32.whl", hash = "sha256:43d8b84b3e2b7505e4902ab6288ceee699cfec5435ed879bb53f876b2966ee33", size = 4527284, upload-time = "2026-10-22T14:59:39.362Z" }, + { url = "https://files.pythonhosted.org/packages/c3/87/4dea740311428c40fa0da674e0ecfa28f572809cba4cbc9c568918c82d2e/grpcio-1.84.1-cp315-cp315-win_amd64.whl", hash = "sha256:c8e1c15e81f65efb317906754fdba4a85d2eac9cff250d8a3fb3ccb3a4e52db9", size = 5360072, upload-time = "2026-10-22T14:59:41.328Z" }, ] [[package]] @@ -3309,6 +3417,13 @@ wheels = [ name = "hiredis" version = "3.4.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.14.*'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", + "python_full_version < '3.11'", +] sdist = { url = "https://files.pythonhosted.org/packages/1f/e2/1654d65851f39fd94e91a77a5655d09d4b64901fdc594020d8348db697b2/hiredis-3.4.0.tar.gz", hash = "sha256:da19331354433af6a2c54c21f2d70ba084933c0d7d2c43578ec5c5b446674ad5", size = 137169, upload-time = "2026-06-03T16:23:46.226Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/4a/48/b0c0e2826eab7543c08980dfab871b3e7c83c47d7496134b04a94df55a1a/hiredis-3.4.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:69d0326f20354ce278cbb86f5ae47cb390e22bb94a66877031038af907c42fa5", size = 138470, upload-time = "2026-06-03T16:21:49.96Z" }, @@ -3397,6 +3512,129 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/43/5f/829287555ce7286be8d6c87c69f93aa1f38fe67c46740806416142231cf3/hiredis-3.4.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7ff29c9f5d3c91fda948c2fde58f457b3244550781d3bc0891b1b9d93c10f47f", size = 37968, upload-time = "2026-06-03T16:23:14.948Z" }, ] +[[package]] +name = "hiredis" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15'", +] +sdist = { url = "https://files.pythonhosted.org/packages/b4/72/64065841e93c3b8b507d7889d7de371e5f397b5efd60f18c2d82d5bd3636/hiredis-3.5.0.tar.gz", hash = "sha256:c227592cc56b7df247abe4755c26723965be68972be2149fa441ac33d82131ba", size = 143423, upload-time = "2026-10-15T18:00:45.176Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/92/d37313b1d536ce6099d5f53a00c2e1b0e59506379a209331df73a7d1abc6/hiredis-3.5.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:2e838fc213f1db0a69b2e15995ab35919fdd4f4108118454169c0b671ef9a5e7", size = 140483, upload-time = "2026-10-15T17:58:03.601Z" }, + { url = "https://files.pythonhosted.org/packages/41/1e/b937b843ed1dd59522f06cbd1522d1680fa1fad43932ecb729eee08c9991/hiredis-3.5.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:63d42b94b9800904249fdd9be9eb37f31677ba400d921d000dabfc5c0ce415cf", size = 74897, upload-time = "2026-10-15T17:58:05.446Z" }, + { url = "https://files.pythonhosted.org/packages/80/29/d0b359be616514831f24e303b2e47f15f2c882bf78b9da921db8bead267a/hiredis-3.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a2c7fbf6f70c8e3e3f6693cd52774f48e149fe2709828643ca0207021852540f", size = 71875, upload-time = "2026-10-15T17:58:06.592Z" }, + { url = "https://files.pythonhosted.org/packages/db/3a/5f9967b6ddd53174a6394fb2c7ae49531dea2ad771368e3b345c8d8c5577/hiredis-3.5.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:926d270b5a2dcd75f91ecbe784e42f515ae8fdfa57bf28031bc6d21accff5e3f", size = 306465, upload-time = "2026-10-15T17:58:07.759Z" }, + { url = "https://files.pythonhosted.org/packages/97/bc/56f5f121b0ed0afc32a8017efc462c27269b41d7d0dc596a7420a0295d9c/hiredis-3.5.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2ccb190ef81bcff7ef9650968782d446a899740fb478ec8f21988ce37d2dff6", size = 339191, upload-time = "2026-10-15T17:58:09.112Z" }, + { url = "https://files.pythonhosted.org/packages/92/3a/6bc12deb33c5494b792b327f071ddf2704c20ffcce02971876f8860bfa3c/hiredis-3.5.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5d7639f8769908176bca3eaaf099f9aa182718b14f364380f5feeb1ff3ea2734", size = 349840, upload-time = "2026-10-15T17:58:10.252Z" }, + { url = "https://files.pythonhosted.org/packages/75/3f/ed12e566e6941d45db1f74ea5b799a4fa6a03d3e166a16a0b91e17251530/hiredis-3.5.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0188f80503cdfb9fc66e29f177e864280aa4e2dec50c7a66cfb29a3f08c129c6", size = 311786, upload-time = "2026-10-15T17:58:11.256Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/f46c0ac51ce5700fd8e7fe228f7c2f71c26946ad43452e3127d93a989316/hiredis-3.5.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a188c39bf426e14b91f71ec9ec54ef1c2377c1cf38a4c2b5d8b506e59f10e14a", size = 300621, upload-time = "2026-10-15T17:58:12.616Z" }, + { url = "https://files.pythonhosted.org/packages/8a/2d/097c42a41ab28e33bbd4a73753ebbd13056413d40f4dfc25d58deb2ee178/hiredis-3.5.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6dd3b5eed6b208a6583ba75c54b01ac82bd547ea29835eee75837ca2348c158a", size = 330943, upload-time = "2026-10-15T17:58:13.757Z" }, + { url = "https://files.pythonhosted.org/packages/9a/93/890c870edec8c74e7f48090397e0e706044603e3425b0fc3b5fa9c4605d7/hiredis-3.5.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:f5d97b54ce858857c12c3b60a89563dc1938fb4837fafa877cd870e05f4a2990", size = 331331, upload-time = "2026-10-15T17:58:14.886Z" }, + { url = "https://files.pythonhosted.org/packages/fd/80/33dc570d42c99c9656d84267517e2d095e51963d054aec02fb761993218d/hiredis-3.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:46259a135ab298fe5197b039afc9d1783a67fb374d95ec096624aeac66ca51ec", size = 310414, upload-time = "2026-10-15T17:58:16.747Z" }, + { url = "https://files.pythonhosted.org/packages/da/f3/3d7356a605e026b442dc4e53e2deb51a16d9f9c4d7eb4d4d27ef3ae361da/hiredis-3.5.0-cp310-cp310-win32.whl", hash = "sha256:9684eb9f45daafc2a9212115c7dbf25426eb4469fe635eec6f7f4493c24d30d3", size = 38744, upload-time = "2026-10-15T17:58:17.781Z" }, + { url = "https://files.pythonhosted.org/packages/60/38/6d2a64bde7d66ef3f553c965590bf7367a4a8e02dc9d937a7031ea8502b0/hiredis-3.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:9dc666937514d4518cc776aa2e079b79e9710536d3d65974c12b5ca257baf14c", size = 40576, upload-time = "2026-10-15T17:58:18.714Z" }, + { url = "https://files.pythonhosted.org/packages/ff/42/9225de321c4733dab591615327d79ccee33e3cf29b47dbc1e7c7178e6bee/hiredis-3.5.0-cp310-cp310-win_arm64.whl", hash = "sha256:b253d6921b3a039a10dff5e3ea2c2e4c5c4f25d88eb5ba47858b6e4fe13ed99f", size = 38016, upload-time = "2026-10-15T17:58:19.625Z" }, + { url = "https://files.pythonhosted.org/packages/f1/cb/5e09c82ad64cb8820c0cdde30f8c74f51cbbde32fa907cf453a83808c967/hiredis-3.5.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:a148b7fb5b6b258fd5f6cdcf0296b6905c9ff43d9607a00860ed14d3bd4a2944", size = 140467, upload-time = "2026-10-15T17:58:20.74Z" }, + { url = "https://files.pythonhosted.org/packages/d4/33/668281eaf59c61e8fd3021fc09ab2293449223cd23833b1be4d128275ae9/hiredis-3.5.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:ead3b37e15075aa2064b9c1c114cbb1b153a3ab0300fd8a69d3cf11a1ec742b0", size = 74892, upload-time = "2026-10-15T17:58:21.958Z" }, + { url = "https://files.pythonhosted.org/packages/bd/e8/5e9de3a6347d980635727e3e9810c419650f6702a849b5f6c35fb3100bea/hiredis-3.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e115862e4e1a22aa78048117f88c21a35291dfbf9ce511359d97bd4abcbb9ebb", size = 71860, upload-time = "2026-10-15T17:58:22.933Z" }, + { url = "https://files.pythonhosted.org/packages/4b/c4/ec550e62cd7cce8512f372e3c03fbce0f4b07f38d69b690a6b48d0b0e8de/hiredis-3.5.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5b5f0e4114cf972d24ad651537d1b2e20449d6db006c6f09eeac58436886960", size = 306993, upload-time = "2026-10-15T17:58:24.347Z" }, + { url = "https://files.pythonhosted.org/packages/0c/7b/89272dbab1804a56e5cb8633ecdf639b0d6d826b45ab5359cc6662ab4809/hiredis-3.5.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0c55c415af16dbc4158a2a7854063089383aa2504febea5b48d20267ff4892e5", size = 339745, upload-time = "2026-10-15T17:58:25.771Z" }, + { url = "https://files.pythonhosted.org/packages/a7/a6/757edd625dafa3d6b8cdf5f3e047fa865dfd16a625359a21013cdc34d0f5/hiredis-3.5.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc91b554353429715cb5e76d8f7c21ec40608831af2c16f9f93dd7462aa87107", size = 350509, upload-time = "2026-10-15T17:58:27.027Z" }, + { url = "https://files.pythonhosted.org/packages/a5/1d/27fca9037040e83cf8b05f2a69d41ce0e28a376f12edb577d84b8c86bf9f/hiredis-3.5.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efe53f63d31f53088492f9e1a4f81fb84d8cf75541927679670ff0bdaf17e11f", size = 312420, upload-time = "2026-10-15T17:58:28.152Z" }, + { url = "https://files.pythonhosted.org/packages/1c/99/60a1629f4f4769b414fd0b3f816e34f2a4b1b38840ad5345ec7417f01966/hiredis-3.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:672e92fbbbe353697828f6c590802b5ae2d8c97bb78726ba4d107f87b0e6289c", size = 301091, upload-time = "2026-10-15T17:58:29.214Z" }, + { url = "https://files.pythonhosted.org/packages/8d/9a/9604f0f6998af9566b921f5467100cd1a1ac916a24f9f3031ffc2be65bac/hiredis-3.5.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c404af8da2fadfbb538195f7ce1a31e1d71900c573879b6f1475415dd474f822", size = 331426, upload-time = "2026-10-15T17:58:30.511Z" }, + { url = "https://files.pythonhosted.org/packages/7b/49/4d7fc61d540d79bc0544baa162da973a838decd440bdb2100d3116de9c57/hiredis-3.5.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d86562ec68af7e5f332d7e4433c0892e6661082beba0718501c9e7ecbac63078", size = 331856, upload-time = "2026-10-15T17:58:32.017Z" }, + { url = "https://files.pythonhosted.org/packages/87/97/909850b4a4ea761731a6ac0043403ee0b3423ad7f7c6b8e2df8b7bfe8b35/hiredis-3.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1df3e11d7366026c4780324a5a0347dce84db2a5adc02e34d3232015854dd7c8", size = 310917, upload-time = "2026-10-15T17:58:33.493Z" }, + { url = "https://files.pythonhosted.org/packages/eb/71/7e4381d3c29b1e90605f3be571944fa2c67861fa954d0862b6464c0a02b8/hiredis-3.5.0-cp311-cp311-win32.whl", hash = "sha256:70fa3bef2838607e9fa911ce6d8cf256350d16057d249e2e4de2c877bc83fd09", size = 38744, upload-time = "2026-10-15T17:58:34.483Z" }, + { url = "https://files.pythonhosted.org/packages/00/6f/100353601702c371ec7c2b846191216958ffb112a53cec7f58bd42955d32/hiredis-3.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:1453cbd66450be310fdf2c6eb2e1d89b7379ca0e396acc14345621eaf68d2be3", size = 40575, upload-time = "2026-10-15T17:58:35.466Z" }, + { url = "https://files.pythonhosted.org/packages/f8/54/a113debd9a6a21f6a425e9b2379fd10bf0eac7e9981a47868cac78e0d30d/hiredis-3.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:03f9b755123e10bf4c25f1ab67bd5223d461c63acb670c0bbe1588c4330d86d6", size = 38023, upload-time = "2026-10-15T17:58:36.473Z" }, + { url = "https://files.pythonhosted.org/packages/ec/f0/af16bdd438ebe098da97bd4035047e673147e5dc36e9259671c0b5301d48/hiredis-3.5.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:10dff7baff891c8ec4174e0d228cad7deefaf827a90d89184aa695437557ce70", size = 140904, upload-time = "2026-10-15T17:58:37.712Z" }, + { url = "https://files.pythonhosted.org/packages/ee/ef/f2324378e589d2ae146614a1da8f715a2b5e5c9d7f8ff93a0ef999521d98/hiredis-3.5.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:2d1c87ba3303efdf259d119fa4ab28a79c8f2edae4b2d04cae7d44f75196864b", size = 75141, upload-time = "2026-10-15T17:58:38.794Z" }, + { url = "https://files.pythonhosted.org/packages/f6/86/368167b4a375473995db547b2b55d46ac4b0b6897bb0522c87946e02cd78/hiredis-3.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:328855d933ad0e25d731374ebb3fdbd18e9e748f8b7e04980ca6c720a59dbdbd", size = 72024, upload-time = "2026-10-15T17:58:39.746Z" }, + { url = "https://files.pythonhosted.org/packages/6c/36/8aa210342266d9b059a09cf7a360be498e7b7aa2cc53d8cc0a9bc0fdbc85/hiredis-3.5.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:231ec55363134d1036094b70b039cd2adef45d694deac635d437e9edfb5f2164", size = 308421, upload-time = "2026-10-15T17:58:40.862Z" }, + { url = "https://files.pythonhosted.org/packages/db/fa/33d1823b55fb67ac613ab60c566c18b78ef1159efa3f50d5fefc0e2d72ad/hiredis-3.5.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95343058583782388564cd0756089a13c827e34abe938720cba102e756df547f", size = 341662, upload-time = "2026-10-15T17:58:42.005Z" }, + { url = "https://files.pythonhosted.org/packages/f1/da/f5bfb59042382f942f42c09cc83cd4340dd0f42f7278c86d34c0776281e1/hiredis-3.5.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a8c6df1dd76b93e94197f7195f4035f7b45ebbd1c89a67887d4f9c19d74fdbe0", size = 353531, upload-time = "2026-10-15T17:58:43.206Z" }, + { url = "https://files.pythonhosted.org/packages/0a/0b/861980a5ebbd00e812058c8838ff13ab16f6a807f394c549379d0f7cd641/hiredis-3.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a59975d62a1f7c55d49828fa3515824cc31993379a8f915cc20b5dd27e4c0d29", size = 314822, upload-time = "2026-10-15T17:58:44.322Z" }, + { url = "https://files.pythonhosted.org/packages/77/05/004ad0628f5df4b7dbf841ff817d879e6f55502571cff86de01a01055520/hiredis-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c043397db3bbb062d275dc3e7adf68544cd1ead1e2c9ab1bc30c616038b3ecd7", size = 302533, upload-time = "2026-10-15T17:58:45.532Z" }, + { url = "https://files.pythonhosted.org/packages/11/ab/da622bd27aa42a61d2b6726d6eaf820bc269f61bdea593b101ea3fd1d157/hiredis-3.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:879c774ac16276219d7f9013d28851a72338c703ddaf8509c20592537acedb25", size = 333799, upload-time = "2026-10-15T17:58:46.86Z" }, + { url = "https://files.pythonhosted.org/packages/da/d2/81fe508df01c14315a7bbbf61f7f3500799a8a1ca7950b35a10e876ee493/hiredis-3.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:55776bd0c298ddd4c07342e212fa50b3e753353c763b1b112a220ea99d2616a8", size = 334657, upload-time = "2026-10-15T17:58:48.114Z" }, + { url = "https://files.pythonhosted.org/packages/d4/01/886628d0b49d81a58b759b80a3594a3a9b4b5d8388b6cfe0e8facb55af9e/hiredis-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ec5e911cfaa57cba61158839c0079bf9524571b4f03023c236bc86ac79a2c229", size = 313271, upload-time = "2026-10-15T17:58:49.438Z" }, + { url = "https://files.pythonhosted.org/packages/9e/27/2eeb6c6f21d3560dc72bb0f89012205eb1f4422d7f50e9358a85dd56a006/hiredis-3.5.0-cp312-cp312-win32.whl", hash = "sha256:cafaf4406d3a6bb1990b8437b94ed592dd81bbb5e99fed6f5290801cac93a8d6", size = 38738, upload-time = "2026-10-15T17:58:50.504Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/b802fc53a12845a69c880e3906d50f05f97bb8978c53bd2b8a070a5224d3/hiredis-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:601bfcb5655e85a79e2ded96ab0159cbac3a20ef991c468e8c9d2e374dc8cf54", size = 40605, upload-time = "2026-10-15T17:58:51.413Z" }, + { url = "https://files.pythonhosted.org/packages/9b/13/354336f3e5adfc97c21650a7be7b1e96df7b9736a7a701a52a90e523adf4/hiredis-3.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:fc5e619354f5ddc7381d244e0b00125180a7910a747ec3582bd4bc569da8e357", size = 38060, upload-time = "2026-10-15T17:58:52.388Z" }, + { url = "https://files.pythonhosted.org/packages/46/de/ec9f269bdc8fdfb3309aa7de184f7f19ce84399f8e76e9023a77c84f0b44/hiredis-3.5.0-cp313-cp313-macosx_10_15_universal2.whl", hash = "sha256:eb9c5d20a593a19a5b9d3a5ce3ff7689adba2cd859cfb49a4aab53154210515f", size = 140915, upload-time = "2026-10-15T17:58:53.417Z" }, + { url = "https://files.pythonhosted.org/packages/cc/8b/9d8b20f4ef669d20c7496786db3e44914c4d44255d49e8719c993ab9409e/hiredis-3.5.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:1286a8a950b6364ab5f87a173e41b39eaa090294fc120f5cbff9cfcf6d92fad2", size = 75143, upload-time = "2026-10-15T17:58:54.572Z" }, + { url = "https://files.pythonhosted.org/packages/67/c7/6af3f82b35339631d9f7f0d77fefd57c4cf20004f376441e1245ba07d9ae/hiredis-3.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:33f988f062da42e167d7887a4cc6df5e48727700d1c9966ee2e761db844a02e8", size = 72023, upload-time = "2026-10-15T17:58:55.649Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ec/35274f646f06a01f3c81faa3260478b0f09f330bd557a0d60180ee8d9d12/hiredis-3.5.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e1170deb21494371005b4beb9451d904cf3adb2b8b62230b1c51f38267feedf3", size = 308391, upload-time = "2026-10-15T17:58:56.9Z" }, + { url = "https://files.pythonhosted.org/packages/3d/55/c47154c01e9e0ef1c22e29f61039686ed5df3e0f805c26a659dae451bb48/hiredis-3.5.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:808ca3e2bbaf4e82bc57508b3ac2a662a0317a1b12762b48d8bcaf31c631c966", size = 341603, upload-time = "2026-10-15T17:58:58.218Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ff/2a0c2e3334e31a048aaa9f5a073200c67fbfd5719f8b2e95d8407e6cde97/hiredis-3.5.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c107d375d5a5687297f99669617b2055e375df41f83ac976ac6aca2bc33c415e", size = 353483, upload-time = "2026-10-15T17:58:59.587Z" }, + { url = "https://files.pythonhosted.org/packages/0c/6d/ff4cb866c22e2a3b03e588d17976589683c9f4ebdbefe9c47e387abf3fc4/hiredis-3.5.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:588d083aace7439afa652e201df7388aeb1a5d3230ffe26fb297cfc6c7f216de", size = 314758, upload-time = "2026-10-15T17:59:00.893Z" }, + { url = "https://files.pythonhosted.org/packages/34/37/a650f8ac92f2a824dbe7fccc9264ce6875133a17b99920788b5f2c25fcf8/hiredis-3.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2bd58ca4b34f7e86fe1de666073081973f26fc75fc7e2551d76b21f49a235855", size = 302581, upload-time = "2026-10-15T17:59:02.308Z" }, + { url = "https://files.pythonhosted.org/packages/22/1f/a883535a8ebd1f7f17754a1b05e7987c4b4f91ac70729c2c2cf0fd1a5b83/hiredis-3.5.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b116b8efa40a1789244129bc2dee9f8dc973d6a48d9331c919308034b27520da", size = 333923, upload-time = "2026-10-15T17:59:03.819Z" }, + { url = "https://files.pythonhosted.org/packages/de/ef/6f6521e8595d4dcfb658d10fb1d43c363a1e2fe06858e6790fc8241343ba/hiredis-3.5.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fbd9de72a958fcdc70633d0fbab3b8010555471c746f39ea48976a4da1680e4a", size = 334703, upload-time = "2026-10-15T17:59:05.142Z" }, + { url = "https://files.pythonhosted.org/packages/4b/40/1b5723e2464e6130e73de5ab91c558b43942ae3247c7ce9e75e8ac654d98/hiredis-3.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0037e3af5f457d47d29eb3dda70302758385670a3378be7daf8c5049403a172f", size = 313283, upload-time = "2026-10-15T17:59:06.35Z" }, + { url = "https://files.pythonhosted.org/packages/ac/f9/b3885abe8a77c9ab74311eb824d224bd4ec5e3b53f947941a12555d0a57c/hiredis-3.5.0-cp313-cp313-win32.whl", hash = "sha256:5201151dfc4098f81485aeac651cbfb402c8aea92607a5ddef5bbff5097d10e6", size = 38743, upload-time = "2026-10-15T17:59:07.582Z" }, + { url = "https://files.pythonhosted.org/packages/34/d9/a6bfa649cc1be25a3cbbc180a4fd77d91d57382a4f3335407cf705e4def1/hiredis-3.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:1ad14a185ece13a456dd8da27b7acd4105b4a2fccbdf54d86a3d455d6df7ffae", size = 40615, upload-time = "2026-10-15T17:59:08.584Z" }, + { url = "https://files.pythonhosted.org/packages/fa/95/fe6e3ada77b59c440ee212d00667336ecfc8957443a0384aefeb0ba0debf/hiredis-3.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:0f09667f00035ccfb83b942018b5e84dc560774372fb237b9f7f757d72603c37", size = 38070, upload-time = "2026-10-15T17:59:09.759Z" }, + { url = "https://files.pythonhosted.org/packages/34/d4/ec6c2ab107ffbb13311e0b086fc47a6ff7bfde4264c6d0208b8c9c4ec393/hiredis-3.5.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:a9ef58e5a9c80a2159c25c6fe5f856734c4e80201b4cecd513f3bd9117d90417", size = 140958, upload-time = "2026-10-15T17:59:10.775Z" }, + { url = "https://files.pythonhosted.org/packages/92/67/c1a29f6555ee51b2c8aa8cea78071e265e42ca6bd599c6623dd5ac0d20ba/hiredis-3.5.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7a47c7136637df55cc021038de2a37c7e8a1183e6331ea4874ce65ea3adf7bce", size = 75151, upload-time = "2026-10-15T17:59:11.91Z" }, + { url = "https://files.pythonhosted.org/packages/73/a9/2d098b3363717763f69e21fd04deba33795e3a6b7736d1f2e70330524207/hiredis-3.5.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:57f8112850e333a2efb12e8db1b7b0ac7585950a1e325b0c7693d98c361dc775", size = 72068, upload-time = "2026-10-15T17:59:12.961Z" }, + { url = "https://files.pythonhosted.org/packages/2e/bf/88b70c2e12efae647530b61bbb31729ed6097436851f590a6d579e8ec89a/hiredis-3.5.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3dc0b74b106f56a6129ea7fea856e5c4f0ada96f26b079a4c2ea7c6e1cf1a5a6", size = 308598, upload-time = "2026-10-15T17:59:14.136Z" }, + { url = "https://files.pythonhosted.org/packages/67/99/c76d37b79b84891b1e747f1b6b3bec908ffc9f2962a64ceb533bbb10d07b/hiredis-3.5.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5001493d8ef9d699eb43bbc39a564796c2af4777cd1a198f0168814551596311", size = 341952, upload-time = "2026-10-15T17:59:15.487Z" }, + { url = "https://files.pythonhosted.org/packages/10/2c/2468faf3a073ccc0e03646cffcb9f517794da2e09aeec3ac547e2eef5aba/hiredis-3.5.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:60b9f1ab9d366f19d53da2c844fed58ea2e7ef3d394db95647ed87e07a2456ce", size = 353577, upload-time = "2026-10-15T17:59:16.816Z" }, + { url = "https://files.pythonhosted.org/packages/82/14/d3a7b1bcb77ff5caac4e76a2eed61b47850233035d837d50d2cb029e4eb3/hiredis-3.5.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf61715d357c58996163ccedb972eec208a882b95bf04a90a95ef0486af5022e", size = 314739, upload-time = "2026-10-15T17:59:18.103Z" }, + { url = "https://files.pythonhosted.org/packages/dd/f8/aa061e52d1f75672f1f7478b90dc69ad5a1873523e1cefab98d736014970/hiredis-3.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:de694f45570bb9102904558a50e2ef82963413f37b0300c8f937f2f06d601ad1", size = 302798, upload-time = "2026-10-15T17:59:19.459Z" }, + { url = "https://files.pythonhosted.org/packages/d8/22/e05a1b19c8900465601b6a005a5fc92b5a7707d470f6997d49b268f2541b/hiredis-3.5.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:583de475ceaa052a8353a91d30c33b42a3267a4455550714935b52575a9d3211", size = 334070, upload-time = "2026-10-15T17:59:20.764Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2a/39002fea0ef3f1b14da719926eb5f8b36480a3c43d460a44e10eb14a38a3/hiredis-3.5.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:1134ddc5f1b2a754adacc2c988485f9db0f5ae479b9a87472b9fdd4c4312a8f7", size = 334831, upload-time = "2026-10-15T17:59:22.011Z" }, + { url = "https://files.pythonhosted.org/packages/72/c2/5eb024e9a51c3fb35106e45f149f3284639d1c260af3fb15f8754c39698b/hiredis-3.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a890b8956729b366ef533249a1db8709714dbb02b6a292a40f1383a54cef0945", size = 313134, upload-time = "2026-10-15T17:59:23.342Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ae/043bfe0869b5ddd30c5d54f242ccf7e77aa5ba2274bb72c9544921159249/hiredis-3.5.0-cp314-cp314-win32.whl", hash = "sha256:2f1c2968ba739fccf94746247c10c869183530c99d0bdd5bddfaa28f0cbbe3fe", size = 39492, upload-time = "2026-10-15T17:59:24.429Z" }, + { url = "https://files.pythonhosted.org/packages/f2/8e/9ede60bca300139eb74034b15c19f8046437c8763d4f65b3ff580805e5e8/hiredis-3.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:fcc7542689996df026486d75a29080e102c6b7131c064e22b762949f44d5e4c2", size = 41189, upload-time = "2026-10-15T17:59:25.447Z" }, + { url = "https://files.pythonhosted.org/packages/35/e1/8c1e7dec56d6735642001308846cfa423d0d7b34014358593f6c74f41c9e/hiredis-3.5.0-cp314-cp314-win_arm64.whl", hash = "sha256:f714f562792f9777926d1cc0675445147b99ff2e3b951baf109d05195131eb10", size = 38773, upload-time = "2026-10-15T17:59:26.424Z" }, + { url = "https://files.pythonhosted.org/packages/a4/aa/f27ed6b41e97e1ce0c4a22f3489e26119ca11a95c3c8309514e250cca658/hiredis-3.5.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6bfdf75a934c7d73e660b9ebea3ade761f6c2c67b6725dc71e556fceb0f95194", size = 141946, upload-time = "2026-10-15T17:59:27.468Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ed/7bdbaade5f840901d76a7c9b61bce7018ddbde3bfe3c21d9ee51b9463e46/hiredis-3.5.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:9d7f99fd94460f9bd83b042c9c5a05f746dbb55f116430b4daf93ee290246bfe", size = 75641, upload-time = "2026-10-15T17:59:28.891Z" }, + { url = "https://files.pythonhosted.org/packages/b3/73/e015902deb78dcf2decfb17ae4229fa18905d6aad5053977d4326d718def/hiredis-3.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:48634663d477efe8b8b76497e123d85d3ad1606cb919a833a4d447f00ff6a8c8", size = 72574, upload-time = "2026-10-15T17:59:30.226Z" }, + { url = "https://files.pythonhosted.org/packages/47/3c/5f615cf57771a10b76a8ab1aeb8302395efd6e05343fdb621416119b4858/hiredis-3.5.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e1590dbc3fbb78715c9673db04c0ff37c0d1b97660740064146ae3ce68e604fb", size = 318075, upload-time = "2026-10-15T17:59:31.585Z" }, + { url = "https://files.pythonhosted.org/packages/77/96/b958bfb025d175f394e58829137967a58616d70f4c67273b6dcccf16e9ab/hiredis-3.5.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:185fdce9552b768ee6c10585c0358a8264d9fbc5e78bfd58f0a89fff0da44f44", size = 350457, upload-time = "2026-10-15T17:59:33.399Z" }, + { url = "https://files.pythonhosted.org/packages/67/2a/9da03e7e447f29f516660aec1b2624c4a498d6b7eba33c0f10f4dee6bd9b/hiredis-3.5.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6b2a3ae55fca9fe7e5ed23ec0310581dd5310d9c66b7420d821bf2d30fa3a4c6", size = 362348, upload-time = "2026-10-15T17:59:34.985Z" }, + { url = "https://files.pythonhosted.org/packages/55/24/26e41a53ed053ca151a02a91e44474e65a1055b599228041fc1676d3c198/hiredis-3.5.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d0ccc663f871efb4ee52ad7ddc72780fb965df47caec38a2ef1995a9eada8b6", size = 323522, upload-time = "2026-10-15T17:59:36.352Z" }, + { url = "https://files.pythonhosted.org/packages/03/70/9452702a27316d48c62c943d5277d06e61107f63b9d2602a6b501250bbfa/hiredis-3.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:dc892768e5fc87566919856cf5a5df189cb45a70c1eaa412c8ae8028202aa015", size = 311520, upload-time = "2026-10-15T17:59:37.633Z" }, + { url = "https://files.pythonhosted.org/packages/3d/02/83d3754a5f97c8e867b0ca2285dfcda4f7cca3dd3a3f955de20dc4c0b7f1/hiredis-3.5.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:7d217165dac546d7781b169c77573f8ef743eacca755ab44568e180b95ab44ae", size = 341899, upload-time = "2026-10-15T17:59:38.932Z" }, + { url = "https://files.pythonhosted.org/packages/ad/35/f6ea1b6bf97f73156ba492ec577c2f22a4af32fd6fc87c5352bc88de6be5/hiredis-3.5.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:7485db3f95ad46346ed702b99e380c2a61652bf5a8246dc101b8cc3f236dafd2", size = 343367, upload-time = "2026-10-15T17:59:40.179Z" }, + { url = "https://files.pythonhosted.org/packages/53/4e/bf80b882d945bba735d4f940892dfa2a46007b80cd96b9771d2cb1a1aebd/hiredis-3.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fca8f8974588f3563decb8812df201f08cd2ac1d4bb84fca295ff820cfbf0524", size = 322127, upload-time = "2026-10-15T17:59:41.45Z" }, + { url = "https://files.pythonhosted.org/packages/22/ac/79c9a3f5811a91ec84bc0c957ee090b6fc5e56d12c64e214f206a1711786/hiredis-3.5.0-cp314-cp314t-win32.whl", hash = "sha256:ceb80b8c9bf8e818dbd934e1fa6eb8c002a82eaf9f323fa0e47da0b750d9e0de", size = 40077, upload-time = "2026-10-15T17:59:42.674Z" }, + { url = "https://files.pythonhosted.org/packages/71/b5/04761a2c2a110afda866cf48a1b64c11a43a427db817dcee749ae912e8ac/hiredis-3.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:29812df61589038cc6d58e8349b95c1da56a6380820f93d5b41ffc9c7f4c5cb0", size = 41758, upload-time = "2026-10-15T17:59:43.716Z" }, + { url = "https://files.pythonhosted.org/packages/04/93/a7519f202c2b2b5fe84e83e58147ed4b6eeae9d239e666de9f3c2076efe3/hiredis-3.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b4411b17db1e12fb9ff5e3f5b74292739887ad288039e98f8d804378b435abd1", size = 39224, upload-time = "2026-10-15T17:59:44.719Z" }, + { url = "https://files.pythonhosted.org/packages/d1/c9/e37d1a4f57407da671ba4ea5e46824ab4c31f992e1cb93c1f5d786be7532/hiredis-3.5.0-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:8f2e4e3af78c3d8a8d800a1d252024185943ca210f1089daf20188526491b8a9", size = 141107, upload-time = "2026-10-15T17:59:45.802Z" }, + { url = "https://files.pythonhosted.org/packages/89/cd/b3bb82dbf7825a817ec03d5ea1e79c70162059f9914e7a6231b62625bc43/hiredis-3.5.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:9132be543d677159d0e6c44bc86c887f2a07f47c163178aca02bc3c0851dfa50", size = 75152, upload-time = "2026-10-15T17:59:47.002Z" }, + { url = "https://files.pythonhosted.org/packages/c2/66/31a5c959ad64dd4c47373410d0414742532c88a639ebb8b749e401b85732/hiredis-3.5.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:931f585f15c1b165dbf6fcc781fb8975d36b99e6de5524485ab9c1164a3f0f91", size = 72221, upload-time = "2026-10-15T17:59:48.092Z" }, + { url = "https://files.pythonhosted.org/packages/a4/d2/9850540d3601b484b88aa11fbcf9782241013644bfff4517f5483776fee1/hiredis-3.5.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92ee5f3fde72816784f12bfec4278a56f158e5a622a353663980b1f00c4eee64", size = 309583, upload-time = "2026-10-15T17:59:49.266Z" }, + { url = "https://files.pythonhosted.org/packages/31/88/905ff269ed52ba7a3ae470816321ff2162e657a88506f7a27bd45a22d5ea/hiredis-3.5.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f10b854dc1894b5a664b66508f0a0e3e935cf6560481ffac378d12138647fd32", size = 342180, upload-time = "2026-10-15T17:59:50.624Z" }, + { url = "https://files.pythonhosted.org/packages/fe/14/d1289646a82d34fe758c4487c53464d65ec99e927ed7df7b9b5130199d8f/hiredis-3.5.0-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d4f3360d8254ff3b58854705f70b01bf72ca550a4943ac76ef0f569239f05970", size = 353744, upload-time = "2026-10-15T17:59:52.101Z" }, + { url = "https://files.pythonhosted.org/packages/33/c3/3cf99726f5a5ef7c7402e024b7ff9b9105d5f0f648d325cbc7313f160f9e/hiredis-3.5.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:86bca28fd6f7f9cb9577a6d4ae38bf55104c0da8ff0fdcca0f5336b6fafd4629", size = 315976, upload-time = "2026-10-15T17:59:53.45Z" }, + { url = "https://files.pythonhosted.org/packages/d1/1d/21c8b325c254f8cb1286c6b23f045c646df386e6ed621b4904618c5b5814/hiredis-3.5.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:4fde6c6d67154c7ff84fd73c7aa9c5c03b9ea6ca0202fda5bd0c79ab99e51171", size = 303921, upload-time = "2026-10-15T17:59:55.005Z" }, + { url = "https://files.pythonhosted.org/packages/52/f9/7d3681c29c0c18066d99abdf7124fde00ba1010ec9973d4fdb1d815a0d66/hiredis-3.5.0-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:dd328ff8bc665f82a42b1f9561789f72ebb90147dbd3413c0990fc8837f7cc8d", size = 334358, upload-time = "2026-10-15T17:59:56.398Z" }, + { url = "https://files.pythonhosted.org/packages/ee/22/fd572b5391ca8fba3b80fd8d06b95c4c85dbd9815cb3caa374e0f87a746b/hiredis-3.5.0-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:c73535d109c0d1f4a95f40dfb667c8baa722acc84187d128551706e8229d7223", size = 335369, upload-time = "2026-10-15T17:59:57.654Z" }, + { url = "https://files.pythonhosted.org/packages/f4/e9/a2aff206d2c95fa354a675ed27c0b168a1e20ae349c4d3fd1de2351bb098/hiredis-3.5.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:c5e8ea62c81282bb845a96460158190259a17a82124d876bb2291d01006fe83e", size = 314003, upload-time = "2026-10-15T17:59:59.026Z" }, + { url = "https://files.pythonhosted.org/packages/15/e9/4289d3b9d1b27cfec1ad82c6011748d039ad6d98757677b3e8c92c8e98dd/hiredis-3.5.0-cp315-cp315-win32.whl", hash = "sha256:ddc4eb264753d7f8d4b68cc164b509824d0d6c84119f896e209c9a679df3b386", size = 39503, upload-time = "2026-10-15T18:00:00.179Z" }, + { url = "https://files.pythonhosted.org/packages/3a/fb/ad69fea7e3a3f4eacd17a1de5197dc27b4181b634e594dc494a57c427828/hiredis-3.5.0-cp315-cp315-win_amd64.whl", hash = "sha256:ef139e9d52843cc91ed3aa1659544ba10b19dfe48e30dac3b21e1f9a14640a22", size = 41185, upload-time = "2026-10-15T18:00:01.753Z" }, + { url = "https://files.pythonhosted.org/packages/74/e1/7d50ae62140d49e41df72de836ef0e22002daeae9119abd385600d906639/hiredis-3.5.0-cp315-cp315-win_arm64.whl", hash = "sha256:392506b5ad5d926b764d6a5663a3833a0110fd9256ff86ac31f53ccd4971941c", size = 38785, upload-time = "2026-10-15T18:00:03.084Z" }, + { url = "https://files.pythonhosted.org/packages/29/7f/fae64ab5a59f8cce5fa3396cd73e88c0ab52488c2c4caf6c93c773776b44/hiredis-3.5.0-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:942771ff3d4121b242c8f3c55ba2c28bae06af426ad459b751bbc905c3c6e944", size = 142058, upload-time = "2026-10-15T18:00:04.278Z" }, + { url = "https://files.pythonhosted.org/packages/d2/70/3f3476ae5b511ef15d34458878808d2af94b311192ba0a2a1c2c9f09f982/hiredis-3.5.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:3bc5378fe7a1a8226a42d31f05f6a553ac471dd495f38dbd3967a38794b4acc0", size = 75647, upload-time = "2026-10-15T18:00:05.591Z" }, + { url = "https://files.pythonhosted.org/packages/9e/f5/27038d5e40be831dfb44fe8c7043b2f312010616608cfff43fe076bfb168/hiredis-3.5.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:43a7b471185ee560ac8b4070b6d7d0b022e298749e4bd0dc95e11321701c9de0", size = 72702, upload-time = "2026-10-15T18:00:06.897Z" }, + { url = "https://files.pythonhosted.org/packages/f9/d6/f13fc5f3ad7f9718bf39383f0f640503442469f6933ad6caad369f64cc50/hiredis-3.5.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:54733f0088369b834677dbb5368e2a4bd16fadfaa43882de4dc13e2466a00b8b", size = 318729, upload-time = "2026-10-15T18:00:08.68Z" }, + { url = "https://files.pythonhosted.org/packages/f4/27/12f54ff4a88dbb2c00f03c0b3d71ace167e6a05704323bc36da0774a8ab2/hiredis-3.5.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dd26a99b1f703c870a9ce0a96615bd2530fcf04c1ba5dedca100e32bbf38d67e", size = 350641, upload-time = "2026-10-15T18:00:10.137Z" }, + { url = "https://files.pythonhosted.org/packages/58/e4/737710596fcaf9cfeae44746717fdc3cc04f69ed7846b2377a8388786e97/hiredis-3.5.0-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0dc22d8c05468603e8fdd821189bd83e657f180b1af88720b51bdc71448f4346", size = 362164, upload-time = "2026-10-15T18:00:11.882Z" }, + { url = "https://files.pythonhosted.org/packages/44/73/18f43a29db4e632401c0f0a1a1a323280fbed18ed60d7ffda4054e8db616/hiredis-3.5.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ae897c05c38267e550acc085cd45f1fa2d0f793abe2ab334f38e18b4e8c5c39d", size = 324653, upload-time = "2026-10-15T18:00:13.415Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5c/b12b06a29879857ac0ac5a62f54f466acc3c5400f9ca89326045db4b8c0f/hiredis-3.5.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:18a8bbb37efb39da559ea7ab1619a56bc051c4490506795d8686f56ca1063224", size = 312130, upload-time = "2026-10-15T18:00:15.049Z" }, + { url = "https://files.pythonhosted.org/packages/9b/67/711e69e532b856d3d1aa935667bf96233e5e9f4c21016e63a14edbcd3884/hiredis-3.5.0-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:5556c12e6c62cb96162135594876e48db98ae3951afac8ef38831ac779d7bd01", size = 342134, upload-time = "2026-10-15T18:00:16.651Z" }, + { url = "https://files.pythonhosted.org/packages/39/03/32044910ffdb441c97a2c7dd4f9a920a28f1921ae5ffb1893e1c22c276bb/hiredis-3.5.0-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:f6335c0d65104fd2f62fdff813067e72fdbcd4c9924066921ab931ef82591730", size = 343637, upload-time = "2026-10-15T18:00:18.164Z" }, + { url = "https://files.pythonhosted.org/packages/d1/8c/02d04f2e4a0339145e4fe00407d2c34fa29c4c1ae6cd3feab46c87fc25d4/hiredis-3.5.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:894391ee4f5465fa1f3e6a0a323c76bb7884eda2698a40f8157d6a96fa268899", size = 322470, upload-time = "2026-10-15T18:00:20.053Z" }, + { url = "https://files.pythonhosted.org/packages/cf/a5/b9d5125ca54e338cae8342f94d937ea8e10b4f8c450f239248c102a7f4ce/hiredis-3.5.0-cp315-cp315t-win32.whl", hash = "sha256:c0b902612627b307dbab51ea556f82b61763fe42d43026e81060ca7ac8642df3", size = 40086, upload-time = "2026-10-15T18:00:21.484Z" }, + { url = "https://files.pythonhosted.org/packages/a4/d3/29a674417766263387038d670137a34f213fdd1030acbb44036c5ce1e882/hiredis-3.5.0-cp315-cp315t-win_amd64.whl", hash = "sha256:1dd0db7d759ea43f0c37c6d841b8e9ea652bdec5e4f542872f773eef3cfcbfe4", size = 41769, upload-time = "2026-10-15T18:00:22.658Z" }, + { url = "https://files.pythonhosted.org/packages/c9/35/ed21d49c8db2460b94bd63e45505c75667231d0dc653d80eb6cf4255cb41/hiredis-3.5.0-cp315-cp315t-win_arm64.whl", hash = "sha256:c360575e546aa093b3bff6634e8c916cd5b52ca766f51b49474acda219dd3c88", size = 39201, upload-time = "2026-10-15T18:00:23.789Z" }, +] + [[package]] name = "hpack" version = "4.2.0" @@ -3483,7 +3721,7 @@ dependencies = [ { name = "httpx2-jsfetch", marker = "python_full_version >= '3.12' and sys_platform == 'emscripten'" }, { name = "idna" }, { name = "truststore", marker = "sys_platform != 'emscripten'" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/a0/e9deef4654132857b5a5dbe4eddd0ac59c2814500e11f2f5044cd81103ee/httpx2-2.13.0.tar.gz", hash = "sha256:81bd07dc67a3701729ef1f777a3c00c915d4539604fdb5afd327f8682f6b7b44", size = 100290, upload-time = "2026-09-14T14:18:05.486Z" } wheels = [ @@ -3522,7 +3760,8 @@ dependencies = [ { name = "pyyaml" }, { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "tqdm" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fe/0f/e83fdd856da8fca26bf78d71709ebd120432a0ce535e72b9597cab1eb5bf/huggingface_hub-1.32.0.tar.gz", hash = "sha256:ed70a45498abe86039df7c2f4e5f7575de524be908d3840e8f828d5525eafd6a", size = 1038662, upload-time = "2026-09-17T10:27:48.049Z" } wheels = [ @@ -3552,7 +3791,7 @@ dependencies = [ { name = "priority" }, { name = "taskgroup", marker = "python_full_version < '3.11'" }, { name = "tomli", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "wsproto" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7e/3a/df6c27642e0dcb7aff688ca4be982f0fb5d89f2afd3096dc75347c16140f/hypercorn-0.17.3.tar.gz", hash = "sha256:1b37802ee3ac52d2d85270700d565787ab16cf19e1462ccfa9f089ca17574165", size = 44409, upload-time = "2024-05-28T20:55:53.06Z" } @@ -4072,7 +4311,8 @@ dependencies = [ { name = "attrs" }, { name = "jsonschema-specifications" }, { name = "referencing" }, - { name = "rpds-py" }, + { name = "rpds-py", version = "0.30.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "rpds-py", version = "2026.9.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } wheels = [ @@ -4255,7 +4495,8 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, { name = "langgraph" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/56/7c/651d0dc4913a7a892156c03dd343b99cfe19ee729e6911ab1f4fe7567b8b/langchain-1.3.9.tar.gz", hash = "sha256:9b14ef0db9ef314299ded858b22ca2a40b8f1b05c8c9cb6b82d53a53075fef00", size = 631514, upload-time = "2026-06-12T16:53:27.083Z" } wheels = [ @@ -4270,7 +4511,8 @@ dependencies = [ { name = "langchain-core" }, { name = "langchain-text-splitters" }, { name = "langsmith" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "pyyaml" }, { name = "requests" }, { name = "sqlalchemy" }, @@ -4311,7 +4553,8 @@ name = "langchain-community" version = "0.4.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", @@ -4345,10 +4588,12 @@ dependencies = [ { name = "langchain-protocol" }, { name = "langsmith" }, { name = "packaging" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "pyyaml" }, { name = "tenacity" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "uuid-utils" }, ] sdist = { url = "https://files.pythonhosted.org/packages/12/e3/bea6d0080acf183332f24dcd74c208aee5857cf8f783c3fb0bd86027d8fb/langchain_core-1.4.8.tar.gz", hash = "sha256:5bf1f8411077c904182ad8f975943d36adcbf579c4e017b3a118b719229ebf9a", size = 957974, upload-time = "2026-06-18T19:39:23.636Z" } @@ -4375,7 +4620,8 @@ name = "langchain-protocol" version = "0.0.18" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d2/59/b5959aea96faa9146e2e49a7a22882b3528c62efafe9a6a95beab30c2305/langchain_protocol-0.0.18.tar.gz", hash = "sha256:ec3e11782f1ed0c9db38e5a9ed01b0e7a0d3fba406faa8aef6594b73c56a63e6", size = 6150, upload-time = "2026-06-18T17:08:26.959Z" } wheels = [ @@ -4405,8 +4651,10 @@ dependencies = [ { name = "opentelemetry-exporter-otlp-proto-http" }, { name = "opentelemetry-sdk" }, { name = "packaging" }, - { name = "pydantic" }, - { name = "typing-extensions" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "wrapt" }, ] sdist = { url = "https://files.pythonhosted.org/packages/97/30/6a64dcf84de2f2eb4d03adbfd22cc7bdc95ce67e3e56cd6288405087fb8a/langfuse-4.15.2.tar.gz", hash = "sha256:7f818f38cc22daba88fdcec62d2addcee4e18d1af4529978b6d07501e86b6946", size = 391727, upload-time = "2026-09-09T16:01:25.73Z" } @@ -4423,7 +4671,8 @@ dependencies = [ { name = "langgraph-checkpoint" }, { name = "langgraph-prebuilt" }, { name = "langgraph-sdk" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "xxhash" }, ] sdist = { url = "https://files.pythonhosted.org/packages/02/7a/ea09b05bb0cbddfa43bd34fc581357e87fc3f21a751cc0d419688c3106da/langgraph-1.2.6.tar.gz", hash = "sha256:f9b45a34f13930c94d96cdb76277447ad2cc70ec2d18cd2764d7fdadb36cdc1b", size = 714400, upload-time = "2026-06-18T20:58:21.514Z" } @@ -4481,7 +4730,8 @@ dependencies = [ { name = "httpx" }, { name = "orjson", marker = "platform_python_implementation != 'PyPy'" }, { name = "packaging" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "requests" }, { name = "requests-toolbelt" }, { name = "uuid-utils" }, @@ -4636,6 +4886,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/3b/f4b1548d4f5b99186737fe27aec238e9823e8d5d23bf4df007c030689dc5/librt-0.15.0-cp314-cp314t-win32.whl", hash = "sha256:6912fa5e635d74529ac7cdb1bdf6ca3af4453da8d1edbe0110ee1cb4ad407ebf", size = 104381, upload-time = "2026-08-07T10:48:21.048Z" }, { url = "https://files.pythonhosted.org/packages/80/b6/134afad262def1de04c0843c376d02135f1168af43f22e09a52bd8394727/librt-0.15.0-cp314-cp314t-win_amd64.whl", hash = "sha256:8e11699ed745931c395acd3621b07062e0f840efa6935aad87a64ed0995f0915", size = 127034, upload-time = "2026-08-07T10:48:22.561Z" }, { url = "https://files.pythonhosted.org/packages/99/5f/1b6846b20572bd699c9e9ec321a5f781845bee477df2aa2a43b28bc40119/librt-0.15.0-cp314-cp314t-win_arm64.whl", hash = "sha256:5d2a91724463bfed4f573cd7a9fdc856d2e230d0c0e5a61416a93481dccd8605", size = 110827, upload-time = "2026-08-07T10:48:23.804Z" }, + { url = "https://files.pythonhosted.org/packages/c6/44/4de9f4ddadb009a55c7758eb5736d62534a7daaf27bd71bc50e64b606b06/librt-0.15.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:8443e38dcfcfdbcf5add5118c623efd788d65ac2e25756d6251a54a06a4d0aca", size = 149843, upload-time = "2026-08-07T10:48:25.148Z" }, + { url = "https://files.pythonhosted.org/packages/1f/eb/5d9ab71e30119c44094e0275f38b47dd327aea0f843a080396677029d508/librt-0.15.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:6d15a29033c57490cfe2069097c6fc4049e4e65ffbb749be7dc453b7c4c68965", size = 154510, upload-time = "2026-08-07T10:48:26.485Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9c/8505d1b8f5e8c19587bd03f7429993b3e9ce5c06819d856bfb11d919374c/librt-0.15.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d2c05c729b589e734c09578bf5964be48a911765484840d017bbc84f49d4c4ad", size = 497543, upload-time = "2026-08-07T10:48:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/1d/9a/3a8390775cb095765aded027ac9c63e7c8ea74e731498607544c6505de0e/librt-0.15.0-cp315-cp315-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:fa60887537e1d0cd2d9982269d33a709bf54b195cd2b9364fc0a758022af5bd9", size = 480452, upload-time = "2026-08-07T10:48:29.531Z" }, + { url = "https://files.pythonhosted.org/packages/e7/40/258a4a7117ee915d66de5cd9b8ade65a440993161107ce3a686f1859955c/librt-0.15.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d8bc24219b24c0af375718942ab75e3544b2763085f40f965be4326734ae8328", size = 507768, upload-time = "2026-08-07T10:48:31.007Z" }, + { url = "https://files.pythonhosted.org/packages/6b/c6/2f4dd296c97a0b85b98894519b279408ec9dd602d4f692b1ea0e25dee670/librt-0.15.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:86a21a7bd3fe3a419512ef424cc1c020f6771d0b29cfddff36d1635a855e63f0", size = 525122, upload-time = "2026-08-07T10:48:32.7Z" }, + { url = "https://files.pythonhosted.org/packages/49/dd/29eab42be13b2bf0ea8cb227135a45d44693e30a7e8b92871981ff56b82b/librt-0.15.0-cp315-cp315-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dbab647e88d90b3167b91efe7091e248653688ed4337e4f90907a722c7361bb9", size = 520371, upload-time = "2026-08-07T10:48:34.294Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/4bad71adeca8fe208b775c2a35417fa5a2584c8f4791daaf89a89450fea1/librt-0.15.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:d8edcf6f550e918dca779c069b9e156385c60b406f99fc7641f32c52f7193659", size = 537258, upload-time = "2026-08-07T10:48:35.88Z" }, + { url = "https://files.pythonhosted.org/packages/4c/63/59dba6143fdcc7240c54458b629f3250000a61b8945890fc9efd451b19c5/librt-0.15.0-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:8b62076030baa2d8b1501a46bf0e19c27a489aa90671c55665bff7887f7660b0", size = 527432, upload-time = "2026-08-07T10:48:37.466Z" }, + { url = "https://files.pythonhosted.org/packages/ec/21/21a24c6a2327d8362580efebe77286bf47b0f4062ec5ea41766e609d3c7d/librt-0.15.0-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:d00d20d1818e82a07a0ee0aa89a98b17ed7916b92441090b683719cb20a59b6d", size = 548108, upload-time = "2026-08-07T10:48:39.384Z" }, + { url = "https://files.pythonhosted.org/packages/5a/6d/fc68c89a7971418b41f9a873623ff935cb864097544c6a2f8ce491c8ef5d/librt-0.15.0-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:4e6ee93fc3cf848dcbf0cce2eca73d8e7dcd0cc2b6df3a529d57750b30a4c55c", size = 529681, upload-time = "2026-08-07T10:48:41.392Z" }, + { url = "https://files.pythonhosted.org/packages/65/7e/c2d98766124400d722063a630b0fde38a9fc768705d37eecca15c47dc192/librt-0.15.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:32896a0af72508ea979e0acb4e4c04cbeeae04938167950d535c83c45597167d", size = 567736, upload-time = "2026-08-07T10:48:43.124Z" }, + { url = "https://files.pythonhosted.org/packages/55/6c/f8c34a95e3a515c6e1c192b89511e7253c89a7760c6b500d57ffdb8d2dc8/librt-0.15.0-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:ec3ba415afaf951f6951b1dd16d3c8e4f540065fc382d7e70b823a79567ca374", size = 81673, upload-time = "2026-08-07T10:48:44.645Z" }, + { url = "https://files.pythonhosted.org/packages/c9/9e/e23fa8e78679ec45728188650b39e8ff476c83b691c96f749217df3b1b7c/librt-0.15.0-cp315-cp315-win32.whl", hash = "sha256:d2813ba2503764f0450680c533d13df7cff9b49df1411062eded5f67db4195b9", size = 100081, upload-time = "2026-08-07T10:48:46.171Z" }, + { url = "https://files.pythonhosted.org/packages/e1/dc/3eb4c5e297343f0620a55532cd7c8d764d3001fa2159212dadf480464827/librt-0.15.0-cp315-cp315-win_amd64.whl", hash = "sha256:b87d67e33afaf265262f2a66db578284b88ee2e6fcd224579cb5c15518677ad8", size = 121228, upload-time = "2026-08-07T10:48:47.631Z" }, + { url = "https://files.pythonhosted.org/packages/97/70/43abce19f04e49762f8ec834c8fafee13cc40fd6b94a72a24e534febfcd0/librt-0.15.0-cp315-cp315-win_arm64.whl", hash = "sha256:713bd7df21170b982e729e46870f31d6b437bd1a9b4648cffb529bd3c2ec5c4b", size = 106487, upload-time = "2026-08-07T10:48:49.095Z" }, + { url = "https://files.pythonhosted.org/packages/de/15/83f2deddb9368b8951ec8c9477269b5b9b8bd9bbf15e57402d0f38817dca/librt-0.15.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:3de789c82752730f94782a5ee518baf9c05edf85733aeaf73bb6e518755cdf54", size = 159448, upload-time = "2026-08-07T10:48:50.649Z" }, + { url = "https://files.pythonhosted.org/packages/06/bf/043097353f9b3c73b583d07f6b8e552795463f4bfc8caf85e42eee50c26a/librt-0.15.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:e0b5deec9a8664eb722c797241970fd4aa1894d25fda36a1ddac0f7407606bd6", size = 161686, upload-time = "2026-08-07T10:48:52.174Z" }, + { url = "https://files.pythonhosted.org/packages/f4/2a/8ae77f9719d42ce71cd708560a3557b38ac3c17a0383e57f87084de45bbe/librt-0.15.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5563302a8359bc2295bb7084d1a8ed1519df96afb30eb2aa4e0bff7b54228988", size = 710668, upload-time = "2026-08-07T10:48:53.782Z" }, + { url = "https://files.pythonhosted.org/packages/61/34/c0436ea134deb9a0d6da80a396a2739a81cb31e0418f7227239e23140898/librt-0.15.0-cp315-cp315t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:22d6263b9d39d7bbb286fa791945646e3218f1be2d693e36fb630f1d0e59cd13", size = 679396, upload-time = "2026-08-07T10:48:55.645Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9f/001e0d99aa9250d5cd5715a9081291a20656083459f9019cda15255329e1/librt-0.15.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:39ffd14646190c454f0d86e0d256b33f00a87a26ab410e619773b841d0e41416", size = 704313, upload-time = "2026-08-07T10:48:57.46Z" }, + { url = "https://files.pythonhosted.org/packages/2d/53/b34fa9d0ff00f136f4d58ebb4c411ff634baed1eb412bb602a2bc8dcafcb/librt-0.15.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c47318cd3a61401452de11282242937e3e057c4fd3dbaf601e269d0928a06c0a", size = 729847, upload-time = "2026-08-07T10:48:59.231Z" }, + { url = "https://files.pythonhosted.org/packages/86/ac/fa4d7a424665040e95baf480a6d523446057684b6758624c85338e8a23b2/librt-0.15.0-cp315-cp315t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a56a1d4f859a82ca5b99fc4b82c9b027b15e3c455c5cd99e7d0719f27bb20b6c", size = 742736, upload-time = "2026-08-07T10:49:01.151Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f1/e17a9bb5de6fb8c3186ed1a7d68d21618b027ac2d3633e03d3b6109c67ae/librt-0.15.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:077471b3182db4e17c36ae91555f36a4d2c00080b267f749bcad34a478a9a302", size = 763454, upload-time = "2026-08-07T10:49:03.039Z" }, + { url = "https://files.pythonhosted.org/packages/1d/ec/ecd02cd30935b931b9cdbfed6ab5a099c51b280b4e7baa274da80978ed27/librt-0.15.0-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:411ca4d1b905b860ceba7570dd6717a71dedaddcc4b0f77ece710aa41ee11f8d", size = 743296, upload-time = "2026-08-07T10:49:04.941Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b5/b3c2b8353ce820a4854f78d19321344242f89fa71c975b71132ba9bf242a/librt-0.15.0-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:1256589e0b0adb31751d685a68bce29d73407ddf4ef05d4188f49d5dcf9566d9", size = 756217, upload-time = "2026-08-07T10:49:06.825Z" }, + { url = "https://files.pythonhosted.org/packages/3c/52/6cc22542ba59146b05cca2a656f9ff8bb67e38e63d12c3b0cc183d837bf1/librt-0.15.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:f42b74a53e5f26a0ba0007411a7455b66c67ce4022a39cc1f56fc4efd65bcbab", size = 741934, upload-time = "2026-08-07T10:49:08.839Z" }, + { url = "https://files.pythonhosted.org/packages/40/32/a04b72b1aa86e3be23b2ecff8c1aad2dcc955bd3956d6d26e7e34267e57a/librt-0.15.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:291bf73caf78b9e88d6fae9bfd693207ff7d832e2fdbe2cf8e746bc13f5f892b", size = 783763, upload-time = "2026-08-07T10:49:10.661Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f0/89eb11dffbe9279ff37144dec786927314502ae0b114f1449dc78c458aab/librt-0.15.0-cp315-cp315t-win32.whl", hash = "sha256:c16d15ee371643ab48dc8248a3e680ebbeca573a13af2c3dd0c985b142d77162", size = 104313, upload-time = "2026-08-07T10:49:12.305Z" }, + { url = "https://files.pythonhosted.org/packages/6d/4a/1f1978c200f563beda63c36adff2d65bbecb81e365e8e69e572f5f70fbc6/librt-0.15.0-cp315-cp315t-win_amd64.whl", hash = "sha256:dbd605739f228912dc49027cb764456b9757750bdc2b6b7773164db7096c6fd1", size = 126889, upload-time = "2026-08-07T10:49:13.881Z" }, + { url = "https://files.pythonhosted.org/packages/38/a6/800800bfed7b1fb10fc3f3d557785c3854e80d3f7a9800d784b176a1fc2d/librt-0.15.0-cp315-cp315t-win_arm64.whl", hash = "sha256:84d244b00604d17df3fc7736c327892d6bba66181254aa4087be807b6c342bdc", size = 110700, upload-time = "2026-08-07T10:49:15.499Z" }, ] [[package]] @@ -4649,7 +4930,7 @@ dependencies = [ { name = "boto3" }, { name = "certifi" }, { name = "click" }, - { name = "fastuuid" }, + { name = "fastuuid", marker = "python_full_version < '3.15'" }, { name = "filelock" }, { name = "httpx", extra = ["http2"] }, { name = "huggingface-hub" }, @@ -4658,13 +4939,15 @@ dependencies = [ { name = "jsonschema" }, { name = "openai" }, { name = "packaging" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "pydantic-settings" }, { name = "python-dotenv" }, { name = "pyyaml" }, { name = "tiktoken" }, { name = "tokenizers" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] [package.optional-dependencies] @@ -4703,7 +4986,8 @@ grpc = [ mcp = [ { name = "httpx2" }, { name = "mcp" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] mlflow = [ { name = "mlflow" }, @@ -4721,7 +5005,8 @@ proxy = [ { name = "fastapi-sso" }, { name = "granian" }, { name = "gunicorn" }, - { name = "hiredis" }, + { name = "hiredis", version = "3.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "hiredis", version = "3.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "httpx2" }, { name = "inquirerpy" }, { name = "litellm-enterprise" }, @@ -4729,7 +5014,8 @@ proxy = [ { name = "mcp" }, { name = "orjson" }, { name = "polars" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "pyjwt" }, { name = "pynacl" }, { name = "pyroscope-io", marker = "sys_platform != 'win32'" }, @@ -4927,7 +5213,7 @@ requires-dist = [ { name = "expression", marker = "extra == 'proxy'", specifier = ">=5.6.0,<6.0" }, { name = "fastapi", marker = "extra == 'proxy'", specifier = ">=0.136.3,<1.0" }, { name = "fastapi-sso", marker = "extra == 'proxy'", specifier = ">=0.19.0,<1.0" }, - { name = "fastuuid", specifier = ">=0.14.0,<1.0" }, + { name = "fastuuid", marker = "python_full_version < '3.15'", specifier = ">=0.14.0,<1.0" }, { name = "filelock", specifier = ">=3.16.1,<4.0" }, { name = "google-cloud-aiplatform", marker = "extra == 'google'", specifier = ">=1.133.0,<2.0" }, { name = "google-cloud-aiplatform", marker = "extra == 'proxy-runtime'", specifier = ">=1.133.0,<2.0" }, @@ -4936,11 +5222,13 @@ requires-dist = [ { name = "google-cloud-speech", marker = "extra == 'proxy-runtime'", specifier = ">=2.40.0,<3.0" }, { name = "google-cloud-speech", marker = "extra == 'stt-vertex-chirp'", specifier = ">=2.40.0,<3.0" }, { name = "google-genai", marker = "extra == 'proxy-runtime'", specifier = ">=1.37.0,<2.0" }, - { name = "granian", marker = "extra == 'proxy'", specifier = ">=2.7.4,<3.0" }, + { name = "granian", marker = "python_full_version >= '3.15' and extra == 'proxy'", specifier = ">=2.8.0,<3.0" }, + { name = "granian", marker = "python_full_version < '3.15' and extra == 'proxy'", specifier = ">=2.7.4,<3.0" }, { name = "grpcio", marker = "extra == 'grpc'", specifier = "==1.84.1" }, { name = "grpcio", marker = "extra == 'proxy-runtime'", specifier = "==1.84.1" }, { name = "gunicorn", marker = "extra == 'proxy'", specifier = ">=23.0.0,<24.0" }, - { name = "hiredis", marker = "extra == 'proxy'", specifier = ">=3.0.0,<4.0" }, + { name = "hiredis", marker = "python_full_version >= '3.15' and extra == 'proxy'", specifier = ">=3.5.0,<4.0" }, + { name = "hiredis", marker = "python_full_version < '3.15' and extra == 'proxy'", specifier = ">=3.0.0,<4.0" }, { name = "httpx", extras = ["http2"], specifier = ">=0.28.0,<1.0" }, { name = "httpx2", marker = "extra == 'mcp'", specifier = ">=2.10.0,<3" }, { name = "httpx2", marker = "extra == 'proxy'", specifier = ">=2.10.0,<3" }, @@ -4967,7 +5255,8 @@ requires-dist = [ { name = "opentelemetry-exporter-otlp", marker = "extra == 'proxy-runtime'", specifier = "==1.45.1" }, { name = "opentelemetry-instrumentation-fastapi", marker = "extra == 'proxy-runtime'", specifier = "==0.66b1" }, { name = "opentelemetry-sdk", marker = "extra == 'proxy-runtime'", specifier = "==1.45.1" }, - { name = "orjson", marker = "extra == 'proxy'", specifier = ">=3.11.6,<4.0" }, + { name = "orjson", marker = "python_full_version >= '3.15' and extra == 'proxy'", specifier = ">=3.12.0,<4.0" }, + { name = "orjson", marker = "python_full_version < '3.15' and extra == 'proxy'", specifier = ">=3.11.6,<4.0" }, { name = "packaging", specifier = ">=24.0" }, { name = "polars", marker = "extra == 'proxy'", specifier = ">=1.38.1,<2.0" }, { name = "prisma", marker = "extra == 'extra-proxy'", specifier = ">=0.11.0,<1.0" }, @@ -4975,7 +5264,8 @@ requires-dist = [ { name = "psycopg", marker = "extra == 'extra-proxy'", specifier = ">=3.2,<4.0" }, { name = "psycopg-binary", marker = "extra == 'extra-proxy'", specifier = ">=3.2,<4.0" }, { name = "pydantic", marker = "python_full_version < '3.14'", specifier = ">=2.11.0,<3.0.0" }, - { name = "pydantic", marker = "python_full_version >= '3.14'", specifier = ">=2.12.0,<3.0.0" }, + { name = "pydantic", marker = "python_full_version == '3.14.*'", specifier = ">=2.12.0,<3.0.0" }, + { name = "pydantic", marker = "python_full_version >= '3.15'", specifier = ">=2.14.0,<3.0.0" }, { name = "pydantic", marker = "extra == 'mcp'", specifier = ">=2.12.0,<3" }, { name = "pydantic", marker = "extra == 'proxy'", specifier = ">=2.12.0,<3" }, { name = "pydantic-settings", specifier = ">=2.14.1,<3.0" }, @@ -5002,7 +5292,8 @@ requires-dist = [ { name = "soundfile", marker = "extra == 'stt-nvidia-riva'", specifier = ">=0.12.1" }, { name = "starlette", marker = "extra == 'proxy'", specifier = ">=1.0.1,<2.0" }, { name = "tiktoken", marker = "python_full_version < '3.14'", specifier = ">=0.8.0,<1.0" }, - { name = "tiktoken", marker = "python_full_version >= '3.14'", specifier = ">=0.12.0,<1.0" }, + { name = "tiktoken", marker = "python_full_version == '3.14.*'", specifier = ">=0.12.0,<1.0" }, + { name = "tiktoken", marker = "python_full_version >= '3.15'", specifier = ">=0.14.0,<1.0" }, { name = "tokenizers", specifier = ">=0.21.0,<1.0" }, { name = "tomlkit", marker = "extra == 'cli'", specifier = ">=0.13.3,<1.0" }, { name = "tomlkit", marker = "extra == 'proxy'", specifier = ">=0.13.3,<1.0" }, @@ -5069,8 +5360,8 @@ dev = [ { name = "opentelemetry-instrumentation-fastapi", specifier = "==0.66b1" }, { name = "opentelemetry-sdk", specifier = "==1.45.1" }, { name = "parameterized", specifier = "==0.9.0" }, - { name = "psycopg", specifier = "==3.3.3" }, - { name = "psycopg-binary", specifier = "==3.3.3" }, + { name = "psycopg", specifier = "==3.3.6" }, + { name = "psycopg-binary", specifier = "==3.3.6" }, { name = "pytest", specifier = "==9.0.3" }, { name = "pytest-asyncio", specifier = "==1.3.0" }, { name = "pytest-cov", specifier = "==5.0.0" }, @@ -5169,7 +5460,8 @@ name = "llm-sandbox" version = "0.3.39" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fb/f1/30b41350f105dbd794639565a1f6e4ab28f95940fc2907cb721471dca105/llm_sandbox-0.3.39.tar.gz", hash = "sha256:7452317a054df3ac20fb652a2e100b5697624886282874fecfb738bac21b27bb", size = 611034, upload-time = "2026-04-20T16:59:45.133Z" } wheels = [ @@ -5196,7 +5488,7 @@ dependencies = [ { name = "pyzmq" }, { name = "requests" }, { name = "tomli", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "werkzeug" }, ] sdist = { url = "https://files.pythonhosted.org/packages/03/4e/62c3fee4af4720aceac2c102a6a1d69707b7b28e83fc44b82581e86155d7/locust-2.45.0.tar.gz", hash = "sha256:9fa840a4ef8c2624d7719f6e642bce3e19079aa4eea755c99c5282087903e13a", size = 1477628, upload-time = "2026-07-09T08:20:46.931Z" } @@ -5219,7 +5511,8 @@ dependencies = [ { name = "requests" }, { name = "rich" }, { name = "tomli", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/56/5a/e6cccc98d30358b8beee47c61c2a1ca66ac7a76a07de9ba333aa5219b9a6/logfire-5.2.0.tar.gz", hash = "sha256:5fb9ddda303759766e86f5704a1ae96f3a0072906d52e21fbaad2e3c3b03f823", size = 1404990, upload-time = "2026-10-13T17:36:30.746Z" } wheels = [ @@ -5239,7 +5532,7 @@ dependencies = [ { name = "jsonpickle" }, { name = "langchain-community", version = "0.3.31", source = { registry = "https://pypi.org/simple" } }, { name = "packaging" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" } }, { name = "pyhumps" }, { name = "requests" }, { name = "setuptools" }, @@ -5255,7 +5548,8 @@ name = "lunary" version = "1.4.37" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", @@ -5266,7 +5560,8 @@ dependencies = [ { name = "jsonpickle" }, { name = "langchain-community", version = "0.4.1", source = { registry = "https://pypi.org/simple" } }, { name = "packaging" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "pyhumps" }, { name = "requests" }, { name = "setuptools" }, @@ -5412,7 +5707,8 @@ name = "mangum" version = "0.17.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a6/3d/c9664d56ad6b32d7c7ebf82a77cacc290ebed033f96adf0c98a9114ace4e/mangum-0.17.0.tar.gz", hash = "sha256:5b4e26375e12eed051687670466d17968f8b74beecaca432edd4eb4127f78509", size = 14222, upload-time = "2022-11-27T11:21:42.084Z" } wheels = [ @@ -5623,14 +5919,17 @@ dependencies = [ { name = "jsonschema" }, { name = "mcp-types" }, { name = "opentelemetry-api" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "pyjwt", extra = ["crypto"] }, { name = "python-multipart" }, { name = "pywin32", marker = "sys_platform == 'win32'" }, { name = "sse-starlette" }, { name = "starlette" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, + { name = "typing-inspection", version = "0.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-inspection", version = "0.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9d/8d/e0d339616f4810e9051d4aba6887afab289ab1f81875fe908b606cdfd0e3/mcp-2.3.0.tar.gz", hash = "sha256:8b147a50441cf059dc88c684e0aeed3687f0aa0f39c6cde7b90330effd2b34d8", size = 4171955, upload-time = "2026-10-02T22:06:56.091Z" } @@ -5643,8 +5942,10 @@ name = "mcp-types" version = "2.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydantic" }, - { name = "typing-extensions" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9e/2d/7c251e34207f6c51000312fc8839111ac45cfe02023f90b44e7f1051dd8e/mcp_types-2.3.0.tar.gz", hash = "sha256:d1e46549edb35ee19a94940fcee6d1addd7e589ab7ea92dda83f5d84781fc362", size = 65898, upload-time = "2026-10-02T22:06:57.863Z" } wheels = [ @@ -5736,13 +6037,15 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "packaging" }, { name = "protobuf" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "python-dotenv" }, { name = "pyyaml" }, { name = "requests" }, { name = "sqlparse" }, { name = "starlette" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "uvicorn" }, ] sdist = { url = "https://files.pythonhosted.org/packages/26/b0/efbc07a721df135c681dd9f29ecb44b42e9664125f82a41168bc38240ed8/mlflow_skinny-3.17.2.tar.gz", hash = "sha256:5f2a0812d019a8f517a58f1377922f75af4f42832be30c8f1441005b51288596", size = 3488534, upload-time = "2026-10-21T02:07:10.993Z" } @@ -5762,7 +6065,8 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "packaging" }, { name = "protobuf" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/af/81/bf76610e755643d454fe1eb52fd6aae99d2e7d472a4a4cd119fa11a7a653/mlflow_tracing-3.17.2.tar.gz", hash = "sha256:35da9f6cc450feae7173941d0d119391f6fe3a41973c0a28ae4a021e6e285c99", size = 1688723, upload-time = "2026-10-21T02:07:48.333Z" } wheels = [ @@ -5891,7 +6195,7 @@ name = "multidict" version = "6.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d6/99/1d4d69c3512d0ddbfa3a1b69cfd9a151012ab2eb4eabbb096201b1f0b7d8/multidict-6.9.1.tar.gz", hash = "sha256:0f06e60fa190aa7abd0914c2a766736fdc8e9f34878c4346338534b73d1b20e2", size = 182404, upload-time = "2026-09-21T17:59:05.362Z" } wheels = [ @@ -6021,6 +6325,49 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/29/f2/f0ff3384a756227741c72f4ee94cadf1f9900df0ae961e48639f96115b1c/multidict-6.9.1-cp314-cp314t-win32.whl", hash = "sha256:35ba0263bae5dd3ad5aad767cc9afc01a8598c7dae30f1b3b2de98b1b32c28bd", size = 57776, upload-time = "2026-09-21T17:57:32.835Z" }, { url = "https://files.pythonhosted.org/packages/8d/87/013e1eed30ad46bec1eb68947bc9caf55ba2fd0d6aed04fe964e3b44c23b/multidict-6.9.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7c8d5882ba25ac0282258be435d8a05aa0cbcacfce15799154a338f847f159b9", size = 64881, upload-time = "2026-09-21T17:57:34.489Z" }, { url = "https://files.pythonhosted.org/packages/7f/95/64dd049bb4c53426e2fee7afb874daaa8b4426c41878785883a94511ecf8/multidict-6.9.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ac348379cf4de5538a0a213be1532d289aa801ec5d267c5909446b9ea2f8e2c3", size = 58582, upload-time = "2026-09-21T17:57:36.19Z" }, + { url = "https://files.pythonhosted.org/packages/a2/2c/26bc1723592cfe965b575652a7749026474b0625b04b04564e1fba265dc5/multidict-6.9.1-cp315-cp315-android_24_x86_64.whl", hash = "sha256:f04551dce5a7db8c9659f2e4245494c182d0663b83661803e08d46bfcae5eda1", size = 62339, upload-time = "2026-09-21T17:57:37.967Z" }, + { url = "https://files.pythonhosted.org/packages/bd/c5/614d2eb9995232d066aae7f17d1d454060b6b5d882b22fbaf6150b05ef1d/multidict-6.9.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c0c88085affd35c33e124e36930c5ad96aff9294ce195eaa0fd9cec962b64a82", size = 55009, upload-time = "2026-09-21T17:57:39.746Z" }, + { url = "https://files.pythonhosted.org/packages/7f/24/f7179ecdf9cdd3581ace3292c3b883a1a500a374aa4e619bf6eb521c62e5/multidict-6.9.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a8b75dd3d3638d9a19f23e84af4ffab3b8940422c0df2da2a77005ef5aa3d7ea", size = 56273, upload-time = "2026-09-21T17:57:41.544Z" }, + { url = "https://files.pythonhosted.org/packages/ac/72/c9dffcecccd1f8b9a58f960ec4eaecab62c4edb08d5bb984d4de67fe3fdb/multidict-6.9.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:006c4478de0a1876f4834e14255776286f09b9846b505fe63f67f9d173a9487c", size = 96390, upload-time = "2026-09-21T17:57:43.385Z" }, + { url = "https://files.pythonhosted.org/packages/e9/3f/76542360e655cecdb7c158c7af88c69c4db0d2aac3421659d41214927229/multidict-6.9.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:2784090c30a586d5b45197bd9c32f87fb927216cde302f6cfd76d76577e90f08", size = 58900, upload-time = "2026-09-21T17:57:45.105Z" }, + { url = "https://files.pythonhosted.org/packages/04/15/f28c71af461dd4ff15653a165cfd8ddaf01f3387ff80d2dd0de057d527d0/multidict-6.9.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:7c708566da8014b120a64b1eb6d200c6c0c8cb36296383723cdb6fc82038270b", size = 56540, upload-time = "2026-09-21T17:57:46.913Z" }, + { url = "https://files.pythonhosted.org/packages/19/de/5ee4050c3ddad88c605a89fc1cd18e89abd40a64f183e6fb5276b6eb7f52/multidict-6.9.1-cp315-cp315-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:042fb0196047e786936a730bd302de83143950da45f2c16078da8f35e1cf7919", size = 331818, upload-time = "2026-09-21T17:57:48.825Z" }, + { url = "https://files.pythonhosted.org/packages/70/e2/2d4e3c3ee70f583334ff236fe6a3babf7a05ce56169a15df65453af647c0/multidict-6.9.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2212a0c842c723d919ea4a22a9296cb6b244b386e4e6ea92adfc7fbf3095519", size = 329800, upload-time = "2026-09-21T17:57:50.91Z" }, + { url = "https://files.pythonhosted.org/packages/d6/14/f64bd05667343a7250cdd99f81bda2270e3959f00832441f10c7f904e1c1/multidict-6.9.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fd882aa29bf402b62bf1fd7c19fd5df4b6528cf468a908864b39368572b662a9", size = 312048, upload-time = "2026-09-21T17:57:53.053Z" }, + { url = "https://files.pythonhosted.org/packages/64/bf/a91256ce00199e7544e3eda79c4e9544eb7388c38fa22f11638c2e357ccb/multidict-6.9.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c57541034d12b215ab0a2bfa371d1a8a198da18176d0426c27105b9161a6862d", size = 343676, upload-time = "2026-09-21T17:57:55.069Z" }, + { url = "https://files.pythonhosted.org/packages/14/48/628b978413518159228c2b2ec38645362ceb6427e935d83d716e26490a49/multidict-6.9.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d73fed4e37158ff00cd271871170b79e40138db51e625fd352fa17c6acb34f67", size = 347028, upload-time = "2026-09-21T17:57:57.157Z" }, + { url = "https://files.pythonhosted.org/packages/8c/73/5f9dc2b5ac7dd7a82458ae9cd283b99c185834f23442ace295362dcc5e45/multidict-6.9.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:79da2491348b30810728050b4a8ec0416f85884125c2fd44655b3d01150d9c3e", size = 335670, upload-time = "2026-09-21T17:57:59.126Z" }, + { url = "https://files.pythonhosted.org/packages/f8/84/6f319ee000f5ab3fa0055b194ad706fddd552c60c76a21d581c87db1d46c/multidict-6.9.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b4da208a63434d21a3df64d29758e650fc4aa8cb05848554b76949c296539cca", size = 307804, upload-time = "2026-09-21T17:58:01.2Z" }, + { url = "https://files.pythonhosted.org/packages/3f/24/61be5fe1616d033532e8d63dc789c0f35b9793917834713eefe050589ba9/multidict-6.9.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:877ca17fdcfdf5c397493a71e5ff97a87bb181417fe717fdadc77c08c09301ac", size = 323409, upload-time = "2026-09-21T17:58:03.383Z" }, + { url = "https://files.pythonhosted.org/packages/22/a6/eab8aaf59892169a0f4e8e78a5516a6caa6abd8e96e6410111258f4ef0ef/multidict-6.9.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:40aec5299e1ed71fbb988389059da381c3c1a60e0c649acceb2a35d9b128848e", size = 315746, upload-time = "2026-09-21T17:58:05.372Z" }, + { url = "https://files.pythonhosted.org/packages/42/b1/d1e14bab80a74babf6e2ac971a878b84fdcf3a972f3ec3400ca12260fac5/multidict-6.9.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:4a409ccc42aefec904038695d5d7fd6d8f3af2721b7b6401d55135ec7d6d298e", size = 334529, upload-time = "2026-09-21T17:58:07.35Z" }, + { url = "https://files.pythonhosted.org/packages/f5/50/a05b1d36f6d3363a5a334cebbf11624c9c7a69d66fd23345eb53b1d43b7e/multidict-6.9.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:0a5769559e3312dd96731fbe15b4abb6033368ac1cad5a98dadd21946a4c7d6c", size = 336455, upload-time = "2026-09-21T17:58:09.112Z" }, + { url = "https://files.pythonhosted.org/packages/cd/0d/bf33cb097380252b08e1177b5ae0cab822c47ce80d236f3f724953c76721/multidict-6.9.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:03fac50ddfd8302175b77863a015eccfae76767cda5506eef86df559ba861e1f", size = 305865, upload-time = "2026-09-21T17:58:10.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/ee/a2bd133391204b1cec5a5caf327dc9a06710439b7403220860a8fb164560/multidict-6.9.1-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:5ecace251ccfa705bf3d7d35c5032cf750f5c629809740405697bce5c118c4a4", size = 337531, upload-time = "2026-09-21T17:58:12.846Z" }, + { url = "https://files.pythonhosted.org/packages/a8/32/c0972486f81a06c3bbb23063c85afaac53c51608f7e35bcb101a1006408b/multidict-6.9.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:ecc89dbd4155b2f8a47f4bbd89242a35ed15e8e0ec581cab5ac65fa38407329d", size = 332464, upload-time = "2026-09-21T17:58:14.673Z" }, + { url = "https://files.pythonhosted.org/packages/69/4f/5cbbe852b45d8a7f0ab3ed042386e4894e463010eb230603596cb15d0fee/multidict-6.9.1-cp315-cp315-win32.whl", hash = "sha256:a3ffe881246d28a862f1985824f484cb7361f44d6b99c4c620436ef56462f38d", size = 51714, upload-time = "2026-09-21T17:58:16.409Z" }, + { url = "https://files.pythonhosted.org/packages/d4/77/1fa118386deeae4e7a48870fa5cb073e748b0ac5265a6990651cefc48068/multidict-6.9.1-cp315-cp315-win_amd64.whl", hash = "sha256:59c123d0e948d760a5f930f316cfefa07e8d632ab84327c0693ee6a88171154f", size = 58619, upload-time = "2026-09-21T17:58:18.369Z" }, + { url = "https://files.pythonhosted.org/packages/dd/ee/51c45748acfce9cb69d89dcfcd09c874b385407fd6993e885d6905659865/multidict-6.9.1-cp315-cp315-win_arm64.whl", hash = "sha256:31199204b3ced121ff5407a2c342326a5d27e3870abbf94bd80dbd2451b7bc8f", size = 53969, upload-time = "2026-09-21T17:58:20.102Z" }, + { url = "https://files.pythonhosted.org/packages/aa/57/89d2d3dedfae2558853c1770c7cbcdfd5211eb4c4090b1e8cb570544a490/multidict-6.9.1-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:8879510a76940670517ea1cb589978da44b86e286ec3e50d664ef330817afce7", size = 107679, upload-time = "2026-09-21T17:58:21.98Z" }, + { url = "https://files.pythonhosted.org/packages/c8/df/0c2a28870181c1762f90952b897f6db46c934ce073efe0442604cbc276db/multidict-6.9.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:aec65b53a07f580606593f877eefbb29a45939bfc0d3fe6e6d9f42b41b749f68", size = 64753, upload-time = "2026-09-21T17:58:23.622Z" }, + { url = "https://files.pythonhosted.org/packages/43/7e/9c6a3e7619459407d8958f7e56ab3dda7f0738f5aed9bb4ad48076e8de83/multidict-6.9.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:14c56f73e78faa1f68bbb826197cd5871994e70e841b8590829c35912ece5c64", size = 62732, upload-time = "2026-09-21T17:58:25.46Z" }, + { url = "https://files.pythonhosted.org/packages/38/eb/fd56c9d83ba0cdc3cf66f9acb278b24ef1f043b620b96d145685a3b57338/multidict-6.9.1-cp315-cp315t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:38c9986f9ce50c459b10de216a05f4bd7ed5ac63887d56e500a52bb464b861ce", size = 288648, upload-time = "2026-09-21T17:58:27.115Z" }, + { url = "https://files.pythonhosted.org/packages/df/ef/51e79c2442b0f56ac4ac580108d713078b31297e028d468fc220971a305d/multidict-6.9.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:15db6a102cbaf1949cf028ecf080aac76d20bcd29ad4e092574db6c6b7af78a5", size = 304919, upload-time = "2026-09-21T17:58:29.143Z" }, + { url = "https://files.pythonhosted.org/packages/6f/e4/774700cc5739353fe88a7204ed30e78a9403d98d4bb2496cc91f096d2e71/multidict-6.9.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:bd82c4977196681a499bb6ca9e462afbc5c91c1c15b6a991dfbd72733fea4dd2", size = 289094, upload-time = "2026-09-21T17:58:30.938Z" }, + { url = "https://files.pythonhosted.org/packages/ca/41/0bec327bc3a8271825376b6d8a469d1f80e3ba1e2e730e019836e29663c3/multidict-6.9.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bb58ba73a3f96f9a3e46b1fab69929d7edbbddb3133ee74b5c3c54074a53c4f1", size = 312007, upload-time = "2026-09-21T17:58:32.964Z" }, + { url = "https://files.pythonhosted.org/packages/c3/7a/9906bf5f1fe20e8e755affccc1a69ecd8aebb079ac65386e5ffc8eefbe75/multidict-6.9.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7c6eecfab7ce4cd9487ff8ba936fe38cdfd68c04faf3d5c710363c6a8e695659", size = 314657, upload-time = "2026-09-21T17:58:35.285Z" }, + { url = "https://files.pythonhosted.org/packages/33/92/cf6d665a45663bbe216420fe632da10e9f4f842b07ce02bc3ddf8eb882b8/multidict-6.9.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c62e71e6289d78c0108d8aeb495f8bd3cad4bc1632fedddc9297ddf287ecc20", size = 301563, upload-time = "2026-09-21T17:58:37.472Z" }, + { url = "https://files.pythonhosted.org/packages/97/4c/9f1b7945526905d58b4b2d422e2ea81326dda71f9bdc9c32c8002639234a/multidict-6.9.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:539c2cd5fed0947c135cd7eabaaac55f48300dfa1de0f3ca4edb5efa6606f471", size = 278540, upload-time = "2026-09-21T17:58:39.551Z" }, + { url = "https://files.pythonhosted.org/packages/5c/d9/133aa472fc1be181de29a0c2e1c4cd778280882f532b404395f646742e46/multidict-6.9.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:3100d169ceb7bc8f05f89a6db11d1b21f119975fc26f29dc45a472e3569f0879", size = 294858, upload-time = "2026-09-21T17:58:41.791Z" }, + { url = "https://files.pythonhosted.org/packages/bd/38/21f2305dd20ac5038fa581b5d7e0d8dbbd70dc6b50231c22ec173dacddda/multidict-6.9.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:696477ad71385c4795e3b8e4cf10b0d2c28c2a1ca6a955e031cb1e62993e9ee3", size = 285461, upload-time = "2026-09-21T17:58:43.844Z" }, + { url = "https://files.pythonhosted.org/packages/ad/21/465a43c214d1d7e2d0c974e727d4b5935f8a0868838b5df9d0984c83c1dc/multidict-6.9.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:f5844e7befc707367807586f550fa97e23dcfef0728f02b98c7bc498a961a5df", size = 288964, upload-time = "2026-09-21T17:58:46.16Z" }, + { url = "https://files.pythonhosted.org/packages/2d/58/72a3e8d56c1e05146d2d1841dbaacf8986375eded1e5a58d30eef191a580/multidict-6.9.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:abc7c2e4b47bfe6a9aea434d3fdebb9597ee636e914e92ba352f2068b9142f4c", size = 303924, upload-time = "2026-09-21T17:58:48.395Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b9/911084c04530a682a54536b4f738d3992a593b5ff6fea1d6a1f02f3acc6b/multidict-6.9.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:7ab379f95caee071a37cbd8be86d4c65accc651d391f9f97748fef006f38769c", size = 278479, upload-time = "2026-09-21T17:58:50.494Z" }, + { url = "https://files.pythonhosted.org/packages/45/eb/9a333346c02c928a9e82534638bce85697c10c46b4901fcd70af19357356/multidict-6.9.1-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:557a4e1708df428ebe6c3081c83a273d275dad2446ce0d81fc648ac71afdb18d", size = 301650, upload-time = "2026-09-21T17:58:52.554Z" }, + { url = "https://files.pythonhosted.org/packages/67/2f/1d936c4e45c8199db6082b6836e9182e31aa393de0de652637790831ca4b/multidict-6.9.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:475d04d5192eba487a3e2f935976340baa24529046e9c1c9c7a3b7bf80445ae1", size = 298024, upload-time = "2026-09-21T17:58:55.03Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0c/ca62ae2882b89a4cfeb0bbca873a4afdb5fc32994c4f8a37a7cb52ede7a0/multidict-6.9.1-cp315-cp315t-win32.whl", hash = "sha256:10083a8a0f4e1b26b599889e90b9802504ce5d3f7722f925bbb7ca47dd22a7c1", size = 57810, upload-time = "2026-09-21T17:58:57.141Z" }, + { url = "https://files.pythonhosted.org/packages/02/8b/b2c2805c3eb6fa4cdf5512fb95ac6174dde71e34dd601c5be52f4df5dc82/multidict-6.9.1-cp315-cp315t-win_amd64.whl", hash = "sha256:e96ca64383efa107262ee3949f047af5ee4f1845ba09463466c04d35a83bd3ec", size = 64984, upload-time = "2026-09-21T17:58:59.446Z" }, + { url = "https://files.pythonhosted.org/packages/fd/34/03f1d204d698c408f4754c969882609b6b7a902c0e3d16a7033f219cdeda/multidict-6.9.1-cp315-cp315t-win_arm64.whl", hash = "sha256:501ed8b02a5990c67a91c732843609d43a6be1f7576fcdfc867331239f37fbd3", size = 58742, upload-time = "2026-09-21T17:59:01.726Z" }, { url = "https://files.pythonhosted.org/packages/be/59/e26cb779be4c591d1a910f59d29aca9fba4de70349840a833beba2652371/multidict-6.9.1-py3-none-any.whl", hash = "sha256:7bf6478188f4e47bf5686e8a33da4ae28bf43b1b2528d9ee144d28492bfac60b", size = 19176, upload-time = "2026-09-21T17:59:03.501Z" }, ] @@ -6033,7 +6380,8 @@ dependencies = [ { name = "mypy-extensions" }, { name = "pathspec" }, { name = "tomli", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/3d/5b373635b3146264eb7a68d09e5ca11c305bbb058dfffbb47c47daf4f632/mypy-1.20.1.tar.gz", hash = "sha256:6fc3f4ecd52de81648fed1945498bf42fa2993ddfad67c9056df36ae5757f804", size = 3815892, upload-time = "2026-04-13T02:46:51.474Z" } wheels = [ @@ -6157,7 +6505,8 @@ name = "numpy" version = "2.4.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", ] @@ -6282,10 +6631,12 @@ dependencies = [ { name = "distro" }, { name = "httpx" }, { name = "jiter" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "sniffio" }, { name = "tqdm" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f0/ee/d056c82f63c05f06baac0cffb4a90952d8274f90c49dfe244f20497b9bbd/openai-2.33.0.tar.gz", hash = "sha256:f850c435e2a4685bba3295bd54912dd26315d9c1b7733068186134d6e0599f9a", size = 693254, upload-time = "2026-04-28T14:04:42.428Z" } wheels = [ @@ -6303,7 +6654,8 @@ dependencies = [ { name = "more-itertools" }, { name = "openapi-schema-validator" }, { name = "openapi-spec-validator" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "werkzeug" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fd/65/ee75f25b9459a02df6f713f8ffde5dacb57b8b4e45145cde4cab28b5abba/openapi_core-0.22.0.tar.gz", hash = "sha256:b30490dfa74e3aac2276105525590135212352f5dd7e5acf8f62f6a89ed6f2d0", size = 109242, upload-time = "2025-12-22T19:19:49.608Z" } @@ -6345,7 +6697,8 @@ name = "opentelemetry-api" version = "1.45.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2e/02/6e0ae9cc61bd3169d401077b507b3ebc344745171e1051ab430be012dcd9/opentelemetry_api-1.45.1.tar.gz", hash = "sha256:aa38ed19bcc084ba42782a73255b3582283eced7ad6dddbd6695189e69adfb75", size = 72804, upload-time = "2026-10-06T17:32:58.133Z" } wheels = [ @@ -6418,7 +6771,8 @@ dependencies = [ { name = "opentelemetry-exporter-otlp-proto-common" }, { name = "opentelemetry-proto" }, { name = "opentelemetry-sdk" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d6/00/a82af0be959dc58495740b169c6669a86e0811f6cd353a01eda34d255db3/opentelemetry_exporter_otlp_proto_grpc-1.45.1.tar.gz", hash = "sha256:3b3dcfbfdcb4e35149fcf309972282054b45228f5c10547d0095d6578510a9a0", size = 25997, upload-time = "2026-10-06T17:33:05.114Z" } wheels = [ @@ -6438,7 +6792,8 @@ dependencies = [ { name = "opentelemetry-proto" }, { name = "opentelemetry-sdk" }, { name = "requests" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1b/17/26487707ea4caa97b17e6e4b5fa72133a53512ffa2f5cf7a49ef284b29cb/opentelemetry_exporter_otlp_proto_http-1.45.1.tar.gz", hash = "sha256:45c218405ce3fd879596924b1874bf9a8f6880206d61065c5a912c8e5c297fb7", size = 28839, upload-time = "2026-10-06T17:33:05.713Z" } wheels = [ @@ -6964,7 +7319,8 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, { name = "opentelemetry-semantic-conventions" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a1/79/7392e21a1c8f0c61d90b223e31c7e48cb9d452e91a6b820ad24cca5f23c4/opentelemetry_sdk-1.45.1.tar.gz", hash = "sha256:63d24a6ca645019a631e6a51999c73e93adcac1196ca640b8ae78a7cc4762bf3", size = 218324, upload-time = "2026-10-06T17:33:13.26Z" } wheels = [ @@ -6977,7 +7333,8 @@ version = "0.66b1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/46/e4/dbbfb2a010c4db2224a5114638acede6fe563d33cc20fb1752cebcbe6298/opentelemetry_semantic_conventions-0.66b1.tar.gz", hash = "sha256:497ca63bf383723411e8eaf60c8779e9877633c936bb641080adab59d0eb6ec8", size = 150250, upload-time = "2026-10-06T17:33:14.073Z" } wheels = [ @@ -7004,83 +7361,68 @@ wheels = [ [[package]] name = "orjson" -version = "3.11.6" +version = "3.13.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/70/a3/4e09c61a5f0c521cba0bb433639610ae037437669f1a4cbc93799e731d78/orjson-3.11.6.tar.gz", hash = "sha256:0a54c72259f35299fd033042367df781c2f66d10252955ca1efb7db309b954cb", size = 6175856, upload-time = "2026-01-29T15:13:07.942Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/72/380b97dc45bd162d23afe5194721ef678d9eac7cfaa549fe2873f7f0a518/orjson-3.13.0.tar.gz", hash = "sha256:d1de5eb04485110c5da4c657e49168995d55e076b1ce60f1a042e254f4186c4f", size = 2732604, upload-time = "2026-10-07T14:09:25.719Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/30/3c/098ed0e49c565fdf1ccc6a75b190115d1ca74148bf5b6ab036554a550650/orjson-3.11.6-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:a613fc37e007143d5b6286dccb1394cd114b07832417006a02b620ddd8279e37", size = 250411, upload-time = "2026-01-29T15:11:17.941Z" }, - { url = "https://files.pythonhosted.org/packages/15/7c/cb11a360fd228ceebade03b1e8e9e138dd4b1b3b11602b72dbdad915aded/orjson-3.11.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46ebee78f709d3ba7a65384cfe285bb0763157c6d2f836e7bde2f12d33a867a2", size = 138147, upload-time = "2026-01-29T15:11:19.659Z" }, - { url = "https://files.pythonhosted.org/packages/4e/4b/e57b5c45ffe69fbef7cbd56e9f40e2dc0d5de920caafefcc6981d1a7efc5/orjson-3.11.6-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a726fa86d2368cd57990f2bd95ef5495a6e613b08fc9585dfe121ec758fb08d1", size = 135110, upload-time = "2026-01-29T15:11:21.231Z" }, - { url = "https://files.pythonhosted.org/packages/b0/6e/4f21c6256f8cee3c0c69926cf7ac821cfc36f218512eedea2e2dc4a490c8/orjson-3.11.6-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:150f12e59d6864197770c78126e1a6e07a3da73d1728731bf3bc1e8b96ffdbe6", size = 140995, upload-time = "2026-01-29T15:11:22.902Z" }, - { url = "https://files.pythonhosted.org/packages/d0/78/92c36205ba2f6094ba1eea60c8e646885072abe64f155196833988c14b74/orjson-3.11.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a2d9746a5b5ce20c0908ada451eb56da4ffa01552a50789a0354d8636a02953", size = 144435, upload-time = "2026-01-29T15:11:24.124Z" }, - { url = "https://files.pythonhosted.org/packages/4d/52/1b518d164005811eb3fea92650e76e7d9deadb0b41e92c483373b1e82863/orjson-3.11.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:afd177f5dd91666d31e9019f1b06d2fcdf8a409a1637ddcb5915085dede85680", size = 142734, upload-time = "2026-01-29T15:11:25.708Z" }, - { url = "https://files.pythonhosted.org/packages/4b/11/60ea7885a2b7c1bf60ed8b5982356078a73785bd3bab392041a5bcf8de7c/orjson-3.11.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d777ec41a327bd3b7de97ba7bce12cc1007815ca398e4e4de9ec56c022c090b", size = 145802, upload-time = "2026-01-29T15:11:26.917Z" }, - { url = "https://files.pythonhosted.org/packages/41/7f/15a927e7958fd4f7560fb6dbb9346bee44a168e40168093c46020d866098/orjson-3.11.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f3a135f83185c87c13ff231fcb7dbb2fa4332a376444bd65135b50ff4cc5265c", size = 147504, upload-time = "2026-01-29T15:11:28.07Z" }, - { url = "https://files.pythonhosted.org/packages/66/1f/cabb9132a533f4f913e29294d0a1ca818b1a9a52e990526fe3f7ddd75f1c/orjson-3.11.6-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:2a8eeed7d4544cf391a142b0dd06029dac588e96cc692d9ab1c3f05b1e57c7f6", size = 421408, upload-time = "2026-01-29T15:11:29.314Z" }, - { url = "https://files.pythonhosted.org/packages/4c/b9/09bda9257a982e300313e4a9fc9b9c3aaff424d07bcf765bf045e4e3ed03/orjson-3.11.6-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9d576865a21e5cc6695be8fb78afc812079fd361ce6a027a7d41561b61b33a90", size = 155801, upload-time = "2026-01-29T15:11:30.575Z" }, - { url = "https://files.pythonhosted.org/packages/98/19/4e40ea3e5f4c6a8d51f31fd2382351ee7b396fecca915b17cd1af588175b/orjson-3.11.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:925e2df51f60aa50f8797830f2adfc05330425803f4105875bb511ced98b7f89", size = 147647, upload-time = "2026-01-29T15:11:31.856Z" }, - { url = "https://files.pythonhosted.org/packages/5a/73/ef4bd7dd15042cf33a402d16b87b9e969e71edb452b63b6e2b05025d1f7d/orjson-3.11.6-cp310-cp310-win32.whl", hash = "sha256:09dded2de64e77ac0b312ad59f35023548fb87393a57447e1bb36a26c181a90f", size = 139770, upload-time = "2026-01-29T15:11:33.031Z" }, - { url = "https://files.pythonhosted.org/packages/b4/ac/daab6e10467f7fffd7081ba587b492505b49313130ff5446a6fe28bf076e/orjson-3.11.6-cp310-cp310-win_amd64.whl", hash = "sha256:3a63b5e7841ca8635214c6be7c0bf0246aa8c5cd4ef0c419b14362d0b2fb13de", size = 136783, upload-time = "2026-01-29T15:11:34.686Z" }, - { url = "https://files.pythonhosted.org/packages/f3/fd/d6b0a36854179b93ed77839f107c4089d91cccc9f9ba1b752b6e3bac5f34/orjson-3.11.6-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:e259e85a81d76d9665f03d6129e09e4435531870de5961ddcd0bf6e3a7fde7d7", size = 250029, upload-time = "2026-01-29T15:11:35.942Z" }, - { url = "https://files.pythonhosted.org/packages/a3/bb/22902619826641cf3b627c24aab62e2ad6b571bdd1d34733abb0dd57f67a/orjson-3.11.6-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:52263949f41b4a4822c6b1353bcc5ee2f7109d53a3b493501d3369d6d0e7937a", size = 134518, upload-time = "2026-01-29T15:11:37.347Z" }, - { url = "https://files.pythonhosted.org/packages/72/90/7a818da4bba1de711a9653c420749c0ac95ef8f8651cbc1dca551f462fe0/orjson-3.11.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6439e742fa7834a24698d358a27346bb203bff356ae0402e7f5df8f749c621a8", size = 137917, upload-time = "2026-01-29T15:11:38.511Z" }, - { url = "https://files.pythonhosted.org/packages/59/0f/02846c1cac8e205cb3822dd8aa8f9114acda216f41fd1999ace6b543418d/orjson-3.11.6-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b81ffd68f084b4e993e3867acb554a049fa7787cc8710bbcc1e26965580d99be", size = 134923, upload-time = "2026-01-29T15:11:39.711Z" }, - { url = "https://files.pythonhosted.org/packages/94/cf/aeaf683001b474bb3c3c757073a4231dfdfe8467fceaefa5bfd40902c99f/orjson-3.11.6-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a5a5468e5e60f7ef6d7f9044b06c8f94a3c56ba528c6e4f7f06ae95164b595ec", size = 140752, upload-time = "2026-01-29T15:11:41.347Z" }, - { url = "https://files.pythonhosted.org/packages/fc/fe/dad52d8315a65f084044a0819d74c4c9daf9ebe0681d30f525b0d29a31f0/orjson-3.11.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:72c5005eb45bd2535632d4f3bec7ad392832cfc46b62a3021da3b48a67734b45", size = 144201, upload-time = "2026-01-29T15:11:42.537Z" }, - { url = "https://files.pythonhosted.org/packages/36/bc/ab070dd421565b831801077f1e390c4d4af8bfcecafc110336680a33866b/orjson-3.11.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0b14dd49f3462b014455a28a4d810d3549bf990567653eb43765cd847df09145", size = 142380, upload-time = "2026-01-29T15:11:44.309Z" }, - { url = "https://files.pythonhosted.org/packages/e6/d8/4b581c725c3a308717f28bf45a9fdac210bca08b67e8430143699413ff06/orjson-3.11.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e0bb2c1ea30ef302f0f89f9bf3e7f9ab5e2af29dc9f80eb87aa99788e4e2d65", size = 145582, upload-time = "2026-01-29T15:11:45.506Z" }, - { url = "https://files.pythonhosted.org/packages/5b/a2/09aab99b39f9a7f175ea8fa29adb9933a3d01e7d5d603cdee7f1c40c8da2/orjson-3.11.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:825e0a85d189533c6bff7e2fc417a28f6fcea53d27125c4551979aecd6c9a197", size = 147270, upload-time = "2026-01-29T15:11:46.782Z" }, - { url = "https://files.pythonhosted.org/packages/b8/2f/5ef8eaf7829dc50da3bf497c7775b21ee88437bc8c41f959aa3504ca6631/orjson-3.11.6-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:b04575417a26530637f6ab4b1f7b4f666eb0433491091da4de38611f97f2fcf3", size = 421222, upload-time = "2026-01-29T15:11:48.106Z" }, - { url = "https://files.pythonhosted.org/packages/3b/b0/dd6b941294c2b5b13da5fdc7e749e58d0c55a5114ab37497155e83050e95/orjson-3.11.6-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:b83eb2e40e8c4da6d6b340ee6b1d6125f5195eb1b0ebb7eac23c6d9d4f92d224", size = 155562, upload-time = "2026-01-29T15:11:49.408Z" }, - { url = "https://files.pythonhosted.org/packages/8e/09/43924331a847476ae2f9a16bd6d3c9dab301265006212ba0d3d7fd58763a/orjson-3.11.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1f42da604ee65a6b87eef858c913ce3e5777872b19321d11e6fc6d21de89b64f", size = 147432, upload-time = "2026-01-29T15:11:50.635Z" }, - { url = "https://files.pythonhosted.org/packages/5d/e9/d9865961081816909f6b49d880749dbbd88425afd7c5bbce0549e2290d77/orjson-3.11.6-cp311-cp311-win32.whl", hash = "sha256:5ae45df804f2d344cffb36c43fdf03c82fb6cd247f5faa41e21891b40dfbf733", size = 139623, upload-time = "2026-01-29T15:11:51.82Z" }, - { url = "https://files.pythonhosted.org/packages/b4/f9/6836edb92f76eec1082919101eb1145d2f9c33c8f2c5e6fa399b82a2aaa8/orjson-3.11.6-cp311-cp311-win_amd64.whl", hash = "sha256:f4295948d65ace0a2d8f2c4ccc429668b7eb8af547578ec882e16bf79b0050b2", size = 136647, upload-time = "2026-01-29T15:11:53.454Z" }, - { url = "https://files.pythonhosted.org/packages/b3/0c/4954082eea948c9ae52ee0bcbaa2f99da3216a71bcc314ab129bde22e565/orjson-3.11.6-cp311-cp311-win_arm64.whl", hash = "sha256:314e9c45e0b81b547e3a1cfa3df3e07a815821b3dac9fe8cb75014071d0c16a4", size = 135327, upload-time = "2026-01-29T15:11:56.616Z" }, - { url = "https://files.pythonhosted.org/packages/14/ba/759f2879f41910b7e5e0cdbd9cf82a4f017c527fb0e972e9869ca7fe4c8e/orjson-3.11.6-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:6f03f30cd8953f75f2a439070c743c7336d10ee940da918d71c6f3556af3ddcf", size = 249988, upload-time = "2026-01-29T15:11:58.294Z" }, - { url = "https://files.pythonhosted.org/packages/f0/70/54cecb929e6c8b10104fcf580b0cc7dc551aa193e83787dd6f3daba28bb5/orjson-3.11.6-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:af44baae65ef386ad971469a8557a0673bb042b0b9fd4397becd9c2dfaa02588", size = 134445, upload-time = "2026-01-29T15:11:59.819Z" }, - { url = "https://files.pythonhosted.org/packages/f2/6f/ec0309154457b9ba1ad05f11faa4441f76037152f75e1ac577db3ce7ca96/orjson-3.11.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c310a48542094e4f7dbb6ac076880994986dda8ca9186a58c3cb70a3514d3231", size = 137708, upload-time = "2026-01-29T15:12:01.488Z" }, - { url = "https://files.pythonhosted.org/packages/20/52/3c71b80840f8bab9cb26417302707b7716b7d25f863f3a541bcfa232fe6e/orjson-3.11.6-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d8dfa7a5d387f15ecad94cb6b2d2d5f4aeea64efd8d526bfc03c9812d01e1cc0", size = 134798, upload-time = "2026-01-29T15:12:02.705Z" }, - { url = "https://files.pythonhosted.org/packages/30/51/b490a43b22ff736282360bd02e6bded455cf31dfc3224e01cd39f919bbd2/orjson-3.11.6-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba8daee3e999411b50f8b50dbb0a3071dd1845f3f9a1a0a6fa6de86d1689d84d", size = 140839, upload-time = "2026-01-29T15:12:03.956Z" }, - { url = "https://files.pythonhosted.org/packages/95/bc/4bcfe4280c1bc63c5291bb96f98298845b6355da2226d3400e17e7b51e53/orjson-3.11.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f89d104c974eafd7436d7a5fdbc57f7a1e776789959a2f4f1b2eab5c62a339f4", size = 144080, upload-time = "2026-01-29T15:12:05.151Z" }, - { url = "https://files.pythonhosted.org/packages/01/74/22970f9ead9ab1f1b5f8c227a6c3aa8d71cd2c5acd005868a1d44f2362fa/orjson-3.11.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2e2e2456788ca5ea75616c40da06fc885a7dc0389780e8a41bf7c5389ba257b", size = 142435, upload-time = "2026-01-29T15:12:06.641Z" }, - { url = "https://files.pythonhosted.org/packages/29/34/d564aff85847ab92c82ee43a7a203683566c2fca0723a5f50aebbe759603/orjson-3.11.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a42efebc45afabb1448001e90458c4020d5c64fbac8a8dc4045b777db76cb5a", size = 145631, upload-time = "2026-01-29T15:12:08.351Z" }, - { url = "https://files.pythonhosted.org/packages/e7/ef/016957a3890752c4aa2368326ea69fa53cdc1fdae0a94a542b6410dbdf52/orjson-3.11.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:71b7cbef8471324966c3738c90ba38775563ef01b512feb5ad4805682188d1b9", size = 147058, upload-time = "2026-01-29T15:12:10.023Z" }, - { url = "https://files.pythonhosted.org/packages/56/cc/9a899c3972085645b3225569f91a30e221f441e5dc8126e6d060b971c252/orjson-3.11.6-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f8515e5910f454fe9a8e13c2bb9dc4bae4c1836313e967e72eb8a4ad874f0248", size = 421161, upload-time = "2026-01-29T15:12:11.308Z" }, - { url = "https://files.pythonhosted.org/packages/21/a8/767d3fbd6d9b8fdee76974db40619399355fd49bf91a6dd2c4b6909ccf05/orjson-3.11.6-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:300360edf27c8c9bf7047345a94fddf3a8b8922df0ff69d71d854a170cb375cf", size = 155757, upload-time = "2026-01-29T15:12:12.776Z" }, - { url = "https://files.pythonhosted.org/packages/ad/0b/205cd69ac87e2272e13ef3f5f03a3d4657e317e38c1b08aaa2ef97060bbc/orjson-3.11.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:caaed4dad39e271adfadc106fab634d173b2bb23d9cf7e67bd645f879175ebfc", size = 147446, upload-time = "2026-01-29T15:12:14.166Z" }, - { url = "https://files.pythonhosted.org/packages/de/c5/dd9f22aa9f27c54c7d05cc32f4580c9ac9b6f13811eeb81d6c4c3f50d6b1/orjson-3.11.6-cp312-cp312-win32.whl", hash = "sha256:955368c11808c89793e847830e1b1007503a5923ddadc108547d3b77df761044", size = 139717, upload-time = "2026-01-29T15:12:15.7Z" }, - { url = "https://files.pythonhosted.org/packages/23/a1/e62fc50d904486970315a1654b8cfb5832eb46abb18cd5405118e7e1fc79/orjson-3.11.6-cp312-cp312-win_amd64.whl", hash = "sha256:2c68de30131481150073d90a5d227a4a421982f42c025ecdfb66157f9579e06f", size = 136711, upload-time = "2026-01-29T15:12:17.055Z" }, - { url = "https://files.pythonhosted.org/packages/04/3d/b4fefad8bdf91e0fe212eb04975aeb36ea92997269d68857efcc7eb1dda3/orjson-3.11.6-cp312-cp312-win_arm64.whl", hash = "sha256:65dfa096f4e3a5e02834b681f539a87fbe85adc82001383c0db907557f666bfc", size = 135212, upload-time = "2026-01-29T15:12:18.3Z" }, - { url = "https://files.pythonhosted.org/packages/ae/45/d9c71c8c321277bc1ceebf599bc55ba826ae538b7c61f287e9a7e71bd589/orjson-3.11.6-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:e4ae1670caabb598a88d385798692ce2a1b2f078971b3329cfb85253c6097f5b", size = 249828, upload-time = "2026-01-29T15:12:20.14Z" }, - { url = "https://files.pythonhosted.org/packages/ac/7e/4afcf4cfa9c2f93846d70eee9c53c3c0123286edcbeb530b7e9bd2aea1b2/orjson-3.11.6-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:2c6b81f47b13dac2caa5d20fbc953c75eb802543abf48403a4703ed3bff225f0", size = 134339, upload-time = "2026-01-29T15:12:22.01Z" }, - { url = "https://files.pythonhosted.org/packages/40/10/6d2b8a064c8d2411d3d0ea6ab43125fae70152aef6bea77bb50fa54d4097/orjson-3.11.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:647d6d034e463764e86670644bdcaf8e68b076e6e74783383b01085ae9ab334f", size = 137662, upload-time = "2026-01-29T15:12:23.307Z" }, - { url = "https://files.pythonhosted.org/packages/5a/50/5804ea7d586baf83ee88969eefda97a24f9a5bdba0727f73e16305175b26/orjson-3.11.6-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8523b9cc4ef174ae52414f7699e95ee657c16aa18b3c3c285d48d7966cce9081", size = 134626, upload-time = "2026-01-29T15:12:25.099Z" }, - { url = "https://files.pythonhosted.org/packages/9e/2e/f0492ed43e376722bb4afd648e06cc1e627fc7ec8ff55f6ee739277813ea/orjson-3.11.6-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:313dfd7184cde50c733fc0d5c8c0e2f09017b573afd11dc36bd7476b30b4cb17", size = 140873, upload-time = "2026-01-29T15:12:26.369Z" }, - { url = "https://files.pythonhosted.org/packages/10/15/6f874857463421794a303a39ac5494786ad46a4ab46d92bda6705d78c5aa/orjson-3.11.6-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905ee036064ff1e1fd1fb800055ac477cdcb547a78c22c1bc2bbf8d5d1a6fb42", size = 144044, upload-time = "2026-01-29T15:12:28.082Z" }, - { url = "https://files.pythonhosted.org/packages/d2/c7/b7223a3a70f1d0cc2d86953825de45f33877ee1b124a91ca1f79aa6e643f/orjson-3.11.6-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ce374cb98411356ba906914441fc993f271a7a666d838d8de0e0900dd4a4bc12", size = 142396, upload-time = "2026-01-29T15:12:30.529Z" }, - { url = "https://files.pythonhosted.org/packages/87/e3/aa1b6d3ad3cd80f10394134f73ae92a1d11fdbe974c34aa199cc18bb5fcf/orjson-3.11.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cded072b9f65fcfd188aead45efa5bd528ba552add619b3ad2a81f67400ec450", size = 145600, upload-time = "2026-01-29T15:12:31.848Z" }, - { url = "https://files.pythonhosted.org/packages/f6/cf/e4aac5a46cbd39d7e769ef8650efa851dfce22df1ba97ae2b33efe893b12/orjson-3.11.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7ab85bdbc138e1f73a234db6bb2e4cc1f0fcec8f4bd2bd2430e957a01aadf746", size = 146967, upload-time = "2026-01-29T15:12:33.203Z" }, - { url = "https://files.pythonhosted.org/packages/0b/04/975b86a4bcf6cfeda47aad15956d52fbeda280811206e9967380fa9355c8/orjson-3.11.6-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:351b96b614e3c37a27b8ab048239ebc1e0be76cc17481a430d70a77fb95d3844", size = 421003, upload-time = "2026-01-29T15:12:35.097Z" }, - { url = "https://files.pythonhosted.org/packages/28/d1/0369d0baf40eea5ff2300cebfe209883b2473ab4aa4c4974c8bd5ee42bb2/orjson-3.11.6-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f9959c85576beae5cdcaaf39510b15105f1ee8b70d5dacd90152617f57be8c83", size = 155695, upload-time = "2026-01-29T15:12:36.589Z" }, - { url = "https://files.pythonhosted.org/packages/ab/1f/d10c6d6ae26ff1d7c3eea6fd048280ef2e796d4fb260c5424fd021f68ecf/orjson-3.11.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75682d62b1b16b61a30716d7a2ec1f4c36195de4a1c61f6665aedd947b93a5d5", size = 147392, upload-time = "2026-01-29T15:12:37.876Z" }, - { url = "https://files.pythonhosted.org/packages/8d/43/7479921c174441a0aa5277c313732e20713c0969ac303be9f03d88d3db5d/orjson-3.11.6-cp313-cp313-win32.whl", hash = "sha256:40dc277999c2ef227dcc13072be879b4cfd325502daeb5c35ed768f706f2bf30", size = 139718, upload-time = "2026-01-29T15:12:39.274Z" }, - { url = "https://files.pythonhosted.org/packages/88/bc/9ffe7dfbf8454bc4e75bb8bf3a405ed9e0598df1d3535bb4adcd46be07d0/orjson-3.11.6-cp313-cp313-win_amd64.whl", hash = "sha256:f0f6e9f8ff7905660bc3c8a54cd4a675aa98f7f175cf00a59815e2ff42c0d916", size = 136635, upload-time = "2026-01-29T15:12:40.593Z" }, - { url = "https://files.pythonhosted.org/packages/6f/7e/51fa90b451470447ea5023b20d83331ec741ae28d1e6d8ed547c24e7de14/orjson-3.11.6-cp313-cp313-win_arm64.whl", hash = "sha256:1608999478664de848e5900ce41f25c4ecdfc4beacbc632b6fd55e1a586e5d38", size = 135175, upload-time = "2026-01-29T15:12:41.997Z" }, - { url = "https://files.pythonhosted.org/packages/31/9f/46ca908abaeeec7560638ff20276ab327b980d73b3cc2f5b205b4a1c60b3/orjson-3.11.6-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:6026db2692041d2a23fe2545606df591687787825ad5821971ef0974f2c47630", size = 249823, upload-time = "2026-01-29T15:12:43.332Z" }, - { url = "https://files.pythonhosted.org/packages/ff/78/ca478089818d18c9cd04f79c43f74ddd031b63c70fa2a946eb5e85414623/orjson-3.11.6-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:132b0ab2e20c73afa85cf142e547511feb3d2f5b7943468984658f3952b467d4", size = 134328, upload-time = "2026-01-29T15:12:45.171Z" }, - { url = "https://files.pythonhosted.org/packages/39/5e/cbb9d830ed4e47f4375ad8eef8e4fff1bf1328437732c3809054fc4e80be/orjson-3.11.6-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b376fb05f20a96ec117d47987dd3b39265c635725bda40661b4c5b73b77b5fde", size = 137651, upload-time = "2026-01-29T15:12:46.602Z" }, - { url = "https://files.pythonhosted.org/packages/7c/3a/35df6558c5bc3a65ce0961aefee7f8364e59af78749fc796ea255bfa0cf5/orjson-3.11.6-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:954dae4e080574672a1dfcf2a840eddef0f27bd89b0e94903dd0824e9c1db060", size = 134596, upload-time = "2026-01-29T15:12:47.95Z" }, - { url = "https://files.pythonhosted.org/packages/cd/8e/3d32dd7b7f26a19cc4512d6ed0ae3429567c71feef720fe699ff43c5bc9e/orjson-3.11.6-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe515bb89d59e1e4b48637a964f480b35c0a2676de24e65e55310f6016cca7ce", size = 140923, upload-time = "2026-01-29T15:12:49.333Z" }, - { url = "https://files.pythonhosted.org/packages/6c/9c/1efbf5c99b3304f25d6f0d493a8d1492ee98693637c10ce65d57be839d7b/orjson-3.11.6-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:380f9709c275917af28feb086813923251e11ee10687257cd7f1ea188bcd4485", size = 144068, upload-time = "2026-01-29T15:12:50.927Z" }, - { url = "https://files.pythonhosted.org/packages/82/83/0d19eeb5be797de217303bbb55dde58dba26f996ed905d301d98fd2d4637/orjson-3.11.6-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8173e0d3f6081e7034c51cf984036d02f6bab2a2126de5a759d79f8e5a140e7", size = 142493, upload-time = "2026-01-29T15:12:52.432Z" }, - { url = "https://files.pythonhosted.org/packages/32/a7/573fec3df4dc8fc259b7770dc6c0656f91adce6e19330c78d23f87945d1e/orjson-3.11.6-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6dddf9ba706294906c56ef5150a958317b09aa3a8a48df1c52ccf22ec1907eac", size = 145616, upload-time = "2026-01-29T15:12:53.903Z" }, - { url = "https://files.pythonhosted.org/packages/c2/0e/23551b16f21690f7fd5122e3cf40fdca5d77052a434d0071990f97f5fe2f/orjson-3.11.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:cbae5c34588dc79938dffb0b6fbe8c531f4dc8a6ad7f39759a9eb5d2da405ef2", size = 146951, upload-time = "2026-01-29T15:12:55.698Z" }, - { url = "https://files.pythonhosted.org/packages/b8/63/5e6c8f39805c39123a18e412434ea364349ee0012548d08aa586e2bd6aa9/orjson-3.11.6-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:f75c318640acbddc419733b57f8a07515e587a939d8f54363654041fd1f4e465", size = 421024, upload-time = "2026-01-29T15:12:57.434Z" }, - { url = "https://files.pythonhosted.org/packages/1d/4d/724975cf0087f6550bd01fd62203418afc0ea33fd099aed318c5bcc52df8/orjson-3.11.6-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:e0ab8d13aa2a3e98b4a43487c9205b2c92c38c054b4237777484d503357c8437", size = 155774, upload-time = "2026-01-29T15:12:59.397Z" }, - { url = "https://files.pythonhosted.org/packages/a8/a3/f4c4e3f46b55db29e0a5f20493b924fc791092d9a03ff2068c9fe6c1002f/orjson-3.11.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f884c7fb1020d44612bd7ac0db0babba0e2f78b68d9a650c7959bf99c783773f", size = 147393, upload-time = "2026-01-29T15:13:00.769Z" }, - { url = "https://files.pythonhosted.org/packages/ee/86/6f5529dd27230966171ee126cecb237ed08e9f05f6102bfaf63e5b32277d/orjson-3.11.6-cp314-cp314-win32.whl", hash = "sha256:8d1035d1b25732ec9f971e833a3e299d2b1a330236f75e6fd945ad982c76aaf3", size = 139760, upload-time = "2026-01-29T15:13:02.173Z" }, - { url = "https://files.pythonhosted.org/packages/d3/b5/91ae7037b2894a6b5002fb33f4fbccec98424a928469835c3837fbb22a9b/orjson-3.11.6-cp314-cp314-win_amd64.whl", hash = "sha256:931607a8865d21682bb72de54231655c86df1870502d2962dbfd12c82890d077", size = 136633, upload-time = "2026-01-29T15:13:04.267Z" }, - { url = "https://files.pythonhosted.org/packages/55/74/f473a3ec7a0a7ebc825ca8e3c86763f7d039f379860c81ba12dcdd456547/orjson-3.11.6-cp314-cp314-win_arm64.whl", hash = "sha256:fe71f6b283f4f1832204ab8235ce07adad145052614f77c876fcf0dac97bc06f", size = 135168, upload-time = "2026-01-29T15:13:05.932Z" }, + { url = "https://files.pythonhosted.org/packages/11/8c/25b6e2bd4f6b8e67a6b5acbc11a8cff4970e35c79837a24ec7db8732238d/orjson-3.13.0-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:4f66eac85b072092e9941c3111882afd7527bf926cbc717038fa3654b582002b", size = 223510, upload-time = "2026-10-07T14:07:54.539Z" }, + { url = "https://files.pythonhosted.org/packages/32/4d/5772e32ebc19d0b76b957a48e69a09546400db35cebe76c21b2c341d1a30/orjson-3.13.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:efa160215c4630836d3b1250af4c7a305acd8239e0d75aff986b8088c2fcacb6", size = 113481, upload-time = "2026-10-07T14:07:56.229Z" }, + { url = "https://files.pythonhosted.org/packages/5a/6a/5ce6adad2c0cb734cb9d19b7b9d9c7bbdb16c136af453dd37adace806547/orjson-3.13.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4e5c8175e1574dcbe446ee654275d353c1d78bbd9a0dc9f209bf35c9df72d171", size = 130791, upload-time = "2026-10-07T14:07:57.751Z" }, + { url = "https://files.pythonhosted.org/packages/96/49/d954f02229efb06850a5f9aaf06e77e03046a009d49eb78f499fbd798ded/orjson-3.13.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78a12d4f8d740cc9ae197f5223682e5e960ba61b4fb2ce5a6a3bb54e83fde28e", size = 129465, upload-time = "2026-10-07T14:07:59.143Z" }, + { url = "https://files.pythonhosted.org/packages/2f/a2/abcb0647268f334cb85768170b164e4c97f7a2ed5fddd146f79297494d9e/orjson-3.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93c70a5e22bbbbdeafc7b273441e8452a196041d67fd4d9a9c450c66370a8486", size = 130727, upload-time = "2026-10-07T14:08:00.659Z" }, + { url = "https://files.pythonhosted.org/packages/fa/b0/5672f0505e6cde410cc7916cc2fbf88d90216d667b37907df041a659db06/orjson-3.13.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7b3bc6b81835ce65f4729ae401607583d41139c6de95bc7453f450f1391d3e7b", size = 135280, upload-time = "2026-10-07T14:08:02.167Z" }, + { url = "https://files.pythonhosted.org/packages/d9/58/c223e3ac16193d00c1c3cbc786cb6db47158bff0558c52133e6dd0be7a12/orjson-3.13.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6d0684895b119ad167fb4ec05113639dc7f728022deec4756a710e838ed92e7a", size = 126844, upload-time = "2026-10-07T14:08:03.549Z" }, + { url = "https://files.pythonhosted.org/packages/49/a2/f6fd98acef1e36b8c8ae0275f0268a0f22bb6a1b436ee4536e1cdaf31b03/orjson-3.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:7991921c5da527a963b6d4cffd0e4ea89c7e71d4be0c8be1bfe6edb223ce7d96", size = 121455, upload-time = "2026-10-07T14:08:05.024Z" }, + { url = "https://files.pythonhosted.org/packages/ce/a3/0be3b115907fea61ed340639fb0e1562cd18969bad5b3f486f808197aaff/orjson-3.13.0-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:948bad47f2e2e43527f14248364a0e5dee26dd3184691010ec4a1ebeb0fd6771", size = 223146, upload-time = "2026-10-07T14:08:06.474Z" }, + { url = "https://files.pythonhosted.org/packages/9e/f7/665935edb16163f8b764182e29a30cf056947a66893ed032191e5f01eb3d/orjson-3.13.0-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:1807c2fa49d393c7ee95fd1ef1b39cbb24aa3ccd81f30b84503ba59407666960", size = 123546, upload-time = "2026-10-07T14:08:08.324Z" }, + { url = "https://files.pythonhosted.org/packages/67/ec/e7cde480c0e212594d17ba2b2bd210c002052e9147fc1a1aeafaabe722fb/orjson-3.13.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:637dbca1fccffe83780e806fbc0f17427c0c59bf822528eb0acc8f0aa9f19acb", size = 113290, upload-time = "2026-10-07T14:08:09.816Z" }, + { url = "https://files.pythonhosted.org/packages/36/59/4455fb11a297af73611dfc437f0f89456220227ed1cb1544a5a0ee9d6c03/orjson-3.13.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:554948becd1110123ef9f6a6e1310fd92b2d07d2cbac6dbf65df3de75702e736", size = 130342, upload-time = "2026-10-07T14:08:11.253Z" }, + { url = "https://files.pythonhosted.org/packages/ca/80/0eec5fbde2e52407646b4cb3118f63175bdcee1e2390c2759dc96e0bc62a/orjson-3.13.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd9d9a101bd8dbfad112170f009cd155e52bb8c936468821a0d03cbb96c0e426", size = 129138, upload-time = "2026-10-07T14:08:12.814Z" }, + { url = "https://files.pythonhosted.org/packages/cd/cc/c0874f13819ae346d69ca00d074d464710b494abd4442bdebf75ac404a98/orjson-3.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89bcf2d4bc6c9a7e1763c8cf534f38712e66b76a0fefda7fb7785462f0d635e4", size = 130518, upload-time = "2026-10-07T14:08:14.392Z" }, + { url = "https://files.pythonhosted.org/packages/25/ab/140dd9adff84bf64b862c4fcfe2d055af6014d5ba03a075f95c9addb2ec7/orjson-3.13.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a79cdc4934fe81f593072c94e13da3095e9d41c2deef8f6ff2901794ca1c5042", size = 134924, upload-time = "2026-10-07T14:08:16.09Z" }, + { url = "https://files.pythonhosted.org/packages/08/0a/e8f6deb032b1d98a39043cf99b863d8b9e842e2ffc2d2067d2e2a88c18e4/orjson-3.13.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:50a5202ba388b3850ba24437951727d3aa6d79a21964a30ae8dc6a059a5fd34c", size = 126704, upload-time = "2026-10-07T14:08:17.439Z" }, + { url = "https://files.pythonhosted.org/packages/af/cf/be64b99ff75f7983488390d4ef5df72115119770eed295691c0a715d492a/orjson-3.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:a0377d6962fa431c93ecd78fdea771bb62ec545b24ee0c5d4e32acf2260af259", size = 121287, upload-time = "2026-10-07T14:08:18.843Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ab/1b8ca186baf3420f12db1f2819fcc5f2cae69e4cf051168501726a64c0fa/orjson-3.13.0-cp311-cp311-win_arm64.whl", hash = "sha256:1d84820b2ec4ac975cba482214032de5b0dbdd17046170c98e642ef9c4a4ee4b", size = 126314, upload-time = "2026-10-07T14:08:20.452Z" }, + { url = "https://files.pythonhosted.org/packages/98/17/ed65f84ed5ed6a1e06eb628611b4172e7480fc4ad92594856751a6363cac/orjson-3.13.0-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:fb8644dc6d705e1269ed2842bf4dbe2b4e50d670de503bf79d5cef3a5148a4c7", size = 223063, upload-time = "2026-10-07T14:08:21.979Z" }, + { url = "https://files.pythonhosted.org/packages/6f/4d/9332eb96d2e379384be0f211f543835eebc81f460c9403b84abe1294c431/orjson-3.13.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:6ff2a2c67f35202f7d823753d38ad371a9b7fc297567cdfff4420e763cb9f6f8", size = 123364, upload-time = "2026-10-07T14:08:24.026Z" }, + { url = "https://files.pythonhosted.org/packages/b4/06/558456b7da27e974a8c9ea09117b07119f6fa131cd62b8b9ecad9eea94e1/orjson-3.13.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:65c4e0e106ccc7265b488385659117a6805c37d042f737558ecd68aa0c67ad8f", size = 113199, upload-time = "2026-10-07T14:08:25.476Z" }, + { url = "https://files.pythonhosted.org/packages/b7/f2/1187a9c09965620348262ec0f406868f6d7c234b2e9b5ee51020bdde5748/orjson-3.13.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:fbbad6b9b1da43f25c1f5b20cd5a268e028a2fc95d5a8d1ade6059973bc71584", size = 130329, upload-time = "2026-10-07T14:08:26.877Z" }, + { url = "https://files.pythonhosted.org/packages/46/07/5d1a151bc11600434fe799e73abfc6a4d463d02e149a20e47c59d3a985ae/orjson-3.13.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae1d895cf7bbfd50ef34bb63bb727b14514f259f3e3f8dd010783bd38e864c6e", size = 129072, upload-time = "2026-10-07T14:08:28.355Z" }, + { url = "https://files.pythonhosted.org/packages/ea/8c/bb07c368abbf4021c4cd01c12edb526e00090f7f750ff1b88da6e6b6c7a6/orjson-3.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bceadfd314bd238f584fc229a4bbaf0e573597e7a026dec5429fbf29fd66c641", size = 130612, upload-time = "2026-10-07T14:08:30.041Z" }, + { url = "https://files.pythonhosted.org/packages/d2/8d/4b66d19619ed344ac000ffea7c006477d0061d580646e736ef0e203759e8/orjson-3.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b74c30e56346aad067937d766846ee74c231d1d18aad3f324e9b9261de3b2d5e", size = 134632, upload-time = "2026-10-07T14:08:31.474Z" }, + { url = "https://files.pythonhosted.org/packages/ea/88/f8221f6593e37eb26ec4706e185b9ac6f38ff0c8f7bad5459844031ffd2d/orjson-3.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4329c19b8a25693f60a77b867c9d2a3ab637b20e36f5b7bea7f5acb492b44b15", size = 126807, upload-time = "2026-10-07T14:08:32.914Z" }, + { url = "https://files.pythonhosted.org/packages/58/9d/a1ca7321eeafd7d72e174cdc388cc96301f41516d863e7b1f64f0a1735be/orjson-3.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:b571236d8393edcd3236e07423f762bfcf571f852aad667a3bce9e7b755e0790", size = 121538, upload-time = "2026-10-07T14:08:34.325Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a0/1f19b4779c910104370932fceb9ed436b47ac077f297db74008062525c04/orjson-3.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:8594956a75223f657e1e68c568c0eeb3dd145f02cd6b78a47fd9a8095dbc4eae", size = 126259, upload-time = "2026-10-07T14:08:35.765Z" }, + { url = "https://files.pythonhosted.org/packages/a9/56/f8ad2546150168858c16915c452b00eecb79597597524d1ad6ae14ad4eab/orjson-3.13.0-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:64e8f345048d988c8b68d3882e5d41028fca1219a9939b32e4a77be34c8ae8e3", size = 222892, upload-time = "2026-10-07T14:08:37.495Z" }, + { url = "https://files.pythonhosted.org/packages/1f/19/725d23160b2471a3f27026c55bb79af34687652d8be8f5f583cee5dcd42f/orjson-3.13.0-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ded33b972cffdaf4ca0ac917338ab61d2bb10d68987dbcae641c313fbfdbf499", size = 123319, upload-time = "2026-10-07T14:08:38.989Z" }, + { url = "https://files.pythonhosted.org/packages/ac/08/e5d81a00b22c73dfcb60d80da3bd92d5a7684346593536565f184dbae3c9/orjson-3.13.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:45e34deb3437509f4ec9888dd9ee5dc426cfe21be10f1eb4ea3a9e4d33034f9e", size = 113196, upload-time = "2026-10-07T14:08:40.383Z" }, + { url = "https://files.pythonhosted.org/packages/67/78/fda6117c69a43e470b1e9dff38dd8c5f0bc6fd8a47e4d4561ab023039335/orjson-3.13.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:9825b954155b345c4759f24e5f8d652b9aec2261bb5d4e1abe06bba0a1200535", size = 130245, upload-time = "2026-10-07T14:08:41.878Z" }, + { url = "https://files.pythonhosted.org/packages/6d/31/d0cfebd456defb234414795ae7599696bf124843dfe077d0c9ece0c93554/orjson-3.13.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b081f0e7b600ff24513dec4ca75507fa05e904607847e386e8310d5b7b96b6c7", size = 128981, upload-time = "2026-10-07T14:08:43.716Z" }, + { url = "https://files.pythonhosted.org/packages/45/46/f8d83189ff5b7b2ff225a58c5908618cc4e86afe09e65d17a30ac68c9da4/orjson-3.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbed5f4c4b88d94bcc36115f4c3bb3aa25da1563a5c3328aa3acebce2b083040", size = 130370, upload-time = "2026-10-07T14:08:45.132Z" }, + { url = "https://files.pythonhosted.org/packages/e6/6a/d6344c305003ea826b3fa0482645a897a3cd6d477ed74e1fe15d3322cb23/orjson-3.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e9b61676116f755126b90e740a9cff36b91562f47ec330056cc88cc3b9f02f4b", size = 134595, upload-time = "2026-10-07T14:08:46.63Z" }, + { url = "https://files.pythonhosted.org/packages/9f/52/d73fa44f88d53e02d10de1cf77c16ed13204ff5bca47e1692da6b406619c/orjson-3.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3ef75ed7e81dae34a3649f82df52cd85f9ac839a7d6ec78ab355b33b3b27ef7f", size = 126513, upload-time = "2026-10-07T14:08:48.111Z" }, + { url = "https://files.pythonhosted.org/packages/fb/f8/bcfc50b4ab851c4f9c0ee62f52bf3b28f0bcd0d9fe08e0ad98d4585148db/orjson-3.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:4ee06e53b998c71ce3eb93b86222912fdd9dcced685ac64d4525d36fac338ea4", size = 121371, upload-time = "2026-10-07T14:08:49.549Z" }, + { url = "https://files.pythonhosted.org/packages/7b/7a/d6927845712ec2b1e89263cd12d7203531db185dbad67f914226f2fca156/orjson-3.13.0-cp313-cp313-win_arm64.whl", hash = "sha256:89efecad02515df7f318d0613b5dfd6d2a1acd323a2b8294712789a715945525", size = 126134, upload-time = "2026-10-07T14:08:51.118Z" }, + { url = "https://files.pythonhosted.org/packages/f0/10/98b5a3cdc086abf78d8cd20bb0cba124485d4b6a745722197bd209d967a5/orjson-3.13.0-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:a7bfc7db961c7d96cb75889dc6a1e4ae1e91d87ee61da564f582bd742b8dfeef", size = 222889, upload-time = "2026-10-07T14:08:52.673Z" }, + { url = "https://files.pythonhosted.org/packages/22/7c/7728c5280ab5202f4891ff4b0b96e2e1dbd5520dfee53edf083c54409a64/orjson-3.13.0-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:91d933e668ff0ffe164d7c2daec36beba6d1ce7fadb71538fbe142a71f8a1e6e", size = 123312, upload-time = "2026-10-07T14:08:54.25Z" }, + { url = "https://files.pythonhosted.org/packages/a9/a5/d9a44321e6f66c0f64b45be587395f87ad94cb447bce7d92286f6b97d46a/orjson-3.13.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:6c8bfe728b81b0fd58a3c7f3f9c5a113f87f2992c9948e0f28707aafd737c0bc", size = 113146, upload-time = "2026-10-07T14:08:55.803Z" }, + { url = "https://files.pythonhosted.org/packages/80/da/d95c80d413f288feb471e16d82e5c1512d2439728e3bac917d058c31f098/orjson-3.13.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:e8e05549f3b30f9d8a8e28c5aba11cc2a4b90b90961ec685ca58444b0815fc09", size = 130348, upload-time = "2026-10-07T14:08:57.31Z" }, + { url = "https://files.pythonhosted.org/packages/04/0f/36fdfb32ad1852997bac00e3ce52c7888d8a1094ba9dcdcbb22fcc6b953a/orjson-3.13.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c749ab3ac30b5ab1ffb7677f8b92eacfdfdc5260210baa398f845bc3714c05d8", size = 128971, upload-time = "2026-10-07T14:08:58.843Z" }, + { url = "https://files.pythonhosted.org/packages/25/de/a82acf93bdcca0c79ccff25ef0c6868d24ccbc2e72f21fae39c8cabce4f1/orjson-3.13.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58a9619d88f8818d9ab6b39d70d203789457ba13c1ed5d274f33ce9ae7e81a36", size = 130359, upload-time = "2026-10-07T14:09:00.412Z" }, + { url = "https://files.pythonhosted.org/packages/71/ca/2bc4f7697cb9f6897bf61aca11803df096a5d971bf69ef5538b243bb1fa8/orjson-3.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2715c4808d1571029ed18fd07a82140bf3ba7def0dc89f8d015c416e3649bf87", size = 134583, upload-time = "2026-10-07T14:09:02.047Z" }, + { url = "https://files.pythonhosted.org/packages/23/b3/12b1af9b87ff9fa0aaf4e5724c87672b30bb5de76f275f7fac64e8219c1b/orjson-3.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:08bf722f923d2100bc5e5a5dcf72c656db557049c1bea26582fdd5dd9d5395a1", size = 126500, upload-time = "2026-10-07T14:09:03.863Z" }, + { url = "https://files.pythonhosted.org/packages/ad/ea/cf257fc8a7f4b18f5677c22b3a9673a1b51d4b7161f25177ed389b76560e/orjson-3.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:6adcaa85d79977659a448b4123a88eb33511a11ed2db243535ad7ea88a6668e0", size = 121378, upload-time = "2026-10-07T14:09:05.375Z" }, + { url = "https://files.pythonhosted.org/packages/05/0a/9f4643f849e9918eab11983b83928af3aac14bedb04002e28e885ee1936f/orjson-3.13.0-cp314-cp314-win_arm64.whl", hash = "sha256:83705c12b4afde10c62a5dd3fe6fdb21b7900bd0dcd5af1c85612ae94d0ee590", size = 126123, upload-time = "2026-10-07T14:09:07.085Z" }, + { url = "https://files.pythonhosted.org/packages/8c/15/d265f2b556c0c7c0b30ea830316d6e5af5b85dde08f234a1ebed60fab386/orjson-3.13.0-cp315-cp315-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:5ef4d4157392a0439b74f7e49e5636b4ea43d9616bd0884effc0195fffcaa2d5", size = 223305, upload-time = "2026-10-07T14:09:08.84Z" }, + { url = "https://files.pythonhosted.org/packages/0c/97/781be8b80a33b8171b3f5acea941af47182c8b4b5827c2b7c3fea706f21c/orjson-3.13.0-cp315-cp315-macosx_15_0_arm64.whl", hash = "sha256:84d87e322e1674408f85adea63f11aa19201eba082755aec20ebc217f493bbd2", size = 123515, upload-time = "2026-10-07T14:09:10.792Z" }, + { url = "https://files.pythonhosted.org/packages/20/68/011bb98fa7da7b430b363db1bb7ef9160c438fc5c43e7468fb593c220037/orjson-3.13.0-cp315-cp315-manylinux_2_39_aarch64.whl", hash = "sha256:8c2ac5c09b017c484df1b4c68b2cf250b4e8ba08204cb58e7cd6cbbc71a9c902", size = 129222, upload-time = "2026-10-07T14:09:12.542Z" }, + { url = "https://files.pythonhosted.org/packages/86/7f/d96fa2aedaaec14c095ea9cd48d2158fdf33c0f4fd6e7a598d899d536b03/orjson-3.13.0-cp315-cp315-manylinux_2_39_armv7l.whl", hash = "sha256:51d11525bc3ca736fa97ce4e4c7da9999cc00bf261522bede43b4e7531bd7965", size = 113152, upload-time = "2026-10-07T14:09:14.059Z" }, + { url = "https://files.pythonhosted.org/packages/e9/2d/ee77aa685c54bd920a1f0e2936986b46269adb0d72bf5098c2c694dbeb36/orjson-3.13.0-cp315-cp315-manylinux_2_39_i686.whl", hash = "sha256:ac81530647c3423107cf61c3481e91f57134e9ddfb6ef83f5150ccbdcbc3a3ee", size = 130749, upload-time = "2026-10-07T14:09:15.835Z" }, + { url = "https://files.pythonhosted.org/packages/48/eb/3411fbfdad61b3f3af22343b5af7ed5c8a1679e35f442e8f1b229b33040e/orjson-3.13.0-cp315-cp315-manylinux_2_39_x86_64.whl", hash = "sha256:0526a3456db67b264c6d661b5f090077f326b6cd074d0ef53a72763595dec5d7", size = 130471, upload-time = "2026-10-07T14:09:17.463Z" }, + { url = "https://files.pythonhosted.org/packages/87/71/abdc2b8c70b8d85a6cb22f404da0f52d7d712f9d49cda039a0cb1adcb973/orjson-3.13.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:dd61e64802d51d1e4f16531c64536354fc3bc67932dc0cff254044f72bf0f187", size = 134793, upload-time = "2026-10-07T14:09:19.084Z" }, + { url = "https://files.pythonhosted.org/packages/0a/2e/1c13552d8b0241083116de02b2f284ee38501ef06ebfb79893f741538168/orjson-3.13.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:c5e3ccaac3106e8fa6e2f2f6962449d7c757d7b067e41b395a19d6f0d6cec892", size = 126711, upload-time = "2026-10-07T14:09:20.645Z" }, + { url = "https://files.pythonhosted.org/packages/85/f8/d4ece953a519d064cf690adaa68cd389d5b64fd261726334841b32978d6a/orjson-3.13.0-cp315-cp315-win_amd64.whl", hash = "sha256:7804dd1d6161da0e53b284c2aebf20f23e78eaac617300803e1467d1828d987f", size = 121496, upload-time = "2026-10-07T14:09:22.359Z" }, + { url = "https://files.pythonhosted.org/packages/70/cf/f691388c4a9bc4af7dcc1648c4b40845869908b517d7c0009d005c7d1fa1/orjson-3.13.0-cp315-cp315-win_arm64.whl", hash = "sha256:f5c05a8fee59309f537590a1ff12d3c1009c485e96a50a9ac60dd085c09d0fc0", size = 126260, upload-time = "2026-10-07T14:09:23.928Z" }, ] [[package]] @@ -7315,6 +7657,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/23/26/fcb2f6e37175b04f53570b59937867e2b80ee1685e744023153028fc14f9/pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7", size = 6474125, upload-time = "2026-07-01T11:55:30.956Z" }, { url = "https://files.pythonhosted.org/packages/90/de/3634abee5f1c9e13c56787b7d5517b0ba8d6de51700b95578cf338349c9f/pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c", size = 7242939, upload-time = "2026-07-01T11:55:34.044Z" }, { url = "https://files.pythonhosted.org/packages/ce/2a/fd13f8eb24de5714a6eb444a3d67e2842c6c576e159a43793adf23051351/pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45", size = 2567506, upload-time = "2026-07-01T11:55:35.988Z" }, + { url = "https://files.pythonhosted.org/packages/5d/dc/8fdce34ec725a33c81c6ba122b904d6b9024e50ea9ac7bede62fab54506c/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139", size = 4162063, upload-time = "2026-07-01T11:55:37.941Z" }, + { url = "https://files.pythonhosted.org/packages/76/66/2044b9a63d3b84ff048228dfcb7cd9bf0df983e8470971bf7d4c57b693de/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402", size = 4255549, upload-time = "2026-07-01T11:55:40.022Z" }, + { url = "https://files.pythonhosted.org/packages/52/7e/1f67e6f4ece6b582ee4b539decbcc9f848dc245a93ed8cd7338bafef72f1/pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c", size = 3696331, upload-time = "2026-07-01T11:55:41.98Z" }, + { url = "https://files.pythonhosted.org/packages/12/40/d306fc2c8e4d45d7f175c77edca7063be7b86fe7fe6e68f4353bf71d808c/pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f", size = 5350370, upload-time = "2026-07-01T11:55:44.028Z" }, + { url = "https://files.pythonhosted.org/packages/dd/44/668fb1437e8ce420f62d6106eb66e44a5971602a4d794615bdf79315d82d/pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701", size = 4780147, upload-time = "2026-07-01T11:55:46.073Z" }, + { url = "https://files.pythonhosted.org/packages/0c/08/93fa2e70e30a2d81547e481b6ee2bb9522117221fb1e0ce4b5df70967677/pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace", size = 6273659, upload-time = "2026-07-01T11:55:48.264Z" }, + { url = "https://files.pythonhosted.org/packages/f8/6d/043e96ff814fc31a33077e4cba86082167db520c93632afdf2042febbb0c/pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4", size = 6947439, upload-time = "2026-07-01T11:55:50.503Z" }, + { url = "https://files.pythonhosted.org/packages/af/92/ba71d2ee2ac0edf3fa33bd9d5ee9ee080da70b1766f3ca3934f9938ddac9/pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39", size = 6353577, upload-time = "2026-07-01T11:55:52.697Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ce/e63064e2122923ff687c8ad792d0d736a7b3920a56a46982e81a7fdd25d6/pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71", size = 7060394, upload-time = "2026-07-01T11:55:55.149Z" }, + { url = "https://files.pythonhosted.org/packages/54/76/a09cc3ccc8d773a7283d34c38bec1708f9e3cc932093cbc4c5e71ac4060b/pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827", size = 6467375, upload-time = "2026-07-01T11:55:57.769Z" }, + { url = "https://files.pythonhosted.org/packages/3e/03/1846c49ba3b1d5550392a4bbd06d6fb4578e1cd91a803198b5c90f5f7d53/pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5", size = 7237048, upload-time = "2026-07-01T11:55:59.975Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bb/89f35dcc79610423f9f195504d7def7f0d1416a711541b42867e25fe3412/pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658", size = 2566006, upload-time = "2026-07-01T11:56:02.143Z" }, + { url = "https://files.pythonhosted.org/packages/30/88/707027ba09942dfa2c28759b5c222d769290a41c6d20ea60ec250801941f/pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf", size = 5352509, upload-time = "2026-07-01T11:56:04.2Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6d/00352fa25332c2569cd387851f568cc5a4b75a9adbfb37ac4fbce4c02eec/pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64", size = 4783167, upload-time = "2026-07-01T11:56:06.631Z" }, + { url = "https://files.pythonhosted.org/packages/13/4f/9e049dfa21af7c22427275720e2490267ba8138120add5c4c574deb69782/pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e", size = 6329237, upload-time = "2026-07-01T11:56:08.868Z" }, + { url = "https://files.pythonhosted.org/packages/36/16/cf6eeaae8d0fce8dd390a33437cf68c5d5bd73834a2bc6e2f14efda0ab45/pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777", size = 6997047, upload-time = "2026-07-01T11:56:11.379Z" }, + { url = "https://files.pythonhosted.org/packages/1e/69/dbf769bdd55f48bf5733cac28edc6364ffaa072ec9ba336266e4fe66be55/pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1", size = 6400440, upload-time = "2026-07-01T11:56:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" }, + { url = "https://files.pythonhosted.org/packages/18/f0/a5595c1e8c3ae44b9828cb2f0fa8155e5095ef04d6327b8f61cf44a3df85/pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8", size = 6474384, upload-time = "2026-07-01T11:56:18.855Z" }, + { url = "https://files.pythonhosted.org/packages/e4/04/62bcd9f844984c5938d3b05264a61d797a29d3e0812341a8204af70bbdee/pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418", size = 7243537, upload-time = "2026-07-01T11:56:21.214Z" }, + { url = "https://files.pythonhosted.org/packages/3d/68/1f3066acedf37673694a7141381d8f811ae97f30d34413d236abe7d489f1/pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59", size = 2567491, upload-time = "2026-07-01T11:56:23.506Z" }, { url = "https://files.pythonhosted.org/packages/75/18/2e8b40223153ccbc60df07f9e8928dc0c76202aa4e55ae9f53962b6510d6/pillow-12.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468", size = 5302510, upload-time = "2026-07-01T11:56:25.736Z" }, { url = "https://files.pythonhosted.org/packages/46/3e/51fabf59d5ab801ceab709453d3ab6b180083496579549de4c45ced6528a/pillow-12.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94", size = 4736058, upload-time = "2026-07-01T11:56:28.041Z" }, { url = "https://files.pythonhosted.org/packages/bf/20/22fe9384b7949e25fb1293bcfc84fb82590ff4ea6b37c95b24d26d793d86/pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e", size = 5237776, upload-time = "2026-07-01T11:56:30.263Z" }, @@ -7327,7 +7690,8 @@ name = "pillow" version = "13.0.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", @@ -7385,6 +7749,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bb/af/c30dd9ef6a3dce84ecb1b2393cac629047a32e66279056c6cb46c3ffd65c/pillow-13.0.0-cp314-cp314t-win32.whl", hash = "sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15", size = 6517204, upload-time = "2026-10-16T22:17:03.722Z" }, { url = "https://files.pythonhosted.org/packages/85/d5/ef3a8ae7609c8ab26570abf18194882534b480d245a9148b895a2262c004/pillow-13.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0", size = 7286695, upload-time = "2026-10-16T22:17:05.543Z" }, { url = "https://files.pythonhosted.org/packages/84/3c/0861db0c69a03689ebe3e7599d35668a414deb5aaafd8255215e4e62e3d3/pillow-13.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692", size = 2698098, upload-time = "2026-10-16T22:17:07.674Z" }, + { url = "https://files.pythonhosted.org/packages/6e/eb/e86a41962a3a31ad495a3d6a912d9002b01bba4a6fa71c28703abdca7f69/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521", size = 4191213, upload-time = "2026-10-16T22:17:09.449Z" }, + { url = "https://files.pythonhosted.org/packages/dd/04/7e7445e706afec84ef06dc25b2c77f9cecff242c23730c62928ad1089d4a/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc", size = 4290650, upload-time = "2026-10-16T22:17:11.196Z" }, + { url = "https://files.pythonhosted.org/packages/46/85/d534089068ac40c0f61f650389a296c812eab745a8a733f81feec39ae955/pillow-13.0.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac", size = 3723078, upload-time = "2026-10-16T22:17:12.772Z" }, + { url = "https://files.pythonhosted.org/packages/ce/2b/00b5c3b401df538b5dbe311d3899730fae5ba1e21cb8b80a128887ac6747/pillow-13.0.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d", size = 5572292, upload-time = "2026-10-16T22:17:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/72/f5/a6b0610f946e9b54ae888ba7ad29d6e2f8d9b11f69e212d70416b7186c09/pillow-13.0.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782", size = 4852222, upload-time = "2026-10-16T22:17:16.521Z" }, + { url = "https://files.pythonhosted.org/packages/1e/d0/c0ff25bb0427ec70f32aac995453321fc3c4494a46c7292ff02d9d260b04/pillow-13.0.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071", size = 6527349, upload-time = "2026-10-16T22:17:18.534Z" }, + { url = "https://files.pythonhosted.org/packages/85/11/e46fa0bcc0fc117f1eacc206b3d51f04d583fab328b4b740672d0438d83f/pillow-13.0.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf", size = 7152873, upload-time = "2026-10-16T22:17:20.415Z" }, + { url = "https://files.pythonhosted.org/packages/7c/53/2bdeca705f3a58679fa9201233193c86a30238340ca29b3a152c3fcc719b/pillow-13.0.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3", size = 6626871, upload-time = "2026-10-16T22:17:22.361Z" }, + { url = "https://files.pythonhosted.org/packages/14/de/3829030f209722ade3753bb456b805a0e7f885cf885b483407365c82e9ca/pillow-13.0.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932", size = 7236439, upload-time = "2026-10-16T22:17:24.352Z" }, + { url = "https://files.pythonhosted.org/packages/44/b1/67fc37388382bc036a74f04da5bb6fcbf90ec3ac1f45893e06176c373fd3/pillow-13.0.0-cp315-cp315-win32.whl", hash = "sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81", size = 6508688, upload-time = "2026-10-16T22:17:26.314Z" }, + { url = "https://files.pythonhosted.org/packages/44/05/fdb6a79aee10861165526b41b7b440e36ee1ff58613b2fbefe3acbf06d4f/pillow-13.0.0-cp315-cp315-win_amd64.whl", hash = "sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6", size = 7281107, upload-time = "2026-10-16T22:17:28.198Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9c/6c6f7dc2a566467baf121394e83de25ac287cffe34cc030726beb4be0224/pillow-13.0.0-cp315-cp315-win_arm64.whl", hash = "sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0", size = 2695513, upload-time = "2026-10-16T22:17:30.013Z" }, + { url = "https://files.pythonhosted.org/packages/23/16/e9120defdb8bff693b582d035312acc6f530cdc2bc5238a87f410efccd82/pillow-13.0.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f", size = 5578129, upload-time = "2026-10-16T22:17:31.642Z" }, + { url = "https://files.pythonhosted.org/packages/47/22/76faf2bb2448f48a84cf894c1e2bb1090d7d2b1aa521ece131cb2f05d631/pillow-13.0.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88", size = 4857428, upload-time = "2026-10-16T22:17:33.43Z" }, + { url = "https://files.pythonhosted.org/packages/b3/64/0861ea4e03c50b14af9ed9dd8b9f388ab42f445c623a9fcbab19aef7aa55/pillow-13.0.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb", size = 6612582, upload-time = "2026-10-16T22:17:35.415Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/cd1e9b3f60a9f3d5ebe46da36955c26004092a501f7f2c42f9fcc26611bc/pillow-13.0.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe", size = 7230176, upload-time = "2026-10-16T22:17:37.362Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ef/ffb0a2fc85690714fb7ac662956949645d57f15bf4426f520da361e7b6e3/pillow-13.0.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22", size = 6703640, upload-time = "2026-10-16T22:17:39.313Z" }, + { url = "https://files.pythonhosted.org/packages/e0/c2/6402ee3ccb3363aeed8fdb3f9fe56703ac5377a539f0b81b53999d7e121f/pillow-13.0.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630", size = 7309231, upload-time = "2026-10-16T22:17:41.021Z" }, + { url = "https://files.pythonhosted.org/packages/fc/96/d6b82e8b8c06ad143b66b12e0b31051e4bf489b4fabca1f390a14fe1ebde/pillow-13.0.0-cp315-cp315t-win32.whl", hash = "sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2", size = 6517200, upload-time = "2026-10-16T22:17:42.886Z" }, + { url = "https://files.pythonhosted.org/packages/60/d4/0062e6005e9b270b165015aafd343364ac378ec5a685431ea42baad33357/pillow-13.0.0-cp315-cp315t-win_amd64.whl", hash = "sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43", size = 7287488, upload-time = "2026-10-16T22:17:45.56Z" }, + { url = "https://files.pythonhosted.org/packages/e4/81/b8eb349746be7bce47dd95337c6634e84910af91b59aaa9159c034c614fa/pillow-13.0.0-cp315-cp315t-win_arm64.whl", hash = "sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8", size = 2698073, upload-time = "2026-10-16T22:17:47.556Z" }, { url = "https://files.pythonhosted.org/packages/68/c0/fb7b1caf25ed5ac5d46116f09b8b0706b9559833c65c667cecb1a53153f2/pillow-13.0.0-pp311-pypy311_pp80-macosx_10_15_x86_64.whl", hash = "sha256:405be6058e206a86fedce2f715ecc2343f94072d7925251632dffa2c05f70dca", size = 5521630, upload-time = "2026-10-16T22:17:50.008Z" }, { url = "https://files.pythonhosted.org/packages/1a/e6/4d11d5b1137f4fc080605cce410cfaae4f303ad400bdec75d27d3083fa29/pillow-13.0.0-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:b84be41387fc00db28c13c02e30d42b02d8d8f6a4a741a1aee6012ca7f1f6039", size = 4806023, upload-time = "2026-10-16T22:17:51.932Z" }, { url = "https://files.pythonhosted.org/packages/f1/45/0923202a658e8ff6aafd72c1a4395ebadcdebaa182c437d8595d3dd3db32/pillow-13.0.0-pp311-pypy311_pp80-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a192360d512305fe92d3053e4f7627bbad1f7708f1f4ad38cccd9336acabe9ad", size = 5464831, upload-time = "2026-10-16T22:17:53.742Z" }, @@ -7509,10 +7894,12 @@ dependencies = [ { name = "httpx" }, { name = "jinja2" }, { name = "nodeenv" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "python-dotenv" }, { name = "tomlkit" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2b/62/23a2638aa14d8eefd16bc4b230a6e65d8de5f69bfaeeaf954654c923f02e/prisma-0.11.0.tar.gz", hash = "sha256:3f2f2fd2361e1ec5ff655f2a04c7860c2f2a5bc4c91f78ca9c5c6349735bf693", size = 94181, upload-time = "2023-10-22T22:59:55.817Z" } wheels = [ @@ -7724,77 +8111,88 @@ wheels = [ [[package]] name = "psycopg" -version = "3.3.3" +version = "3.3.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, { name = "tzdata", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d3/b6/379d0a960f8f435ec78720462fd94c4863e7a31237cf81bf76d0af5883bf/psycopg-3.3.3.tar.gz", hash = "sha256:5e9a47458b3c1583326513b2556a2a9473a1001a56c9efe9e587245b43148dd9", size = 165624, upload-time = "2026-02-18T16:52:16.546Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/26/3ea4ca5eaea1c0debcdf7ee7c1613fbe721dc27a03c461c0817ffd8a0601/psycopg-3.3.6.tar.gz", hash = "sha256:c081f2250df751a943036e42db6df4571c66cd0aabe8291a7a506512b12007d2", size = 168171, upload-time = "2026-09-18T13:22:55.152Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/5b/181e2e3becb7672b502f0ed7f16ed7352aca7c109cfb94cf3878a9186db9/psycopg-3.3.3-py3-none-any.whl", hash = "sha256:f96525a72bcfade6584ab17e89de415ff360748c766f0106959144dcbb38c698", size = 212768, upload-time = "2026-02-18T16:46:27.365Z" }, + { url = "https://files.pythonhosted.org/packages/4e/de/748bd7609c71cae5d737f0ba9192f19329f70180ecda8fff3cac02c5abe3/psycopg-3.3.6-py3-none-any.whl", hash = "sha256:a1db9f7148b06a28606767efaca51fa6f9398c5c0a3810519be69d7000bdb631", size = 215490, upload-time = "2026-09-18T13:15:29.374Z" }, ] [[package]] name = "psycopg-binary" -version = "3.3.3" +version = "3.3.6" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/d8/a763308a41e2ecfb6256ba0877d340c2f2b124c8b2746401863d96fa2c7a/psycopg_binary-3.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b3385b58b2fe408a13d084c14b8dcf468cd36cbbe774408250facc128f9fa75c", size = 4609758, upload-time = "2026-02-18T16:46:33.132Z" }, - { url = "https://files.pythonhosted.org/packages/6c/a9/f8a683e85400c1208685e7c895abc049dc13aa0b6ea989e6adf0a3681fe0/psycopg_binary-3.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1bef235a50a80f6aba05147002bc354559657cb6386dbd04d8e1c97d1d7cbe84", size = 4676740, upload-time = "2026-02-18T16:46:42.904Z" }, - { url = "https://files.pythonhosted.org/packages/e3/7d/03512c4aaac8a58fc3b1221f38293aa517a1950d10ef8646c72c49addc7d/psycopg_binary-3.3.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:97c839717bf8c8df3f6d983a20949c4fb22e2a34ee172e3e427ede363feda27b", size = 5496335, upload-time = "2026-02-18T16:46:51.517Z" }, - { url = "https://files.pythonhosted.org/packages/8a/bc/23319b4b1c2c0b810d225e1b6f16efbb16150074fc0ea96bfcabdf59ee09/psycopg_binary-3.3.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:48e500cf1c0984dacf1f28ea482c3cdbb4c2288d51c336c04bc64198ab21fc51", size = 5172032, upload-time = "2026-02-18T16:47:00.878Z" }, - { url = "https://files.pythonhosted.org/packages/aa/c8/6d61dc0a56654c558a37b2d9b2094e470aa12621305cc7935fd769122e32/psycopg_binary-3.3.3-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb36a08859b9432d94ea6b26ec41a2f98f83f14868c91321d0c1e11f672eeae7", size = 6763107, upload-time = "2026-02-18T16:47:11.784Z" }, - { url = "https://files.pythonhosted.org/packages/9e/b5/e2a3c90aa1059f5b5f593379caad7be3cc3c2ce1ddfc7730e39854e174fe/psycopg_binary-3.3.3-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0dde92cfde09293fb63b3f547919ba7d73bd2654573c03502b3263dd0218e44e", size = 5006494, upload-time = "2026-02-18T16:47:17.062Z" }, - { url = "https://files.pythonhosted.org/packages/5d/3e/bf126e0a1f864e191b7f3eeea667ee2ce13d582b036255fb8b12946d1f7a/psycopg_binary-3.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:78c9ce98caaf82ac8484d269791c1b403d7598633e0e4e2fa1097baae244e2f1", size = 4533850, upload-time = "2026-02-18T16:47:21.673Z" }, - { url = "https://files.pythonhosted.org/packages/f4/d8/bb5e8d395deb945629aa0c65d12ab90ec3bfcbdf56be89e2a84d001864c9/psycopg_binary-3.3.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:d593612758d0041cb13cb0003f7f8d3fabb7ad9319e651e78afae49b1cf5860e", size = 4223316, upload-time = "2026-02-18T16:47:25.82Z" }, - { url = "https://files.pythonhosted.org/packages/c2/70/33eef61b0f0fd41ebf93b9699f44067313a45016827f67b3c8cc41f0a7ab/psycopg_binary-3.3.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:f24e8e17035200a465c178e9ea945527ad0738118694184c450f1192a452ff25", size = 3954515, upload-time = "2026-02-18T16:47:30.434Z" }, - { url = "https://files.pythonhosted.org/packages/ea/db/27c2b3b9698e713e83e11e8540daa27516f9e90390ec21a41091cb15fcaf/psycopg_binary-3.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e7b607f0e14f2a4cf7e78a05ebd13df6144acfba87cb90842e70d3f125d9f53f", size = 4260274, upload-time = "2026-02-18T16:47:36.128Z" }, - { url = "https://files.pythonhosted.org/packages/a1/3b/71e5d603059bf5474215f573a3e2d357a4e95672b26e04d41674400d4862/psycopg_binary-3.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:b27d3a23c79fa59557d2cc63a7e8bb4c7e022c018558eda36f9d7c4e6b99a6e0", size = 3557375, upload-time = "2026-02-18T16:47:42.799Z" }, - { url = "https://files.pythonhosted.org/packages/be/c0/b389119dd754483d316805260f3e73cdcad97925839107cc7a296f6132b1/psycopg_binary-3.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a89bb9ee11177b2995d87186b1d9fa892d8ea725e85eab28c6525e4cc14ee048", size = 4609740, upload-time = "2026-02-18T16:47:51.093Z" }, - { url = "https://files.pythonhosted.org/packages/cf/e3/9976eef20f61840285174d360da4c820a311ab39d6b82fa09fbb545be825/psycopg_binary-3.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f7d0cf072c6fbac3795b08c98ef9ea013f11db609659dcfc6b1f6cc31f9e181", size = 4676837, upload-time = "2026-02-18T16:47:55.523Z" }, - { url = "https://files.pythonhosted.org/packages/9f/f2/d28ba2f7404fd7f68d41e8a11df86313bd646258244cb12a8dd83b868a97/psycopg_binary-3.3.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:90eecd93073922f085967f3ed3a98ba8c325cbbc8c1a204e300282abd2369e13", size = 5497070, upload-time = "2026-02-18T16:47:59.929Z" }, - { url = "https://files.pythonhosted.org/packages/de/2f/6c5c54b815edeb30a281cfcea96dc93b3bb6be939aea022f00cab7aa1420/psycopg_binary-3.3.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:dac7ee2f88b4d7bb12837989ca354c38d400eeb21bce3b73dac02622f0a3c8d6", size = 5172410, upload-time = "2026-02-18T16:48:05.665Z" }, - { url = "https://files.pythonhosted.org/packages/51/75/8206c7008b57de03c1ada46bd3110cc3743f3fd9ed52031c4601401d766d/psycopg_binary-3.3.3-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b62cf8784eb6d35beaee1056d54caf94ec6ecf2b7552395e305518ab61eb8fd2", size = 6763408, upload-time = "2026-02-18T16:48:13.541Z" }, - { url = "https://files.pythonhosted.org/packages/d4/5a/ea1641a1e6c8c8b3454b0fcb43c3045133a8b703e6e824fae134088e63bd/psycopg_binary-3.3.3-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a39f34c9b18e8f6794cca17bfbcd64572ca2482318db644268049f8c738f35a6", size = 5006255, upload-time = "2026-02-18T16:48:22.176Z" }, - { url = "https://files.pythonhosted.org/packages/aa/fb/538df099bf55ae1637d52d7ccb6b9620b535a40f4c733897ac2b7bb9e14c/psycopg_binary-3.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:883d68d48ca9ff3cb3d10c5fdebea02c79b48eecacdddbf7cce6e7cdbdc216b8", size = 4532694, upload-time = "2026-02-18T16:48:27.338Z" }, - { url = "https://files.pythonhosted.org/packages/a1/d1/00780c0e187ea3c13dfc53bd7060654b2232cd30df562aac91a5f1c545ac/psycopg_binary-3.3.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:cab7bc3d288d37a80aa8c0820033250c95e40b1c2b5c57cf59827b19c2a8b69d", size = 4222833, upload-time = "2026-02-18T16:48:31.221Z" }, - { url = "https://files.pythonhosted.org/packages/7a/34/a07f1ff713c51d64dc9f19f2c32be80299a2055d5d109d5853662b922cb4/psycopg_binary-3.3.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:56c767007ca959ca32f796b42379fc7e1ae2ed085d29f20b05b3fc394f3715cc", size = 3952818, upload-time = "2026-02-18T16:48:35.869Z" }, - { url = "https://files.pythonhosted.org/packages/d3/67/d33f268a7759b4445f3c9b5a181039b01af8c8263c865c1be7a6444d4749/psycopg_binary-3.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:da2f331a01af232259a21573a01338530c6016dcfad74626c01330535bcd8628", size = 4258061, upload-time = "2026-02-18T16:48:41.365Z" }, - { url = "https://files.pythonhosted.org/packages/b4/3b/0d8d2c5e8e29ccc07d28c8af38445d9d9abcd238d590186cac82ee71fc84/psycopg_binary-3.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:19f93235ece6dbfc4036b5e4f6d8b13f0b8f2b3eeb8b0bd2936d406991bcdd40", size = 3558915, upload-time = "2026-02-18T16:48:46.679Z" }, - { url = "https://files.pythonhosted.org/packages/90/15/021be5c0cbc5b7c1ab46e91cc3434eb42569f79a0592e67b8d25e66d844d/psycopg_binary-3.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6698dbab5bcef8fdb570fc9d35fd9ac52041771bfcfe6fd0fc5f5c4e36f1e99d", size = 4591170, upload-time = "2026-02-18T16:48:55.594Z" }, - { url = "https://files.pythonhosted.org/packages/f1/54/a60211c346c9a2f8c6b272b5f2bbe21f6e11800ce7f61e99ba75cf8b63e1/psycopg_binary-3.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:329ff393441e75f10b673ae99ab45276887993d49e65f141da20d915c05aafd8", size = 4670009, upload-time = "2026-02-18T16:49:03.608Z" }, - { url = "https://files.pythonhosted.org/packages/c1/53/ac7c18671347c553362aadbf65f92786eef9540676ca24114cc02f5be405/psycopg_binary-3.3.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:eb072949b8ebf4082ae24289a2b0fd724da9adc8f22743409d6fd718ddb379df", size = 5469735, upload-time = "2026-02-18T16:49:10.128Z" }, - { url = "https://files.pythonhosted.org/packages/7f/c3/4f4e040902b82a344eff1c736cde2f2720f127fe939c7e7565706f96dd44/psycopg_binary-3.3.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:263a24f39f26e19ed7fc982d7859a36f17841b05bebad3eb47bb9cd2dd785351", size = 5152919, upload-time = "2026-02-18T16:49:16.335Z" }, - { url = "https://files.pythonhosted.org/packages/0c/e7/d929679c6a5c212bcf738806c7c89f5b3d0919f2e1685a0e08d6ff877945/psycopg_binary-3.3.3-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5152d50798c2fa5bd9b68ec68eb68a1b71b95126c1d70adaa1a08cd5eefdc23d", size = 6738785, upload-time = "2026-02-18T16:49:22.687Z" }, - { url = "https://files.pythonhosted.org/packages/69/b0/09703aeb69a9443d232d7b5318d58742e8ca51ff79f90ffe6b88f1db45e7/psycopg_binary-3.3.3-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9d6a1e56dd267848edb824dbeb08cf5bac649e02ee0b03ba883ba3f4f0bd54f2", size = 4979008, upload-time = "2026-02-18T16:49:27.313Z" }, - { url = "https://files.pythonhosted.org/packages/cc/a6/e662558b793c6e13a7473b970fee327d635270e41eded3090ef14045a6a5/psycopg_binary-3.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73eaaf4bb04709f545606c1db2f65f4000e8a04cdbf3e00d165a23004692093e", size = 4508255, upload-time = "2026-02-18T16:49:31.575Z" }, - { url = "https://files.pythonhosted.org/packages/5f/7f/0f8b2e1d5e0093921b6f324a948a5c740c1447fbb45e97acaf50241d0f39/psycopg_binary-3.3.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:162e5675efb4704192411eaf8e00d07f7960b679cd3306e7efb120bb8d9456cc", size = 4189166, upload-time = "2026-02-18T16:49:35.801Z" }, - { url = "https://files.pythonhosted.org/packages/92/ec/ce2e91c33bc8d10b00c87e2f6b0fb570641a6a60042d6a9ae35658a3a797/psycopg_binary-3.3.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:fab6b5e37715885c69f5d091f6ff229be71e235f272ebaa35158d5a46fd548a0", size = 3924544, upload-time = "2026-02-18T16:49:41.129Z" }, - { url = "https://files.pythonhosted.org/packages/c5/2f/7718141485f73a924205af60041c392938852aa447a94c8cbd222ff389a1/psycopg_binary-3.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a4aab31bd6d1057f287c96c0effca3a25584eb9cc702f282ecb96ded7814e830", size = 4235297, upload-time = "2026-02-18T16:49:46.726Z" }, - { url = "https://files.pythonhosted.org/packages/57/f9/1add717e2643a003bbde31b1b220172e64fbc0cb09f06429820c9173f7fc/psycopg_binary-3.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:59aa31fe11a0e1d1bcc2ce37ed35fe2ac84cd65bb9036d049b1a1c39064d0f14", size = 3547659, upload-time = "2026-02-18T16:49:52.999Z" }, - { url = "https://files.pythonhosted.org/packages/03/0a/cac9fdf1df16a269ba0e5f0f06cac61f826c94cadb39df028cdfe19d3a33/psycopg_binary-3.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:05f32239aec25c5fb15f7948cffdc2dc0dac098e48b80a140e4ba32b572a2e7d", size = 4590414, upload-time = "2026-02-18T16:50:01.441Z" }, - { url = "https://files.pythonhosted.org/packages/9c/c0/d8f8508fbf440edbc0099b1abff33003cd80c9e66eb3a1e78834e3fb4fb9/psycopg_binary-3.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7c84f9d214f2d1de2fafebc17fa68ac3f6561a59e291553dfc45ad299f4898c1", size = 4669021, upload-time = "2026-02-18T16:50:08.803Z" }, - { url = "https://files.pythonhosted.org/packages/04/05/097016b77e343b4568feddf12c72171fc513acef9a4214d21b9478569068/psycopg_binary-3.3.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e77957d2ba17cada11be09a5066d93026cdb61ada7c8893101d7fe1c6e1f3925", size = 5467453, upload-time = "2026-02-18T16:50:14.985Z" }, - { url = "https://files.pythonhosted.org/packages/91/23/73244e5feb55b5ca109cede6e97f32ef45189f0fdac4c80d75c99862729d/psycopg_binary-3.3.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:42961609ac07c232a427da7c87a468d3c82fee6762c220f38e37cfdacb2b178d", size = 5151135, upload-time = "2026-02-18T16:50:24.82Z" }, - { url = "https://files.pythonhosted.org/packages/11/49/5309473b9803b207682095201d8708bbc7842ddf3f192488a69204e36455/psycopg_binary-3.3.3-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ae07a3114313dd91fce686cab2f4c44af094398519af0e0f854bc707e1aeedf1", size = 6737315, upload-time = "2026-02-18T16:50:35.106Z" }, - { url = "https://files.pythonhosted.org/packages/d4/5d/03abe74ef34d460b33c4d9662bf6ec1dd38888324323c1a1752133c10377/psycopg_binary-3.3.3-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d257c58d7b36a621dcce1d01476ad8b60f12d80eb1406aee4cf796f88b2ae482", size = 4979783, upload-time = "2026-02-18T16:50:42.067Z" }, - { url = "https://files.pythonhosted.org/packages/f0/6c/3fbf8e604e15f2f3752900434046c00c90bb8764305a1b81112bff30ba24/psycopg_binary-3.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:07c7211f9327d522c9c47560cae00a4ecf6687f4e02d779d035dd3177b41cb12", size = 4509023, upload-time = "2026-02-18T16:50:50.116Z" }, - { url = "https://files.pythonhosted.org/packages/9c/6b/1a06b43b7c7af756c80b67eac8bfaa51d77e68635a8a8d246e4f0bb7604a/psycopg_binary-3.3.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:8e7e9eca9b363dbedeceeadd8be97149d2499081f3c52d141d7cd1f395a91f83", size = 4185874, upload-time = "2026-02-18T16:50:55.97Z" }, - { url = "https://files.pythonhosted.org/packages/2b/d3/bf49e3dcaadba510170c8d111e5e69e5ae3f981c1554c5bb71c75ce354bb/psycopg_binary-3.3.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:cb85b1d5702877c16f28d7b92ba030c1f49ebcc9b87d03d8c10bf45a2f1c7508", size = 3925668, upload-time = "2026-02-18T16:51:03.299Z" }, - { url = "https://files.pythonhosted.org/packages/f8/92/0aac830ed6a944fe334404e1687a074e4215630725753f0e3e9a9a595b62/psycopg_binary-3.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4d4606c84d04b80f9138d72f1e28c6c02dc5ae0c7b8f3f8aaf89c681ce1cd1b1", size = 4234973, upload-time = "2026-02-18T16:51:09.097Z" }, - { url = "https://files.pythonhosted.org/packages/2e/96/102244653ee5a143ece5afe33f00f52fe64e389dfce8dbc87580c6d70d3d/psycopg_binary-3.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:74eae563166ebf74e8d950ff359be037b85723d99ca83f57d9b244a871d6c13b", size = 3551342, upload-time = "2026-02-18T16:51:13.892Z" }, - { url = "https://files.pythonhosted.org/packages/a2/71/7a57e5b12275fe7e7d84d54113f0226080423a869118419c9106c083a21c/psycopg_binary-3.3.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:497852c5eaf1f0c2d88ab74a64a8097c099deac0c71de1cbcf18659a8a04a4b2", size = 4607368, upload-time = "2026-02-18T16:51:19.295Z" }, - { url = "https://files.pythonhosted.org/packages/c7/04/cb834f120f2b2c10d4003515ef9ca9d688115b9431735e3936ae48549af8/psycopg_binary-3.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:258d1ea53464d29768bf25930f43291949f4c7becc706f6e220c515a63a24edd", size = 4687047, upload-time = "2026-02-18T16:51:23.84Z" }, - { url = "https://files.pythonhosted.org/packages/40/e9/47a69692d3da9704468041aa5ed3ad6fc7f6bb1a5ae788d261a26bbca6c7/psycopg_binary-3.3.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:111c59897a452196116db12e7f608da472fbff000693a21040e35fc978b23430", size = 5487096, upload-time = "2026-02-18T16:51:29.645Z" }, - { url = "https://files.pythonhosted.org/packages/0b/b6/0e0dd6a2f802864a4ae3dbadf4ec620f05e3904c7842b326aafc43e5f464/psycopg_binary-3.3.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:17bb6600e2455993946385249a3c3d0af52cd70c1c1cdbf712e9d696d0b0bf1b", size = 5168720, upload-time = "2026-02-18T16:51:36.499Z" }, - { url = "https://files.pythonhosted.org/packages/6f/0d/977af38ac19a6b55d22dff508bd743fd7c1901e1b73657e7937c7cccb0a3/psycopg_binary-3.3.3-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:642050398583d61c9856210568eb09a8e4f2fe8224bf3be21b67a370e677eead", size = 6762076, upload-time = "2026-02-18T16:51:43.167Z" }, - { url = "https://files.pythonhosted.org/packages/34/40/912a39d48322cf86895c0eaf2d5b95cb899402443faefd4b09abbba6b6e1/psycopg_binary-3.3.3-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:533efe6dc3a7cba5e2a84e38970786bb966306863e45f3db152007e9f48638a6", size = 4997623, upload-time = "2026-02-18T16:51:47.707Z" }, - { url = "https://files.pythonhosted.org/packages/98/0c/c14d0e259c65dc7be854d926993f151077887391d5a081118907a9d89603/psycopg_binary-3.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:5958dbf28b77ce2033482f6cb9ef04d43f5d8f4b7636e6963d5626f000efb23e", size = 4532096, upload-time = "2026-02-18T16:51:51.421Z" }, - { url = "https://files.pythonhosted.org/packages/39/21/8b7c50a194cfca6ea0fd4d1f276158307785775426e90700ab2eba5cd623/psycopg_binary-3.3.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:a6af77b6626ce92b5817bf294b4d45ec1a6161dba80fc2d82cdffdd6814fd023", size = 4208884, upload-time = "2026-02-18T16:51:57.336Z" }, - { url = "https://files.pythonhosted.org/packages/c7/2c/a4981bf42cf30ebba0424971d7ce70a222ae9b82594c42fc3f2105d7b525/psycopg_binary-3.3.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:47f06fcbe8542b4d96d7392c476a74ada521c5aebdb41c3c0155f6595fc14c8d", size = 3944542, upload-time = "2026-02-18T16:52:04.266Z" }, - { url = "https://files.pythonhosted.org/packages/60/e9/b7c29b56aa0b85a4e0c4d89db691c1ceef08f46a356369144430c155a2f5/psycopg_binary-3.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e7800e6c6b5dc4b0ca7cc7370f770f53ac83886b76afda0848065a674231e856", size = 4254339, upload-time = "2026-02-18T16:52:10.444Z" }, - { url = "https://files.pythonhosted.org/packages/98/5a/291d89f44d3820fffb7a04ebc8f3ef5dda4f542f44a5daea0c55a84abf45/psycopg_binary-3.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:165f22ab5a9513a3d7425ffb7fcc7955ed8ccaeef6d37e369d6cc1dff1582383", size = 3652796, upload-time = "2026-02-18T16:52:14.02Z" }, + { url = "https://files.pythonhosted.org/packages/52/92/00350a66de0af05e41d01aa3134e3970045e816afed3f99d58ec1abe15b2/psycopg_binary-3.3.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7beb3e41c9a1e509f3ed85263386588cbe3e975aa67be21f79f44fd35ffaeefc", size = 4728682, upload-time = "2026-09-18T13:15:36.605Z" }, + { url = "https://files.pythonhosted.org/packages/91/fc/afa9c7fd316a469af7ede6ebb020eac482f5d827fae57d5310c9bc0c41ae/psycopg_binary-3.3.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:aa73160077345ec21b3f51e8e24b3de2e99586217e497629326eb9b2ea88c52e", size = 4777111, upload-time = "2026-09-18T13:15:46.566Z" }, + { url = "https://files.pythonhosted.org/packages/f2/44/7c1e015f1bc56b36ff1369f09e852b2d83ccefd5a669a42633a916cdedc4/psycopg_binary-3.3.6-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f87dbdc42e78ee0f7ea180c03f8c78e80a949e373066629bd90fefff10552dff", size = 5592755, upload-time = "2026-09-18T13:15:52.886Z" }, + { url = "https://files.pythonhosted.org/packages/3b/ae/314a251ca918cdac380bce1b87839ade9355382ea749e6ef3ba75ba0c09f/psycopg_binary-3.3.6-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a9348c5b43a3bb5ef8c2e89d5237c9c87eeafb01d338c84a7aebbc5cd0313299", size = 5267949, upload-time = "2026-09-18T13:16:00.53Z" }, + { url = "https://files.pythonhosted.org/packages/b6/9f/3bb0cfe9bb0f31ca57cf486ddc8c9ac51251aed8181bf88ff870b2623105/psycopg_binary-3.3.6-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a52991594ac4db888c7d39bccef331797e30cb31a95cae02cf2607f83a42dc2", size = 6862403, upload-time = "2026-09-18T13:16:10.385Z" }, + { url = "https://files.pythonhosted.org/packages/c4/d6/7032c10309c3155e9b24300fdcc9a1afa539cfd20ce52fdef74a46f10161/psycopg_binary-3.3.6-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5ea8beeb5541780b4b50b462eeacbc4f594ce3b911dc20c81c75f267876f71d2", size = 5107271, upload-time = "2026-09-18T13:16:16.843Z" }, + { url = "https://files.pythonhosted.org/packages/61/cc/79add2cf92684cf1a81da134b32caa662c25c72d0cc905d181ef4455f834/psycopg_binary-3.3.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:198a48e68cc99ccac03ba95ac857e73aa66f3bf6be77019fafb0832a05f7ad03", size = 4627984, upload-time = "2026-09-18T13:16:23.889Z" }, + { url = "https://files.pythonhosted.org/packages/c9/48/6dfb14f9350c14af6a2edb3c31262051b8cd94e2186e4b831e46dbbe8cd9/psycopg_binary-3.3.6-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:fa34eb47969297471db7b7f193622c7e3ee839ec05abd05f1fe104d5b1b1dcf4", size = 4322054, upload-time = "2026-09-18T13:16:29.33Z" }, + { url = "https://files.pythonhosted.org/packages/29/35/2982338716a91cbb4dfc866be015be4457ee8106a445aabf3d1fb6a270e0/psycopg_binary-3.3.6-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b979a42815410432420275412633960807178b1ce26591a16ce06e78a5bd4bb2", size = 4048471, upload-time = "2026-09-18T13:16:34.119Z" }, + { url = "https://files.pythonhosted.org/packages/24/e1/171b1db1542c5f76a678b7ee0a7800bebc9735a0a03417c76cf948bfd63c/psycopg_binary-3.3.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:889e42acec10450185e0cdfb396f375e2c1a8d7737c114830a7fde4654f59e30", size = 4353931, upload-time = "2026-09-18T13:16:38.692Z" }, + { url = "https://files.pythonhosted.org/packages/08/89/4424e62a944eef40bd9326ada4ae23802b28eab6502af91e84ef7bba74fb/psycopg_binary-3.3.6-cp310-cp310-win_amd64.whl", hash = "sha256:cbd5f73073ed19c378d4c35499db1e3e703a5b1a324e521204065967bfaa7a18", size = 3674362, upload-time = "2026-09-18T13:16:44.454Z" }, + { url = "https://files.pythonhosted.org/packages/70/86/b71166048974d49c6d136b2ed1c0e5bec0b974d8c4de5cbce7e86a9e412a/psycopg_binary-3.3.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:be4f9b3c9338ac5dd217c5847e21521b396c8117f78dc420d495a5c49bbef874", size = 4728002, upload-time = "2026-09-18T13:16:53.393Z" }, + { url = "https://files.pythonhosted.org/packages/12/1d/1e06c0de7ed5aed898acb87544eac6ef0bc7d752a67ec6e5d6b835e9b40c/psycopg_binary-3.3.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f0535693ce476a722b718b002d5d2c27d47e71ca945276ac194409c98e74c492", size = 4775403, upload-time = "2026-09-18T13:16:58.939Z" }, + { url = "https://files.pythonhosted.org/packages/84/02/2ffcbc43f8e4bbc38e5286a22013bcac01898d13cd38325f60dd5428a8af/psycopg_binary-3.3.6-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3c9e663b2e800e3218994cf948c11bcc2844e6491b34aa80d089baf6531827bf", size = 5594145, upload-time = "2026-09-18T13:17:08.515Z" }, + { url = "https://files.pythonhosted.org/packages/e1/25/031dae2c7d2e7e77dcf5b1962c1e0684fa548d7af0ff6707b6b5e6054ca7/psycopg_binary-3.3.6-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a2e44a342d2aee40508e28a563d8961c39d9bbd8cae36d8578f0a3c6658aab0f", size = 5267985, upload-time = "2026-09-18T13:17:16.24Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e5/94c89ada3c003a4d858178f3bba49a35e0297ef2aad659b80eb5e380e690/psycopg_binary-3.3.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f598f19fa9a91540b5cee17932ffd227b7b53a481605bcc4573c0eafa647300", size = 6860986, upload-time = "2026-09-18T13:17:23.348Z" }, + { url = "https://files.pythonhosted.org/packages/9d/a0/81bf499d095adee8413bd19822a6872fbfa21663ec78014a68d83a8db83c/psycopg_binary-3.3.6-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6ff05561e4a067d35507dc5c90f1deb2ec1c9703ac5cccc1bc26e08a197f9c5a", size = 5106702, upload-time = "2026-09-18T13:17:28.847Z" }, + { url = "https://files.pythonhosted.org/packages/00/75/99d56da64c27bd985fd82c6ecbf7976b724ac638fdd1654ef995323a1a26/psycopg_binary-3.3.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:566dd827f17728efdf7d88a5b066f815170f6fdad13967ae952842d90e6aaa9f", size = 4627090, upload-time = "2026-09-18T13:17:36.668Z" }, + { url = "https://files.pythonhosted.org/packages/3e/0c/0222171d11233332c6a24b1cef1578215f0ffddf3642eb8dd8c4448ad69f/psycopg_binary-3.3.6-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9b2f11794e017ce340934e35de46181c46ef71ec75ea3d85dd75cd836761c01e", size = 4320353, upload-time = "2026-09-18T13:17:42.526Z" }, + { url = "https://files.pythonhosted.org/packages/62/6f/e1cc2a28dd1228c67c969ba6fd37cd8726b312e2ff51380f847ddb38ccde/psycopg_binary-3.3.6-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:910ace140e3e7b7596898d083f37a8fe90c5c40684252ad4e682364b2cd3deba", size = 4047253, upload-time = "2026-09-18T13:17:47.068Z" }, + { url = "https://files.pythonhosted.org/packages/d8/fd/38b64790ce7a515b1dbd2bab3d119637a858aeb22c380cf4859bc4ce0e42/psycopg_binary-3.3.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37e517c146b185f9c0c6e8d0a0ebbdeeeb67896af28466e032bc810d0c7dc7a7", size = 4353208, upload-time = "2026-09-18T13:17:52.41Z" }, + { url = "https://files.pythonhosted.org/packages/f7/dc/45386530ceb2a8c789a226de9b9b34eca8fccf1feba2e4ef68a6aca50c56/psycopg_binary-3.3.6-cp311-cp311-win_amd64.whl", hash = "sha256:c7f92daa0d2a1c76f07264abddf8cbabd30152a2f09c3270e50f0c7efdf5dcac", size = 3675638, upload-time = "2026-09-18T13:17:58.112Z" }, + { url = "https://files.pythonhosted.org/packages/e6/01/2cdd1824e58b4467ee0b9498664cd28c42d8794db6b1e35b6bcb834f0044/psycopg_binary-3.3.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3f84dab25e0385692ee13274c68678377e0b1a70ab9d14e56264cbf61f60c62d", size = 4707086, upload-time = "2026-09-18T13:18:05.138Z" }, + { url = "https://files.pythonhosted.org/packages/f6/76/de9948ac06895261c84d5b9fbe283d8f3c5bc9f070691b8d9eaa1b51e322/psycopg_binary-3.3.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:612382ac3ed13651c7fa44b5fee9fbf7baaa2ddbc6f500391672682c5f1df9e0", size = 4769607, upload-time = "2026-09-18T13:18:12.83Z" }, + { url = "https://files.pythonhosted.org/packages/76/a9/72436c9915ee4905964689e7f0e182ce7767cc0a0390b3ce703be8177625/psycopg_binary-3.3.6-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:366db6e97e66b37211475f20c4c1324a2dc0dd825e46d4e87f9d599304d276f9", size = 5554134, upload-time = "2026-09-18T13:18:21.175Z" }, + { url = "https://files.pythonhosted.org/packages/0a/42/948bb3d2617795093512613fd96ba380e922992c7908fbc073858147d196/psycopg_binary-3.3.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1679a1cb93fbe5a6d1fd58d82cbddcc6fcb8c61446ba7cae6eb2a7b19bc585de", size = 5235723, upload-time = "2026-09-18T13:18:27.071Z" }, + { url = "https://files.pythonhosted.org/packages/99/47/93e823ff1b0088400703410939c9bda3e63ed9c850b3ee088e8769f4c10b/psycopg_binary-3.3.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:37d40450659401600e6d043ff586c89a71a69f33cbb8bcdba6cdb2569beecdbe", size = 6833587, upload-time = "2026-09-18T13:18:33.794Z" }, + { url = "https://files.pythonhosted.org/packages/5e/2d/ecc69c847795aa704041a9f5667a6b0938a088cf1853636d762a6938e493/psycopg_binary-3.3.6-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a5165300324efd5a772c48a88ab3a928513ab3979fca76553e62ee815f7b2b9c", size = 5070013, upload-time = "2026-09-18T13:18:39.628Z" }, + { url = "https://files.pythonhosted.org/packages/92/36/6126f0dac21713dcae91404f2a76da18598a6252339a8c669c46370d43b2/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d636338c8f21b0df2f84657b00bc34f9313f826ef93f1155bc743607e4a0c5eb", size = 4597367, upload-time = "2026-09-18T13:18:45.023Z" }, + { url = "https://files.pythonhosted.org/packages/4d/29/7ecfc04243b46c89ffd49924e9c5634ea904ef96c7d0f37e4073623584c1/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a4ee3bdd5468a725f2a4d9aab8a74b6d0279f768c8b5d3aeb102c5307ff3d59c", size = 4275419, upload-time = "2026-09-18T13:18:49.299Z" }, + { url = "https://files.pythonhosted.org/packages/6e/90/2f46d2e0de79706ac170df0a3637fe63c4498fc04f131f6049520b78b806/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:289aadd6a00e151203c081f708348ec89f1e483c9b510ef4ac3981f847f01f79", size = 4007358, upload-time = "2026-09-18T13:18:53.944Z" }, + { url = "https://files.pythonhosted.org/packages/03/48/6744e91291b751a8cf12d63d719977974bb94c84ceba913e7ddb2e478e51/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f21d057f3e5f5491067e5b292498073b73847d48799b099803fef100775fcc52", size = 4320156, upload-time = "2026-09-18T13:18:59.258Z" }, + { url = "https://files.pythonhosted.org/packages/1a/9b/94ff7fce53a64d5b286e2ec454e0a025cf3d6e6b4a9189bef16aa5de98b2/psycopg_binary-3.3.6-cp312-cp312-win_amd64.whl", hash = "sha256:e23a66a763fbe83fcc210bc77c27e5a5ea380ebf091c06f34d8561b695e5a40f", size = 3658864, upload-time = "2026-09-18T13:19:06.503Z" }, + { url = "https://files.pythonhosted.org/packages/b4/c3/c072584b69ad44a747b448cfc9766fecb8aae56e372a017e2ef668790057/psycopg_binary-3.3.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ad8f35e67cc16d1fad1fa8c88972dc9b3a3141ea67897399904edab96a301b6", size = 4712284, upload-time = "2026-09-18T13:19:13.451Z" }, + { url = "https://files.pythonhosted.org/packages/0a/b9/4283b785339e8e2318d03048994b093d650ea6289fabaa806b765dc0d449/psycopg_binary-3.3.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:373704aea331d3f3e3402c125a1543f5875e2986ebb54f97d1647942161f803f", size = 4772031, upload-time = "2026-09-18T13:19:18.524Z" }, + { url = "https://files.pythonhosted.org/packages/6f/72/7a1321d359246769fff1affffbd0132785a28f7f63c18524c15a502398f4/psycopg_binary-3.3.6-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b82491019b884d62318b5f30706c3d7e6d4e5a6cb7eabcb3edc0c1b0fdaceae9", size = 5556392, upload-time = "2026-09-18T13:19:24.418Z" }, + { url = "https://files.pythonhosted.org/packages/de/b0/c6f8a0585a5dacbea74e130bcfc66629390e8f5bbc79d2a8e806e8952150/psycopg_binary-3.3.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cec5ea900390897d0b46130f60bc2883bf19c314f9044235217c8be88b0ef269", size = 5237855, upload-time = "2026-09-18T13:19:31.257Z" }, + { url = "https://files.pythonhosted.org/packages/e2/fc/c3a7a8bbef7e945ec584ac61d460a612363ea398511cd0e220242b1d69f1/psycopg_binary-3.3.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98c02090d88f2ebc0ec1e8da538f77d225ce0fffecf372aa39262e62a1b054ef", size = 6833856, upload-time = "2026-09-18T13:19:43.622Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f2/8e80b921db728ebb68fc105bd7c4277f908210ad755bd6481d5ea7add740/psycopg_binary-3.3.6-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ee2c4728c691245e24501fcd7a97b5b381236b9985bc445bba88cdce7d1b5784", size = 5070730, upload-time = "2026-09-18T13:19:49.968Z" }, + { url = "https://files.pythonhosted.org/packages/54/6a/5b313e0c5348244f0e973aff3258bf86766656256d5ece8d541a53e35b4a/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f19cc87343eaa55255e76b31259a570072ac95d6ae82c92dd34b97691f5e49dc", size = 4598089, upload-time = "2026-09-18T13:19:56.426Z" }, + { url = "https://files.pythonhosted.org/packages/32/e9/db7f76ec24bf6699e92bf604e5c4bae10664a681a8999ef42aa0faf0f2c6/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdccb3a0e184b03e9baa673b15a809cf36c339c85dbda0ebc25a698846dfbee8", size = 4278481, upload-time = "2026-09-18T13:20:04.681Z" }, + { url = "https://files.pythonhosted.org/packages/61/83/72c67013656f4d6b547caabffb193e91d57e63f90eefdcc6d045c400e97d/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9892188bb15e5803beb51afe8a25add6b56be391a53058e8bca03b74e1e6bf22", size = 4009229, upload-time = "2026-09-18T13:20:11.905Z" }, + { url = "https://files.pythonhosted.org/packages/82/35/5e4500df2c999eb0faed8b184e6958b834172128274f06167a5deef4c19c/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3af90f92769d8cc10f94515ee7a0aef36ea85ca733a0ce22858f6e0953f41138", size = 4321467, upload-time = "2026-09-18T13:20:17.949Z" }, + { url = "https://files.pythonhosted.org/packages/55/7f/e350e1cf498ba2565c3f87b12f429d2012eb86b76c2b3845a19ee5fbb4d6/psycopg_binary-3.3.6-cp313-cp313-win_amd64.whl", hash = "sha256:0ebfad5d131de9f892ae9e70cc7616207768b6714b66a52d4612b8ceaf78b372", size = 3658179, upload-time = "2026-09-18T13:20:22.691Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b9/60711317c284a442511644ea7185b56ebe627606d6741e732cd16108c47b/psycopg_binary-3.3.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:b3f75dee0f9afafabe4edc52c4842f1e1878ed2069bd05b22d6fe961e97e4dba", size = 4720512, upload-time = "2026-09-18T13:20:29.278Z" }, + { url = "https://files.pythonhosted.org/packages/63/da/28befc84454cbc6374550de7746f591f8fe1b6165c1fce249652cc8291c4/psycopg_binary-3.3.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5927b7ba63153cd8e9862987290a2b783a5c590daf2a4ef981700cc3569166d4", size = 4782318, upload-time = "2026-09-18T13:20:35.401Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8a/0d21c2c833cdc0d4244c77e858e0ed37fa2abec2623be4fd686f617109ce/psycopg_binary-3.3.6-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:0bf08b749cc144f33b44a91b78e3f71c60eb07963746a0df5a100b36ce3d7475", size = 5567460, upload-time = "2026-09-18T13:20:41.902Z" }, + { url = "https://files.pythonhosted.org/packages/49/6d/7692d0d4e656b6cc9868d8acc2e3b42f17a0db4a625400a6d093cb0533a1/psycopg_binary-3.3.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:31cd942c23f613276b81a6e6598cefa12960058b0f46e1e874b540c793f6aca5", size = 5246902, upload-time = "2026-09-18T13:20:47.661Z" }, + { url = "https://files.pythonhosted.org/packages/d4/c1/b8a1f18fb1b7558a17f57f7cb3fc8bc93189feea2958925950b3acb15743/psycopg_binary-3.3.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4690cf67738f0e0e49a32aeec99bf0e4595cc2b4f1af984a4345394b1dcff91a", size = 6847192, upload-time = "2026-09-18T13:20:56.874Z" }, + { url = "https://files.pythonhosted.org/packages/a5/76/404f33519167c65cca88ec4998776f1dbebccc301ee977f0e62c47fb0826/psycopg_binary-3.3.6-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ad1c785e784cfd87e8436c6b7702f2d321fc39601bbaf29bc63a41a867091638", size = 5079573, upload-time = "2026-09-18T13:21:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d9/79e8fbc8f37262a415f3550f0bcc5f98037442bf3d12ef6cbae2056655ae/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:79a2a1c3449f6c3409427078ed1cec10de79f3023cb5f2504f0597d350ad46c7", size = 4613633, upload-time = "2026-09-18T13:21:10.664Z" }, + { url = "https://files.pythonhosted.org/packages/d4/47/96225db74be7d2ce04b3a58678b53cda610225055edf5faa775c9f501d8b/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:86147cb5d140341c3363fb5bacce31f8d5543902a46699d3c536b101bbceaf9e", size = 4293375, upload-time = "2026-09-18T13:21:16.027Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d2/18e9c779a5efd565250329adaf529ecc2b8b2ed5be5cb0f6ccee208cbfd9/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:7308c93cf0b19bbaf8e6ff0a6ad50d3c442385739245fe15a8d593bf841734a6", size = 4019883, upload-time = "2026-09-18T13:21:21.587Z" }, + { url = "https://files.pythonhosted.org/packages/ef/28/0cc654afc6c2cda982767f5679d3646b30b1ec86545bdaa9402202d6776c/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:05a83ac9fd52b9bca7cb5ab04b3691163170bd16f53defa27216ea3aa07ee781", size = 4332607, upload-time = "2026-09-18T13:21:27.63Z" }, + { url = "https://files.pythonhosted.org/packages/f1/3e/0a753a74fbd7aef120f286c016e09d3cc3f1daf7688f4a145d27281260b2/psycopg_binary-3.3.6-cp314-cp314-win_amd64.whl", hash = "sha256:1fbd30e537dab22cafdf080608f10148fe2a5f3a61294ddb5113caac8a623840", size = 3755671, upload-time = "2026-09-18T13:21:33.855Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b1/a372b9c02aea50148e71c9853e19efca8fa5ae2010a8e27243b9b8f790c0/psycopg_binary-3.3.6-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:bf8c8481d026b85dd70c5fa7dde85b2333aed0b32a2602bcd38a900cbd78a49c", size = 4719571, upload-time = "2026-09-18T13:21:41.437Z" }, + { url = "https://files.pythonhosted.org/packages/65/7c/811e3828c6b82e2f10c6c9cdd963cfc66f3e024026e5a69ac18530bad984/psycopg_binary-3.3.6-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:b599defe9190b17e9907c8b4d114c181e702c87efcd1b8a0ad40971cdcc4634a", size = 4781230, upload-time = "2026-09-18T13:21:49.516Z" }, + { url = "https://files.pythonhosted.org/packages/3e/15/9a784eed813ea9e97c294af3ead63d02b7b203502c66380336c50065e441/psycopg_binary-3.3.6-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b8ece331509f7a975b90501f41e83ad905e4141753fedf3f2711b2bc70a8efbc", size = 5566111, upload-time = "2026-09-18T13:21:58.089Z" }, + { url = "https://files.pythonhosted.org/packages/68/16/47194e002007c27337b11e49bf459c4b19727463f9aff2e1a90917bcc806/psycopg_binary-3.3.6-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c61617eaae0112ca154da87ffb99b73af2c74067acac28dfb9a4455b019dff2e", size = 5249963, upload-time = "2026-09-18T13:22:06.695Z" }, + { url = "https://files.pythonhosted.org/packages/53/84/5dcf9f310b11f0675cd860c6b2c70f58ce61798a3ee3f6f962b53fa358ca/psycopg_binary-3.3.6-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c6d19cb4999d03231e8730a5f66c8f5068bc3b532677eb39dab0f600bff3e312", size = 6847925, upload-time = "2026-09-18T13:22:13.088Z" }, + { url = "https://files.pythonhosted.org/packages/f3/06/1957a06dc22963c418c27b284929579de84f29c37ad1abe6dc6ee9e8cf25/psycopg_binary-3.3.6-cp315-cp315-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8cbb54454dbf1bbf2ff08dd7693e8d94ac94b1a20f70f4b3b813d52ecb5cbc1", size = 5087720, upload-time = "2026-09-18T13:22:17.959Z" }, + { url = "https://files.pythonhosted.org/packages/21/43/ac07d042bae99b57bf123bb473632f29af544008094da0ffd285ab8011e2/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:dc75da5a20951049f7b773145f998f69d181adad9c58a0ff36e0cf1d73c10e10", size = 4613412, upload-time = "2026-09-18T13:22:26.719Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b1/019156fbeafcefb4cccc9d109de4699493bceb8313c7545c8349e089dfbc/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:955e3dd94da361e052d2e49acf591017158dc8f8ed2c8a42c2e3943403c39dc2", size = 4292618, upload-time = "2026-09-18T13:22:33.042Z" }, + { url = "https://files.pythonhosted.org/packages/5d/0f/62113dc6b1df65983a1f2fc816c04b1edfa22f2ae9d4abee74ed267f4a96/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:c7753871eb57e6a5f4646f6168590c6653073dea5e9e720b201c8875332df4c8", size = 4027121, upload-time = "2026-09-18T13:22:38.334Z" }, + { url = "https://files.pythonhosted.org/packages/5d/d5/cf0cbd1ea5a7d8167fe2c6953efde19101f7b193bd61a23e6d622ad6854c/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:303732e798fe6729f8e12021b9c96107df8e95ecec4dd487c67b98ec2a59435e", size = 4336388, upload-time = "2026-09-18T13:22:45.576Z" }, + { url = "https://files.pythonhosted.org/packages/98/33/e2a5b36edf8aa422f6fa4b894756eb33dc93b36df5f65121280bb8b929c4/psycopg_binary-3.3.6-cp315-cp315-win_amd64.whl", hash = "sha256:2f122603f36050937982abf9668d8bc4769a79f7c93a65013b1c49f1cab7b56b", size = 3756154, upload-time = "2026-09-18T13:22:51.283Z" }, ] [[package]] @@ -7951,11 +8349,18 @@ wheels = [ name = "pydantic" version = "2.13.4" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.14.*'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", + "python_full_version < '3.11'", +] dependencies = [ { name = "annotated-types" }, - { name = "pydantic-core" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, + { name = "pydantic-core", version = "2.46.4", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-inspection", version = "0.4.2", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } wheels = [ @@ -7967,12 +8372,42 @@ email = [ { name = "email-validator" }, ] +[[package]] +name = "pydantic" +version = "2.14.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15'", +] +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core", version = "2.50.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-inspection", version = "0.4.4", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7c/0b/8e10b2e693af8ec54346a14caf36221334775975a747c9ceaa3f8371d96d/pydantic-2.14.1.tar.gz", hash = "sha256:94f478203dd03404682a1ada216965651dd74b1d2d5ffd62e00e0837caab5c26", size = 900362, upload-time = "2026-10-11T18:37:55.396Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/ea/a56b9fe5066f3537b7882f77e9c5dfb26d8c2eefdaed9b5fc73d57b4dc22/pydantic-2.14.1-py3-none-any.whl", hash = "sha256:9195d967ec791692a04438115466764fb8b9a27b31f14a760437694f40d6b454", size = 488106, upload-time = "2026-10-11T18:37:53.437Z" }, +] + +[package.optional-dependencies] +email = [ + { name = "email-validator" }, +] + [[package]] name = "pydantic-core" version = "2.46.4" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.14.*'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", + "python_full_version < '3.11'", +] dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } wheels = [ @@ -8083,14 +8518,166 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, ] +[[package]] +name = "pydantic-core" +version = "2.50.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15'", +] +dependencies = [ + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/24/af4ec4be49fbc810f35b0bdcacc3d433b56bbfa469fd3bfd4ae116cd9bf1/pydantic_core-2.50.1.tar.gz", hash = "sha256:e50d7b94baac6c7d09927fa5ca5800a0c7ee5015c7fcff65beb3a1931b5a6e09", size = 513515, upload-time = "2026-10-11T18:35:44.82Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/db/c9cc34707ba2c23ce90d54e8a4ee9307583ad054b9df2ce570ba550cb576/pydantic_core-2.50.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b281a3b0f0822618fe5e3e0d8a2048b6356b14388505dc9374ccffeb69989713", size = 2154222, upload-time = "2026-10-11T18:31:38.345Z" }, + { url = "https://files.pythonhosted.org/packages/c1/17/6963f4ee55b7df7658f5c036c0f90baf8cd3d413f5d8b551f227e8467afc/pydantic_core-2.50.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1fa4c8bc12c1354c5550c0c35c1852c8c1901e89e06561724e03f8d0342e1f87", size = 2048126, upload-time = "2026-10-11T18:31:41.061Z" }, + { url = "https://files.pythonhosted.org/packages/bf/be/f75637e222085d29a3076edbc5cfd0dd9b77d145e310ff7dc1150f07558a/pydantic_core-2.50.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3aa9de446b793de2beb6fa2d9d0961803126c4e2a99c2f25ab59b9fd6ea125c0", size = 2052460, upload-time = "2026-10-11T18:31:42.321Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1e/3d192ffa39ce3bd0e98e988fb5c055cfbf4af6e6e155a4ca5baa86d1f2cc/pydantic_core-2.50.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:de531ce1e2a3364e8767878b58f4ff728a434b4fde089781fe30b1e08e2396e0", size = 2190750, upload-time = "2026-10-11T18:31:43.572Z" }, + { url = "https://files.pythonhosted.org/packages/b2/7e/12ea94df5220ab316ee5bb498ebcf710cff2db352653039fb6033f6997d1/pydantic_core-2.50.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a7c58106de36ac6a56314182958de20db8d3a29dfd5db527192cc754e4f8e7fb", size = 2390043, upload-time = "2026-10-11T18:31:44.745Z" }, + { url = "https://files.pythonhosted.org/packages/f6/bc/4fbb6dc90af038628ec824434a9841be9d46854f9267bb658f09e315024d/pydantic_core-2.50.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8b4c3df25bd323bf1d36a648d563cf1fc69d717451569927151bdad7cad07a77", size = 2432397, upload-time = "2026-10-11T18:31:46.186Z" }, + { url = "https://files.pythonhosted.org/packages/1b/1a/21a58791021397d9ca35295e3f4291c16e319155c5904334b89eb07e97e1/pydantic_core-2.50.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f77ac30b19221cd9bd3fcfa3d4614eff93140d0572ab730cded17b64adca05f3", size = 2160719, upload-time = "2026-10-11T18:31:47.557Z" }, + { url = "https://files.pythonhosted.org/packages/73/6a/d5b2c5876162fd2c444eaed0064b5d9d9d0ef0d31530fc6a7997e8685ad8/pydantic_core-2.50.1-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:d939de9c82e2126f7f48a7e658f8a85ed46d57662d53f44c49b8895fe94a3eb7", size = 2256164, upload-time = "2026-10-11T18:31:48.797Z" }, + { url = "https://files.pythonhosted.org/packages/af/e8/1a6625d6e99250388dd0ae817c5d75839cd6103e7b790f59e0ffff74955d/pydantic_core-2.50.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:30ddf019d082c117b5d309e5b86710c2a78909907ec1a9381feec3eec02eca0b", size = 2271176, upload-time = "2026-10-11T18:31:49.987Z" }, + { url = "https://files.pythonhosted.org/packages/19/9f/bbdd137d7ec41284701dab4d1489ea597d401dff2f5f3e6b41ea872e0b12/pydantic_core-2.50.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2ab756b72bd5054e4c7ef3ded331b35786cbd3cf931531a508f79a9537517064", size = 2304637, upload-time = "2026-10-11T18:31:51.251Z" }, + { url = "https://files.pythonhosted.org/packages/49/5d/7bd322e649b41b45130bc00bfa34dc57498c853aa8e6fc9604e33bc7087c/pydantic_core-2.50.1-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:b087b1c5be7ac687cf22eabfe4b6b608d40df23610651e93611e1f49118baf84", size = 2467374, upload-time = "2026-10-11T18:31:52.587Z" }, + { url = "https://files.pythonhosted.org/packages/53/48/41a91111590eb7ba68a7c64c03139235a4b8349930abcb87931aa80da796/pydantic_core-2.50.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a44101320cfe99432db74237545a63057dc7a88dfe792cbcad0647f2af56cb81", size = 2480832, upload-time = "2026-10-11T18:31:54.06Z" }, + { url = "https://files.pythonhosted.org/packages/2a/02/4ac93a3d3f0ff412df1a36a9619743336fe6eec68c0df8a914163a3d22b4/pydantic_core-2.50.1-cp310-cp310-win32.whl", hash = "sha256:a4aaaa791bdae1c972a7e81765f4f3571c926b8e0b9b6e47346499fb80079665", size = 2123154, upload-time = "2026-10-11T18:31:55.396Z" }, + { url = "https://files.pythonhosted.org/packages/6d/ba/3a7af3dcd480c4739aa82cd9fedee79a5531a7c553346abcf095bd05f0de/pydantic_core-2.50.1-cp310-cp310-win_amd64.whl", hash = "sha256:2eedf82ee4753cdab8e50044c6bd569577eebc3859b11fecf4eb9223761ff966", size = 2179366, upload-time = "2026-10-11T18:31:56.697Z" }, + { url = "https://files.pythonhosted.org/packages/45/1b/ca81811da3d2aea724fb3c4ba6b83f50ab8f210576306ccd601973142d35/pydantic_core-2.50.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:c531166c42ea7bdfecc8c50049581f05dd1993b09cc7c52bb36a14e96deaec7d", size = 2152145, upload-time = "2026-10-11T18:31:58.046Z" }, + { url = "https://files.pythonhosted.org/packages/59/20/938ef7e0541af0a314e2f9d63848dbd1656ba2ebfe5ca0469f7ebee00d5e/pydantic_core-2.50.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b6d0c2183008c188e19f4906d426b293bdc4f67ab17df8e180fe16cda208fa71", size = 2045761, upload-time = "2026-10-11T18:31:59.35Z" }, + { url = "https://files.pythonhosted.org/packages/2f/4b/456497fe2affa18f49b226e89bde31e8a204751ee343dd9beaac5438123e/pydantic_core-2.50.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94be440c03fede26969a5ce75468e0e6a9927a1b46d9b679ee8adc1b057b0350", size = 2049557, upload-time = "2026-10-11T18:32:01.121Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9d/cd5dfe13928f55e236dff3a1bee06721d318ca6c955d9ca993b0a5d09f39/pydantic_core-2.50.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:36c426eac0af8d1529ff8467e612b933346caec1fdc0d774f78f67a1a11e16c1", size = 2188153, upload-time = "2026-10-11T18:32:02.442Z" }, + { url = "https://files.pythonhosted.org/packages/fb/8d/d0b25bc484f8d9136ea76ece69e0b2d24ab370cc2e7bbd1b79e61723b763/pydantic_core-2.50.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:028e2f212273d4a39b1ec1e0de8166b1165a65fc0f1111452a9d94fc7c625c63", size = 2388308, upload-time = "2026-10-11T18:32:03.767Z" }, + { url = "https://files.pythonhosted.org/packages/c5/64/6513cac7c9f78492abafa01d57a69de1d246bf3085fd7b0330bf14b025b3/pydantic_core-2.50.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e6f0cc1bb9900dc558960894adeb30b0c083366fc1d69b856209fb2ca5c36fe5", size = 2431187, upload-time = "2026-10-11T18:32:05.159Z" }, + { url = "https://files.pythonhosted.org/packages/ed/60/7f2ae89c37e69a3dc0da9ff57b0388c285542c688d05f36767770429de85/pydantic_core-2.50.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8812592c85d0edf423f10eadcef42716d71e8219085ad9e85b775057b7306133", size = 2157623, upload-time = "2026-10-11T18:32:06.616Z" }, + { url = "https://files.pythonhosted.org/packages/87/02/e3a71373b85a7973099b9a229a4302c7792262717e0b4997e41c01f7dd16/pydantic_core-2.50.1-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:bbce99252ba3167b2b6277f1829d5bf4b43b754524bddf7f944707c3db7d2253", size = 2254539, upload-time = "2026-10-11T18:32:07.968Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d5/5b620aaf40dc631d911565b64d49d3fa8422027c046a7a96dc4e4921ae98/pydantic_core-2.50.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:476f6ed8e43cd1e0b460920e23571700872b284e77331cb30c4faf459cf48a4b", size = 2267344, upload-time = "2026-10-11T18:32:09.399Z" }, + { url = "https://files.pythonhosted.org/packages/e0/48/f46baf4580d8a56ca4782aa65842e16d3cf39736b29fcce8e9c0521b3422/pydantic_core-2.50.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2cbd1b75b09e976ed0d6b6ca297675632ca35df86130088457cdc60ef36970ae", size = 2300877, upload-time = "2026-10-11T18:32:10.903Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9d/e7d28505e6092c1fe075838a4f793ccbe6dd676e44ab16175815ee06cfe0/pydantic_core-2.50.1-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:5958c72adb417c39b12ac87525ac60b0d73315fcdc59e21f44ee4a5e2512c9ef", size = 2464270, upload-time = "2026-10-11T18:32:12.329Z" }, + { url = "https://files.pythonhosted.org/packages/22/b1/3bf63b82efd93ebc9968f9c12fc46037a28bcb452c88e4688f33e5ff7dca/pydantic_core-2.50.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d8f9e8a6c4ab04b78d61f78627370d834eb004b2869dcb28cfffa647b4ea1980", size = 2478653, upload-time = "2026-10-11T18:32:13.735Z" }, + { url = "https://files.pythonhosted.org/packages/28/17/26d1e4b9c2ba6a92b7dc75bbcb99ef44456b7e3af1bced868403667ea7d3/pydantic_core-2.50.1-cp311-cp311-win32.whl", hash = "sha256:4be846f55c9477f5f3ddde8f2ce941137e16862a56d018ed885d422bb6ae02f2", size = 2120246, upload-time = "2026-10-11T18:32:15.591Z" }, + { url = "https://files.pythonhosted.org/packages/24/65/3e875e7991eaecd4d70686293ff0d8a99aa8f93dc295e6a21c3d702912fb/pydantic_core-2.50.1-cp311-cp311-win_amd64.whl", hash = "sha256:0048b6dddc8ef4b64fccaad878bd143b0c3882ea9936279dc11d613f6b7dd1bc", size = 2176332, upload-time = "2026-10-11T18:32:17.153Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5c/58dbd17c61f209d91c70f549eb7a3bbec7023e89ea12dead091462d6d658/pydantic_core-2.50.1-cp311-cp311-win_arm64.whl", hash = "sha256:6a733778df2f7087ec1100ed0b41533e4f3001976e99570fa34f57c66e7f8e3e", size = 2182471, upload-time = "2026-10-11T18:32:18.49Z" }, + { url = "https://files.pythonhosted.org/packages/ee/94/101b49a6c63f99586755690696eae14532772587d9e39e93f7b8d28a49da/pydantic_core-2.50.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:704075d10b74f2f3c6e15407c696d88701df35fc8953f434a431add0d0074db0", size = 2222428, upload-time = "2026-10-11T18:32:20.228Z" }, + { url = "https://files.pythonhosted.org/packages/db/a3/f8b1b39349c33037c73d7a1b794c9c29d8f07d5bd2b74399c1375a529cc6/pydantic_core-2.50.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e8e1d6ce820aa23317e8209a86bd65a540973c12dc7552b48a4f6c8e9926815e", size = 2025779, upload-time = "2026-10-11T18:32:21.723Z" }, + { url = "https://files.pythonhosted.org/packages/d2/2d/862a7d115305cc621358e51ba13f296e0ffdf4fbd984be9f63bc7862730b/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c18db21573bd2c6489f9a544b7499f0df2853958c568e5e783536ee1f690af41", size = 2048429, upload-time = "2026-10-11T18:32:23.3Z" }, + { url = "https://files.pythonhosted.org/packages/c0/7d/fed95d18bf2b5abe2d5ab2bbec4184bfc22b6805994a33ccdae8edffd25a/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cb57f304525a5e3c13333b772bf9a473f36326e9c821b2e8e1b2fd36f80ae2c3", size = 2173274, upload-time = "2026-10-11T18:32:24.697Z" }, + { url = "https://files.pythonhosted.org/packages/93/1e/c6f22ceac65dc3d291ff79bea0eb74cbd1de8b128f01fee48af03bcea1e1/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a27c09d86600f1bf2fe3f37e1ae697faf3143931c09322cd799da94deee923b5", size = 2360403, upload-time = "2026-10-11T18:32:26.281Z" }, + { url = "https://files.pythonhosted.org/packages/2c/97/190b5b6bfe4c72f265feb657913eb30749193927a5ad7b83cfa7866a25a2/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:46b3301d3b5c886f77de7546e47274a5842c622ea2020b8c6524c6b66913b4a6", size = 2439799, upload-time = "2026-10-11T18:32:27.844Z" }, + { url = "https://files.pythonhosted.org/packages/90/24/48cd98388e4ac2c08d0800af36d15413db15e20387f1038123a2c86cb2f6/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93ba4e9d8210d941c200431a56b2c0400b131865947903937ed3ec5404307d2e", size = 2160157, upload-time = "2026-10-11T18:32:29.553Z" }, + { url = "https://files.pythonhosted.org/packages/00/86/d909133d0b9979ed14c20f110f21124a0e346caf071bd66ac5af3009062b/pydantic_core-2.50.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:e5faeaee74a57d32b3ab3aebad2e348f06d3ba946fc5d28c1728455f00a3d13a", size = 2229523, upload-time = "2026-10-11T18:32:31.027Z" }, + { url = "https://files.pythonhosted.org/packages/90/31/4b43f08131b2098e73a852fd932063385f3103b33839fd9d1abdb89e673d/pydantic_core-2.50.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a3cda0e538208e5d722bbf3698b24f19c0a7d05bc8d5f8a7f9b121ea7fa243d9", size = 2242700, upload-time = "2026-10-11T18:32:32.622Z" }, + { url = "https://files.pythonhosted.org/packages/88/77/de39658569dc7a4c293d110a89629f61145e7538f00d1e4280c28debefd3/pydantic_core-2.50.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:57f51b31ff826e2859120cf4737c5a758a48d96f3e97da40ccee1796d58078ff", size = 2278452, upload-time = "2026-10-11T18:32:34.165Z" }, + { url = "https://files.pythonhosted.org/packages/46/bd/1eeb9e4e791e81210ef9096f58080b32ba15b5b13c108d50aaa3f8ce9f73/pydantic_core-2.50.1-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8daa7ee75245d43ad7d747e5c9ecc1b1d06552f72b14887e9276f787d57375f4", size = 2450098, upload-time = "2026-10-11T18:32:35.547Z" }, + { url = "https://files.pythonhosted.org/packages/1c/e6/b2accfdff17e5a88d6358536702146de00f9339adaa4d5bbe1ee93cb4832/pydantic_core-2.50.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:acbf31f37c53a5ac0c34706c80b4f5107ba20b05fdd3816124bf236ef0c57dd2", size = 2476839, upload-time = "2026-10-11T18:32:37.119Z" }, + { url = "https://files.pythonhosted.org/packages/3b/7e/c8a83eb5d2cd42c2e13507150fb777b1edea5a018adc9cbf2858a1d08285/pydantic_core-2.50.1-cp312-cp312-win32.whl", hash = "sha256:45b11cac094aa25725581d9304eee93c9028516b9ea80dd9e175e13a5a2c840e", size = 2101425, upload-time = "2026-10-11T18:32:38.593Z" }, + { url = "https://files.pythonhosted.org/packages/c5/17/8bbd530b8659e9d963f5b16f6cb8e159f4cd174b4d6304e48969ab12dba8/pydantic_core-2.50.1-cp312-cp312-win_amd64.whl", hash = "sha256:132529c83901437ff642f585216831bf5fd7a91df66829907e155192ead62498", size = 2172522, upload-time = "2026-10-11T18:32:40.112Z" }, + { url = "https://files.pythonhosted.org/packages/af/9f/a79d690b127e4e3cff143d031538ab9aa715f6b2fbeecbd07cd6f2bf9fb7/pydantic_core-2.50.1-cp312-cp312-win_arm64.whl", hash = "sha256:4e834f6a8e4ff772dcc34f58ef5504147a3ea5b0f4eeb13b0f8eb2ca75ac57f1", size = 2161366, upload-time = "2026-10-11T18:32:41.606Z" }, + { url = "https://files.pythonhosted.org/packages/ae/57/0e237d7091d2cd44a35d243b7344227f90440166860469cd036afc23a04d/pydantic_core-2.50.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:d5e062c01286d861fd6a1c4ff6e063547b3e713067f2df033c0ff97ac2ca006b", size = 2222774, upload-time = "2026-10-11T18:32:43.103Z" }, + { url = "https://files.pythonhosted.org/packages/f7/b9/c720e56858d4e1539503297ed37063e0c08e0f3541c41b777f6a800f75fa/pydantic_core-2.50.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0c003c3b7f49debb893d2d85ae099ac5959c9839e2f330fadb1fcdf7a6594482", size = 2024738, upload-time = "2026-10-11T18:32:44.587Z" }, + { url = "https://files.pythonhosted.org/packages/4d/b8/fbfc25875219cc060e613170ff10e850c6d8924beb4910da57c2ee3ba1d2/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:409e0ea40ec30d9158f33574fd758e689f6045a0f2596701828c27816ca9687d", size = 2047049, upload-time = "2026-10-11T18:32:46.164Z" }, + { url = "https://files.pythonhosted.org/packages/d7/43/34210124d504c553688f2f04b48500b131237528ac545b445e0d6d30e0d5/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:131059670f1d2444269b8585cb888963994871932447c08b39ac6a51fcfef658", size = 2173415, upload-time = "2026-10-11T18:32:47.722Z" }, + { url = "https://files.pythonhosted.org/packages/65/cf/6e178e8fdc11da5965bef980983bf46326a42f436871dd51ba0a57f39df1/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6dbcbee53bf17196a7f745aa9bf5a9603953a1e365b1f020be3207c676a3e7c4", size = 2361719, upload-time = "2026-10-11T18:32:49.217Z" }, + { url = "https://files.pythonhosted.org/packages/11/14/bd5169d356aa91bf777e28ba0b281c192d286d03c2812c9c9db023a2f00e/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:325c23f3e35cfbf0fe3486fa5f7260d1e45885173002d30a28ca019994124255", size = 2439803, upload-time = "2026-10-11T18:32:51.025Z" }, + { url = "https://files.pythonhosted.org/packages/1c/bc/d79d000e5203ebef39af839f2ce77a777fcad6e08ad26af9a2fcd114ffc8/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17e722e156d0444ecaefbe640bdb60928752bf2013e2b7a11cdb099aaae19bec", size = 2158402, upload-time = "2026-10-11T18:32:52.714Z" }, + { url = "https://files.pythonhosted.org/packages/1b/a5/4902cb5fd599422c130bd3124ab31ee8b771199bd56662702eed07d3fec7/pydantic_core-2.50.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:aa8224f10880d9bf1b5993988ba153d42a8b4f3f4f511f93b1f09c93ff613c72", size = 2229755, upload-time = "2026-10-11T18:32:54.126Z" }, + { url = "https://files.pythonhosted.org/packages/1a/f5/c1481f8669f6060d89110c9b1374173fc8767ca276b84f4870bd8acd5e3f/pydantic_core-2.50.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:41bc8237121bd8dc8d888dfd6279fc166ffc88c1f1bf3a8bf00869680533ca4c", size = 2242303, upload-time = "2026-10-11T18:32:55.641Z" }, + { url = "https://files.pythonhosted.org/packages/04/f9/77fc3c7653ba9b6e42049e17e96b25388187d4a7c274ab1cb07f58ac8419/pydantic_core-2.50.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:45c6266d071c241f2a168d45bf8c54344f0effce35e7e6b73afdec11f3687568", size = 2278345, upload-time = "2026-10-11T18:32:57.38Z" }, + { url = "https://files.pythonhosted.org/packages/95/9b/0579c5d12e7f2b16b27e6782427987341fcce07b0725e02ddb0b74add0c4/pydantic_core-2.50.1-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:1deeacb112d14d3f4fcb16b165f7dbaf76c70ba6e82f37ba042bdab51970a0b8", size = 2449947, upload-time = "2026-10-11T18:32:58.896Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ac/1b677db91eba54cc5922f4de6edc46ba45c2bd712dfdc0130382d158e214/pydantic_core-2.50.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:1c96fd793b73d1b92e65570132505498fe7b21eaef73cdf74e67e5dfba7ac9e4", size = 2476889, upload-time = "2026-10-11T18:33:00.665Z" }, + { url = "https://files.pythonhosted.org/packages/4d/2a/3a9f6624ee3ea9ccba5249dde11418bb4c35780a7a92608f8768bd3fea39/pydantic_core-2.50.1-cp313-cp313-win32.whl", hash = "sha256:06ead20d39ffd6f2f6f2a8f8a6de67ff8bb1b4f14a8a30e058502514ee2ac685", size = 2101118, upload-time = "2026-10-11T18:33:02.329Z" }, + { url = "https://files.pythonhosted.org/packages/2d/1f/323f78ddd9d9938aac420c1abb4e8ba799fc8bdab0acc67c0593b837c979/pydantic_core-2.50.1-cp313-cp313-win_amd64.whl", hash = "sha256:7816e98acc08119dc0f340ab167048ecc54126316330c1f0caf7c6756c88e28f", size = 2170840, upload-time = "2026-10-11T18:33:03.919Z" }, + { url = "https://files.pythonhosted.org/packages/cb/09/8497c52a739ae425c3ac2f7f56414cbc711c67d374346174f40fe2062644/pydantic_core-2.50.1-cp313-cp313-win_arm64.whl", hash = "sha256:c17799a62c142d61b8a3c51752a7cbc87fe2ad4ccfab10e628a77b405075c662", size = 2161633, upload-time = "2026-10-11T18:33:05.518Z" }, + { url = "https://files.pythonhosted.org/packages/49/33/28b96e81677153715e3eafb9f26663a80841e859fde282a380359b0d3fa1/pydantic_core-2.50.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:1cf41f1ae3fa155cf167a72689ad044bcc1e3c97e064123677149bdfb5dafc4a", size = 2224428, upload-time = "2026-10-11T18:33:07.153Z" }, + { url = "https://files.pythonhosted.org/packages/94/40/15c06410c9b7b8da5805d27b64e09bd3f900e986728106db2689dbc51513/pydantic_core-2.50.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4df197990c15b5a37c5a277d131d9f2c67de6133f2e5dafd80d9bba4b99f46f9", size = 2023740, upload-time = "2026-10-11T18:33:08.763Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4e/5eb629f6efc2a27e421d789dd4bfacbb5f6d09c80c28b13d1e87d73163d5/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0036473f5583e6a60e50b8b21651511564277a3f05cc5dab8cf579f552cd5f6c", size = 2049435, upload-time = "2026-10-11T18:33:10.366Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8e/f195aebec49ad12318876ac2368c197a7939f5d52f8e156d2238ef8a4588/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:992c3514ec891fa7858099183e4d64e6bd5a5d4ff452fae29df22faa77a006bb", size = 2172975, upload-time = "2026-10-11T18:33:12.368Z" }, + { url = "https://files.pythonhosted.org/packages/e0/f5/7ad9fb83010cd5ea0948409db105676ed779c4e709e2d3d89b9fe2558794/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:739dc730e6be3bd5ec2f4ab5cfc7eb047cc45fc1497b3bafec74ff2ed07df597", size = 2364197, upload-time = "2026-10-11T18:33:14.244Z" }, + { url = "https://files.pythonhosted.org/packages/ca/fb/bf0aab3e78301d202b82a0322ec968cd703b11fc0623fde9a28708936f62/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:32fad3a91e51b6d2039c572db04a5a873260b399f6bd62c3552671fa7a4a2899", size = 2444259, upload-time = "2026-10-11T18:33:15.79Z" }, + { url = "https://files.pythonhosted.org/packages/45/35/38f6d6564fae57d9b12e5676dfa947e0e7d5c46dbeaa7eb3352261d6d299/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42b54c2c90ad348b5e3a85e03e715d572c1fde357ef104cdfe3b03b697a404ea", size = 2159857, upload-time = "2026-10-11T18:33:17.51Z" }, + { url = "https://files.pythonhosted.org/packages/65/a0/fb0a3ca10f139dcf765b2d312d10cf0f65c57c59993229d00ad12b14ceff/pydantic_core-2.50.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:2df1ff41884de2bc4b307bafd7c40a691094fad2ff8e767e5b45a319257bcf4e", size = 2233954, upload-time = "2026-10-11T18:33:19.591Z" }, + { url = "https://files.pythonhosted.org/packages/8c/6a/e63842252702aa4ec6e6b7178ca85e541a77459076592c23ebe2d9840b33/pydantic_core-2.50.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe90228920fd8ff2be62622b6bb8a2b11acd65046d50c6b130614b5879605a20", size = 2243876, upload-time = "2026-10-11T18:33:21.393Z" }, + { url = "https://files.pythonhosted.org/packages/39/25/5991cf8318b37e0dfab47b87541619a1df8501a793cba0d978846cba37a7/pydantic_core-2.50.1-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:844b869f118e22a41a091bdcedda8a71bc1b0f62c38d1a0c3211cece47e1d8fc", size = 2277908, upload-time = "2026-10-11T18:33:23.324Z" }, + { url = "https://files.pythonhosted.org/packages/a7/3e/3ee8baaa6cc25a6961c69168cf9ff0f002d56f4e0d541ad6724c18fb61f3/pydantic_core-2.50.1-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:2eb75304506894a281d346220a4f7481a1b8729577c5ed2a05395991966a8396", size = 2449285, upload-time = "2026-10-11T18:33:24.942Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c7/acbec6deac13fe697a80c275a9b6661db62c4d323bac8a47347b1f39c7cd/pydantic_core-2.50.1-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:6b20a4bffabdad0db2927ac034ae3b8a681b1f7a0182f3e60b479ad2fde21ebb", size = 2479013, upload-time = "2026-10-11T18:33:26.925Z" }, + { url = "https://files.pythonhosted.org/packages/10/08/21a3f237b264389f6219d053ee78fc1b5c2fd402c1b1cb2b6cb8b85f9834/pydantic_core-2.50.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:99ba9bc2b8062ea0c326a990f7f00e6530c23579de66dd246e72c4cafef950a5", size = 1484400, upload-time = "2026-10-11T18:33:28.693Z" }, + { url = "https://files.pythonhosted.org/packages/09/ce/077a6d262d12ef09108377ac0717f029f420d773ace63cafd6143af75568/pydantic_core-2.50.1-cp314-cp314-win32.whl", hash = "sha256:cf356f70551d40374eaffb1aa63f1eb6d2006681cbd7a9faea173ce0f4dd7cd2", size = 2103222, upload-time = "2026-10-11T18:33:30.326Z" }, + { url = "https://files.pythonhosted.org/packages/14/4c/350a2415209c43d670eb71d3c040f332c04a30583d39e311b05c7ac15762/pydantic_core-2.50.1-cp314-cp314-win_amd64.whl", hash = "sha256:d32f3acc081cc3923386d88f422cde8892335e95f034e0104bb4cf9310d9915f", size = 2171586, upload-time = "2026-10-11T18:33:32.139Z" }, + { url = "https://files.pythonhosted.org/packages/bf/92/9bea6ca96580a0902fed366f064f0829e404f41889b34543279a1162b888/pydantic_core-2.50.1-cp314-cp314-win_arm64.whl", hash = "sha256:bed5163e03b98bc1fa2eb05d74c63d9c5c95d8ed6254985481640fbf5e237dea", size = 2155773, upload-time = "2026-10-11T18:33:33.896Z" }, + { url = "https://files.pythonhosted.org/packages/86/8c/f121f073cf32bdd5cba7e6230b1ce0ac845b0cf43e44dd597d95af272db2/pydantic_core-2.50.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:9572c1369e9c9da2d64a7b7992c786d90ff295abc93964cfe3125e4290768070", size = 2234017, upload-time = "2026-10-11T18:33:35.588Z" }, + { url = "https://files.pythonhosted.org/packages/a4/69/2bb2bcd6146cffe98d760a46c42ae71efd5151d9b2f9c9bf6619a3b32083/pydantic_core-2.50.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2005207aafe1231315718bf6ed5d064a7300fb4772754af35ee72fc68159492e", size = 2001016, upload-time = "2026-10-11T18:33:37.57Z" }, + { url = "https://files.pythonhosted.org/packages/20/b3/fbf854c7d07ec114260c26e9e2071a4381740f9ae09641dbfcbdf2a18c45/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64f6047f62a6c5ae08d0a6afb035667aa2d97c3d20d69762e034c5ea144d92a5", size = 2023015, upload-time = "2026-10-11T18:33:39.433Z" }, + { url = "https://files.pythonhosted.org/packages/65/20/6de55b2f92cdb614b745c6e9ced639fc4fd7e1e77604825a88bdece6fcbd/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1ef800dd7d85bcdadf4c3076e4c94e43939493558a3b69a1ea830c706d4617bb", size = 2192543, upload-time = "2026-10-11T18:33:41.381Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d9/19e91c94bd5c405945ce2f15526808aea37e162c253160de8ed7bf70b406/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b0135bcdcaa0f23573f286e4cb5e0fd2962700964ed13df085b85f2b97aeab9e", size = 2388704, upload-time = "2026-10-11T18:33:43.167Z" }, + { url = "https://files.pythonhosted.org/packages/b6/bc/2e24c8415eae1a25ee5a5484946a917123bad2e9d01689a759236928175a/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0b3a6f334c6a2345ca15318ff894502a90012536404b37c844a976c76c846e0b", size = 2464118, upload-time = "2026-10-11T18:33:44.856Z" }, + { url = "https://files.pythonhosted.org/packages/bd/1f/945b8053cb061c64e102bcaf7bfb9ed740c0bd4349f9bb978a7d4ddff4ab/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06e01fbbfdb9be777b316a71b6c49efaf4a08b615d0a98d678cda3023f79d019", size = 2150142, upload-time = "2026-10-11T18:33:46.661Z" }, + { url = "https://files.pythonhosted.org/packages/2b/78/96a3e50bf0d64aaae781107eb9335b7529d05a85793ed6e7241c4cd1d931/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:a29a061fec0b4e2d714f277e70a3a18125ecff803f2fea6eade2f2e53711d112", size = 2245577, upload-time = "2026-10-11T18:33:48.574Z" }, + { url = "https://files.pythonhosted.org/packages/7a/5d/a6038a0322232758a6ebfa709f4ca14a60bf5b93940cd0b345056a557da4/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f5187624823423e1d1b82b1072ac41dc837389e18d3d0572cc19bbee46cd550a", size = 2263728, upload-time = "2026-10-11T18:33:50.527Z" }, + { url = "https://files.pythonhosted.org/packages/0d/4b/76ded3333a457a9344c4f2d63f2d82d0101654b05178fa4ddfe7d3627674/pydantic_core-2.50.1-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:3e46a9eb0a0901dd6275e6b06ac3a464885ef350ec4121fe486869de8053e4bb", size = 2293630, upload-time = "2026-10-11T18:33:52.362Z" }, + { url = "https://files.pythonhosted.org/packages/a9/00/9eca378335c9c1b72bc779bf6e4c2a82ce784f14ec48a0e87102c9870e05/pydantic_core-2.50.1-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:756d669f04e62ec4148ecfe22be6a4484d9b1181a6ef32e205ebfd200540858b", size = 2468048, upload-time = "2026-10-11T18:33:54.118Z" }, + { url = "https://files.pythonhosted.org/packages/35/ca/e3832e9cf93651251de43c8c5c029ed680a3c1a0a1b17ee26c81bed08cbd/pydantic_core-2.50.1-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:c516cc5367ca3448995d42cb994bf3f4c9002d2a7c22eac9622551269ad1b807", size = 2512159, upload-time = "2026-10-11T18:33:55.99Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f2/773469b5a10a39116a2cb17edaf6d722f017dd8da07161b371465311c542/pydantic_core-2.50.1-cp314-cp314t-win32.whl", hash = "sha256:9d1bed94af6a63835461f3cf7502058eb166c58c4778e11d0f433cfb1bd69e19", size = 2098406, upload-time = "2026-10-11T18:33:58.043Z" }, + { url = "https://files.pythonhosted.org/packages/ee/42/0bb74f8f25204b259b11ab7c12dc7f180893b6bd706118b385147fb6efd5/pydantic_core-2.50.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c8dce1f1e0e5358b682a6ad3fa5e31b31d4560997b8e61417e9217c8d60f8a0c", size = 2148481, upload-time = "2026-10-11T18:33:59.913Z" }, + { url = "https://files.pythonhosted.org/packages/47/0d/d801646c9679a4e630e15cf521d4108b93854b42a6e6e02391bd4c6b1095/pydantic_core-2.50.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ceff0acc940be2715bd6ad17b24c0e5304abf44f6efd0f81ee8499e640f9dc86", size = 2174972, upload-time = "2026-10-11T18:34:01.875Z" }, + { url = "https://files.pythonhosted.org/packages/b2/84/23984b763d8862a02a13d27a44b6e8169428fd85ecfed88f54c29108604d/pydantic_core-2.50.1-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:8a6791afa2245e6c6b180122d105941644f5bd410bb18623b408808cc41a3102", size = 2224599, upload-time = "2026-10-11T18:34:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/4d/90/a63cf8586abc1d1a3f6d9b18f0224789ebf003851f092ebda3c7c863fd32/pydantic_core-2.50.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:84f34323a61a365b4e9295de6028474754829aaddd59c7bf1a040e7487ef8f3c", size = 2024191, upload-time = "2026-10-11T18:34:05.901Z" }, + { url = "https://files.pythonhosted.org/packages/a6/6a/f34bff9808ffb4907fbf5f5040457d253cacf2da7fce9efbc99ca1b1a44d/pydantic_core-2.50.1-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23edad659e8dbd8ca7e4e877fe6c81573abbdf215bd25a68b53e1272f58b80c7", size = 2048252, upload-time = "2026-10-11T18:34:07.757Z" }, + { url = "https://files.pythonhosted.org/packages/b2/54/13f419bf1eb59852003818e25935aaf175687d978f4c4bca70e08fe40a3b/pydantic_core-2.50.1-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a5fce22f1e87d181e924e12da7d81cfe031fb3881a5ddf26ad28f141756ca43", size = 2173376, upload-time = "2026-10-11T18:34:09.592Z" }, + { url = "https://files.pythonhosted.org/packages/6d/6c/b5a34d24cd0c81669d8f8339d74e6815abcf2f8fb48ab4b49b84c09be1d5/pydantic_core-2.50.1-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c73622ef819328873b53109ee4f77ceb598bffedd02daf916102be3228866b78", size = 2364559, upload-time = "2026-10-11T18:34:11.534Z" }, + { url = "https://files.pythonhosted.org/packages/eb/8d/d64d6216a8df365082665927ff923f183056f9049fee08e9777c9ac05296/pydantic_core-2.50.1-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ce8c25ca38cc0e3d7753ba180808de2c0c8cb24eae0df64491e40921454e9831", size = 2444697, upload-time = "2026-10-11T18:34:13.535Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0d/1b1149f60a00ea21ba5f70e28acbd40feb4598af414f80f53c921fac07c9/pydantic_core-2.50.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7689580e72a642ab5ec64d5f55b2e33636fa43b4ebe63c0c2c965ef307c7d1aa", size = 2159961, upload-time = "2026-10-11T18:34:15.56Z" }, + { url = "https://files.pythonhosted.org/packages/1e/35/f236549299dcc78e71e945495d7ec67e78d20844d0999c60601685003031/pydantic_core-2.50.1-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:d5c0e32fdbce7f1e8ef4d11f655694bf5f4175c757a9f1dc2be09b8864e5bcf5", size = 2234581, upload-time = "2026-10-11T18:34:17.502Z" }, + { url = "https://files.pythonhosted.org/packages/6a/85/26901a490522b7f75ef9bb9a7afb73e5bb550f0e1cb5b99a0069183e2eb9/pydantic_core-2.50.1-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:40f523349960fa30f3ea51404308ff50f9997a90df639590f47a057c1f32b415", size = 2244090, upload-time = "2026-10-11T18:34:19.402Z" }, + { url = "https://files.pythonhosted.org/packages/ca/2c/481bcfc70ceeb77a778ca6e5b705fe592cb64735c108157f81b7dd9c280e/pydantic_core-2.50.1-cp315-cp315-musllinux_1_1_aarch64.whl", hash = "sha256:d4193206b6587047437f6f11d7e776df23e1c1e23af2a54d9347275614791e10", size = 2277948, upload-time = "2026-10-11T18:34:21.317Z" }, + { url = "https://files.pythonhosted.org/packages/24/eb/f1e09333faa7ba447cde967310f758430cc7c5e987bdab5228816c70030d/pydantic_core-2.50.1-cp315-cp315-musllinux_1_1_armv7l.whl", hash = "sha256:84bc765b282a9d5b7fe0348b8648904f25a6a04b2139da52b1dd30c8ac3a2c8f", size = 2449688, upload-time = "2026-10-11T18:34:23.321Z" }, + { url = "https://files.pythonhosted.org/packages/d1/b3/036bde636db8f76d92996e81aefc75678ab5cec4a07eea1ad0c72a893fc3/pydantic_core-2.50.1-cp315-cp315-musllinux_1_1_x86_64.whl", hash = "sha256:ed1e728b39a383c81035b2459cfcb35d99dfb01f7d6ebe3a913bc1cc5b81e459", size = 2479286, upload-time = "2026-10-11T18:34:25.214Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a3/07f018294ee18d144afeb6df1a47d9e92960f014be45c5ed13519fa5af95/pydantic_core-2.50.1-cp315-cp315-win32.whl", hash = "sha256:bc94f474417604bd383d2cd445d071b07dd55fedceed3ce33407bf1fcc107290", size = 2103683, upload-time = "2026-10-11T18:34:27.42Z" }, + { url = "https://files.pythonhosted.org/packages/5f/98/f9bd7e1f9b6709f155acb9ef826d9c3884fe925f811fd8e55b9b52280bad/pydantic_core-2.50.1-cp315-cp315-win_amd64.whl", hash = "sha256:983a662de2571cb2502fc8ff47b6770b03d025d2eb314c92f77b3f07c74720ed", size = 2171681, upload-time = "2026-10-11T18:34:29.508Z" }, + { url = "https://files.pythonhosted.org/packages/10/87/4bb3e1e7f385c076ab5af4d6dd0571b22042cd8207eb810d5b9fef15ae31/pydantic_core-2.50.1-cp315-cp315-win_arm64.whl", hash = "sha256:94845ff54dc5193f228cab81b2662a04bfbb892e95bdc15edf7399000ce57d54", size = 2156251, upload-time = "2026-10-11T18:34:31.455Z" }, + { url = "https://files.pythonhosted.org/packages/47/47/83643225b08f2aef6c8cc4bbe6e3f79c5e139c4364a6e450d0f399d87774/pydantic_core-2.50.1-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:4a53d13cdfbedbfa87f08b83c1a0a5efcc767d785a4b41934fa9cb672670493a", size = 2234363, upload-time = "2026-10-11T18:34:33.65Z" }, + { url = "https://files.pythonhosted.org/packages/5e/66/127ca649ba2f2462039dc1e694c6c01e00a3689f023a617364d3507e6d97/pydantic_core-2.50.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:efbecf43d321f7b9281441f1f213f7c21c66988b0e06c2730ba13ed47a46bb08", size = 2000972, upload-time = "2026-10-11T18:34:36.037Z" }, + { url = "https://files.pythonhosted.org/packages/5e/4f/e421e0a5d653b1203b090b2e48745976988d7338a647940704b5b9c2b399/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc1f08f68dac9f9e83845a8039880aba2ab553eb9b2259c3243a313182c253fe", size = 2022925, upload-time = "2026-10-11T18:34:37.972Z" }, + { url = "https://files.pythonhosted.org/packages/d5/4e/ea5568e2491e1a71100f15ae8c2d01ef52db42a184a2d4e716bc79e5eb8f/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5dfe41f232befddb9c4377f6cfc702b51595e2d78ed082672adf8758d2c4619f", size = 2192867, upload-time = "2026-10-11T18:34:39.921Z" }, + { url = "https://files.pythonhosted.org/packages/ae/5e/3b8c3a35acbe219909ada5defad5d7d9fed845fb2b37bd5ec518f453c119/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:adc06d218a1cadfd2ec4628424d7d79ce4eba69c2965e7e7b55106f0da5208c8", size = 2388687, upload-time = "2026-10-11T18:34:42.184Z" }, + { url = "https://files.pythonhosted.org/packages/b6/aa/7889b4e515f91a2e8c0ae6b5081fec0feb30c4398d1434e14793c60f173a/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2cf91809d0721ab81592ba67bea7694821679c10b1a2e3c3460082b286c1918a", size = 2464415, upload-time = "2026-10-11T18:34:44.384Z" }, + { url = "https://files.pythonhosted.org/packages/08/78/93449e628eb8a6fdcce3eff9043081179d1bc7ce6f1bff32dc5006f41e00/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23923ab9292c40da026330b1ecf4dc2618c8e86e0422e5d1fbf50d94d64ca4f8", size = 2150823, upload-time = "2026-10-11T18:34:46.392Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f1/72c5bc129fceb0d00f05dc1e67f518c1728de1928c55f81fc13d7690de39/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:f3377c8c2b3ce898423c5e5dd94c7982e30aa7717a7e6ab2470b9de364963709", size = 2245806, upload-time = "2026-10-11T18:34:48.805Z" }, + { url = "https://files.pythonhosted.org/packages/28/2a/922a0e78f3aa6ab837b59f88190233fb8dad546c17995fde9bb3ed3b9b49/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:455a773617b5913bf5c20d0692e5787b119e52c4d40ea644ca31f5758fd31be2", size = 2264160, upload-time = "2026-10-11T18:34:50.876Z" }, + { url = "https://files.pythonhosted.org/packages/52/a8/0f1449e3e1b20941c9372faa18e7b6a092e30cdfa841902473f7989532ac/pydantic_core-2.50.1-cp315-cp315t-musllinux_1_1_aarch64.whl", hash = "sha256:1a9006395dece0e32e704c315eff8a00bede494f6108546cfc5539c89fef4f9a", size = 2294031, upload-time = "2026-10-11T18:34:52.876Z" }, + { url = "https://files.pythonhosted.org/packages/d1/d0/1031f492857de70355fb16524bbb03efce5fd34c93ed4a1ec60be07e0d4b/pydantic_core-2.50.1-cp315-cp315t-musllinux_1_1_armv7l.whl", hash = "sha256:d2d82aa62521c55ddfb000ae70f88cdd8de974078f6024e821dfe5addd0c818f", size = 2468423, upload-time = "2026-10-11T18:34:54.995Z" }, + { url = "https://files.pythonhosted.org/packages/46/52/269ffffa645b8e47906395a39cf9db1151ae9fa4bcd7f47b960bc31baf37/pydantic_core-2.50.1-cp315-cp315t-musllinux_1_1_x86_64.whl", hash = "sha256:009634b83993777ddcd69cad0ffcace43dabde692109528e35f0fde91e386a8b", size = 2512598, upload-time = "2026-10-11T18:34:57.149Z" }, + { url = "https://files.pythonhosted.org/packages/31/5c/e47e28281f20326ff6f3c31d626f0a83e615d2d94ba41bb0ad6ec237184a/pydantic_core-2.50.1-cp315-cp315t-win32.whl", hash = "sha256:3fde4fdc6487a58d944ca87cf5adc95d5f266e872c19599f5f4c0a8a1b1f9f9f", size = 2098535, upload-time = "2026-10-11T18:34:59.313Z" }, + { url = "https://files.pythonhosted.org/packages/03/ad/759e181e69c1b472c60b2049e5f61d5da1deefdd5ce1df4bb2ebcf771f25/pydantic_core-2.50.1-cp315-cp315t-win_amd64.whl", hash = "sha256:1c8632d4ac04e6f91128fca584b3a8a507d81604c24eeaaad00d4be42765c32b", size = 2148645, upload-time = "2026-10-11T18:35:01.571Z" }, + { url = "https://files.pythonhosted.org/packages/6d/56/8a702c27e5be9f47e5f19d8669227424290e4c024e7c370279cbaf244b4e/pydantic_core-2.50.1-cp315-cp315t-win_arm64.whl", hash = "sha256:c3ede305158e75510be50869b319550ab072008c13d64d4ab1e094fb286b6f44", size = 2175297, upload-time = "2026-10-11T18:35:04.079Z" }, + { url = "https://files.pythonhosted.org/packages/e7/73/a23a327237d9bb985298ce47702e196f995ce850224c12c70886bd66bb5c/pydantic_core-2.50.1-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:062e891facce5ca296a1c37098e5e466780457f86413894b399f0cf22934f769", size = 2162560, upload-time = "2026-10-11T18:35:06.274Z" }, + { url = "https://files.pythonhosted.org/packages/df/b2/33b37b5e82408f402a5e82649a43e7c95cd5dab7beaeef2c35cedbfd18f2/pydantic_core-2.50.1-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:49c2cbb2397fe4d0987e84606e691af6cb87bc0ee1bd3e7b737f7e10b4c142f9", size = 2075250, upload-time = "2026-10-11T18:35:08.455Z" }, + { url = "https://files.pythonhosted.org/packages/04/62/c7d5b9249466ccab90f3ce4790c2399dbfb8172ae9e74a7753dd295fe65a/pydantic_core-2.50.1-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e4472072de0137ee0d8e72d6620e85939c271d2f90f6bbb4b15c24638b79f92", size = 2107891, upload-time = "2026-10-11T18:35:10.918Z" }, + { url = "https://files.pythonhosted.org/packages/d4/7e/28d29bc1657195933147f772dee0558d9e0f045bbc11db7f992f40c37bb5/pydantic_core-2.50.1-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ed4f3cef55164b026fefb41341b7754cc6b624c75dfe7142d2ecceb5ad21c87", size = 2268026, upload-time = "2026-10-11T18:35:13.336Z" }, + { url = "https://files.pythonhosted.org/packages/a5/09/1bcf160f3cd6333e2a76982fae5963e5032de0bcaaba38464223dc266bfe/pydantic_core-2.50.1-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:76e2e83fa6ec8cdc972d438dafc2522b3a47bee4ec0ae668b29cfb1977ab5242", size = 2217070, upload-time = "2026-10-11T18:35:15.688Z" }, + { url = "https://files.pythonhosted.org/packages/3d/81/f02de95eec7794dfc7a5bc69a834f0e834b31b3009014ce6de903855f775/pydantic_core-2.50.1-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:a51eee75939cf811ac09b278745a6cee7dc873ccfbc8b9af3cc88fe4b7ce25b5", size = 2039027, upload-time = "2026-10-11T18:35:18.07Z" }, + { url = "https://files.pythonhosted.org/packages/4d/16/227746ed3771d9bf2304568b55134849ece34e13b38b1d76de763015c77f/pydantic_core-2.50.1-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae28183297fb0d2b8dc46a1f01d51f5e45825fc5afe76a835a6cb7fb34821295", size = 2109009, upload-time = "2026-10-11T18:35:20.218Z" }, + { url = "https://files.pythonhosted.org/packages/e5/b9/7664d1592a0e5a74903f357cd5b28ceb7336f3473256c5a4e8c8432f1881/pydantic_core-2.50.1-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88e492e8b9d0312e7dc13667c30222abf284dc3b79b5302b3607b41a5784ce61", size = 2269883, upload-time = "2026-10-11T18:35:22.48Z" }, + { url = "https://files.pythonhosted.org/packages/bd/35/002a378302ff91d2a7f149bc8d22363faf581ba40ac39d0123444315a32d/pydantic_core-2.50.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7456d699b13954e9c0164dcb267250a10ae0dfb03e6e26d6796ab0d46e189c84", size = 2154076, upload-time = "2026-10-11T18:35:24.743Z" }, + { url = "https://files.pythonhosted.org/packages/4b/5c/7a034330082c00f3c9795052bd9b161aaaf6e1951e897ac00a094823ef7e/pydantic_core-2.50.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b0d955195bbbe489ad343fcc956eacea9357b79cb22192c66cacdefcbc14b32f", size = 2085850, upload-time = "2026-10-11T18:35:27.009Z" }, + { url = "https://files.pythonhosted.org/packages/57/fa/d5b44a7d3d4eceed6ef8713a0003af56fefde6ec75915513bcdd5d730cd9/pydantic_core-2.50.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2c634642694e6a0dad2ab1d375589fa671fd442edd5caf7d9737b8f6ca22906", size = 2259754, upload-time = "2026-10-11T18:35:29.763Z" }, + { url = "https://files.pythonhosted.org/packages/b6/88/e01dd37301710bd8b3b2daf34ec9e2a6e83a333130831c0d468b0292a48c/pydantic_core-2.50.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ee6db2fbed51a7991302e8fac498cd67e336246026d0dfa84cf5166ce1412760", size = 2268712, upload-time = "2026-10-11T18:35:32.654Z" }, + { url = "https://files.pythonhosted.org/packages/3e/43/5bb80d4a4d2b2d84612a30205662cc521a20d209c94abc079ea48b4e0df2/pydantic_core-2.50.1-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:79490e33c4c0fcb933bbbcfc3a62184d8803b99f535863dfbb925e1bcb6945ad", size = 2305049, upload-time = "2026-10-11T18:35:35.392Z" }, + { url = "https://files.pythonhosted.org/packages/e8/7b/e3ef76b269cfe4a5bd3bdcf5d7124cf05e5a8909d443b91639307116bdae/pydantic_core-2.50.1-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:48569b0ade9edfbe065cad1d700175546592aebbb42f02adcebcc26e75b896fe", size = 2466001, upload-time = "2026-10-11T18:35:37.791Z" }, + { url = "https://files.pythonhosted.org/packages/8f/7a/ea0947aadc1d9f72d4be00d2e19939611ec17363bcb5e6eb9ec7d8221c34/pydantic_core-2.50.1-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:5f3cae32fc46121f787cb2486de9cf95a8bf72aec5cc78f64c606fa1735a6ef5", size = 2481777, upload-time = "2026-10-11T18:35:40.287Z" }, + { url = "https://files.pythonhosted.org/packages/24/f3/d15bc0b1fb0c4f1e07b3326ffb7489f7ae70da2b56a0884d22641b0eb47c/pydantic_core-2.50.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7f476456ac2bb0d937f75191494a09c83a30765fea4f70f3b404942fe25f6cdf", size = 2336787, upload-time = "2026-10-11T18:35:42.558Z" }, +] + [[package]] name = "pydantic-settings" version = "2.14.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "python-dotenv" }, - { name = "typing-inspection" }, + { name = "typing-inspection", version = "0.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-inspection", version = "0.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5c/b5/8f48e906c3e0205276e8bd8cb7512217a87b2685304d64be27cad5b3019f/pydantic_settings-2.14.2.tar.gz", hash = "sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f", size = 237700, upload-time = "2026-06-19T13:44:56.324Z" } wheels = [ @@ -8102,7 +8689,8 @@ name = "pyee" version = "13.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" } wheels = [ @@ -8117,7 +8705,8 @@ dependencies = [ { name = "pyjwt", extra = ["crypto"] }, { name = "pynacl" }, { name = "requests" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c1/74/e560bdeffea72ecb26cff27f0fad548bbff5ecc51d6a155311ea7f9e4c4c/pygithub-2.8.1.tar.gz", hash = "sha256:341b7c78521cb07324ff670afd1baa2bf5c286f8d9fd302c1798ba594a5400c9", size = 2246994, upload-time = "2025-09-02T17:41:54.674Z" } @@ -8148,7 +8737,7 @@ name = "pyjwt" version = "2.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/02/a5/5197bfd06417837ac079921c66fa6393f1dea3557272a263cebfef69e432/pyjwt-2.15.0.tar.gz", hash = "sha256:b11c5f9791d7bf51c2b39a81ed669f6b2dbbd669df2942f6c60167e9e3d1abe4", size = 120513, upload-time = "2026-09-23T16:56:00.689Z" } wheels = [ @@ -8228,7 +8817,7 @@ name = "pypdf" version = "6.21.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/28/081f1cbcca7042441dafc8de00f48ba60853f89e5bff92c5f26477346d81/pypdf-6.21.2.tar.gz", hash = "sha256:0a3e1c5c886a1be03646c16a04827cc166c2a0575c971b26a4bf89686551c2d4", size = 7144571, upload-time = "2026-10-21T16:44:55.272Z" } wheels = [ @@ -8283,7 +8872,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backports-asyncio-runner", marker = "python_full_version < '3.11'" }, { name = "pytest" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/90/2c/8af215c0f776415f3590cac4f9086ccefd6fd463befeae41cd4d3f193e5a/pytest_asyncio-1.3.0.tar.gz", hash = "sha256:d7f52f36d231b80ee124cd216ffb19369aa168fc10095013c6b014a34d3ee9e5", size = 50087, upload-time = "2025-11-10T16:07:47.256Z" } wheels = [ @@ -8699,7 +9288,8 @@ dependencies = [ { name = "ml-dtypes" }, { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "python-ulid" }, { name = "pyyaml" }, { name = "redis" }, @@ -8717,8 +9307,9 @@ version = "0.36.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, - { name = "rpds-py" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "rpds-py", version = "0.30.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "rpds-py", version = "2026.9.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2f/db/98b5c277be99dd18bfd91dd04e1b759cad18d1a338188c936e92f921c7e2/referencing-0.36.2.tar.gz", hash = "sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa", size = 74744, upload-time = "2025-01-25T08:48:16.138Z" } wheels = [ @@ -8905,7 +9496,8 @@ version = "2.23.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "requests" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/96/a3/20003e7d14604fef778bd30c69604df3560a657a95a5c29a9688610759b6/resend-2.23.0.tar.gz", hash = "sha256:df613827dcc40eb1c9de2e5ff600cd4081b89b206537dec8067af1a5016d23c7", size = 31416, upload-time = "2026-02-23T19:01:57.603Z" } wheels = [ @@ -8966,7 +9558,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown-it-py" }, { name = "pygments" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ab/3a/0316b28d0761c6734d6bc14e770d85506c986c85ffb239e688eeaab2c2bc/rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098", size = 223149, upload-time = "2024-11-01T16:43:57.873Z" } wheels = [ @@ -8986,6 +9578,9 @@ wheels = [ name = "rpds-py" version = "0.30.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/06/0c/0c411a0ec64ccb6d104dcabe0e713e05e153a9a2c3c2bd2b32ce412166fe/rpds_py-0.30.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:679ae98e00c0e8d68a7fda324e16b90fd5260945b45d3b824c892cec9eea3288", size = 370490, upload-time = "2025-11-30T20:21:33.256Z" }, @@ -9104,6 +9699,148 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/b7/b95708304cd49b7b6f82fdd039f1748b66ec2b21d6a45180910802f1abf1/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ac37f9f516c51e5753f27dfdef11a88330f04de2d564be3991384b2f3535d02e", size = 562191, upload-time = "2025-11-30T20:24:36.853Z" }, ] +[[package]] +name = "rpds-py" +version = "2026.9.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/42/68/3bd46b8a5e01d3c2ebdf9c5e9497912e3fe0cde02bac21a7130ca866e403/rpds_py-2026.9.1.tar.gz", hash = "sha256:4793ef7f78268b124b73fa933440f01d258bbae01de9fa53e9080c9ab0425a12", size = 63948, upload-time = "2026-10-04T16:32:36.469Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/6d/ddb8dfcf41892ba9c672d27988592c038d30db61708230313831b6780a1b/rpds_py-2026.9.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:2711d29b653b3bce48a63d18b9c6b53274669e6d6c4094dddeb4d9a0e45128b2", size = 344947, upload-time = "2026-10-04T16:28:55.207Z" }, + { url = "https://files.pythonhosted.org/packages/9e/b2/632c3d034961149eff606e3e357547620a9ab37b849ae5815f67838753fd/rpds_py-2026.9.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3231c4c0e521dafa5be0c9f114ee2c2ad46650836f2d72caa86801950c3e7044", size = 346770, upload-time = "2026-10-04T16:28:56.884Z" }, + { url = "https://files.pythonhosted.org/packages/6d/45/4df346410bb873d03ec60c3b891d1fceb8cabde1971d29844bbb625cc296/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e01b3c878c8641913e688edd1b3f08658c6783d29cf6b826bd3c0d1ae7a1ffaa", size = 377795, upload-time = "2026-10-04T16:28:58.193Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b8/a70f302207e4f29f873c01a986fcac5def49b9c8224328cef1ad7da6b7a9/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3e524c7874ac72884d28e16dd5b8d839fd09e0fe76b020d3fbca23212a7b8c52", size = 383137, upload-time = "2026-10-04T16:28:59.411Z" }, + { url = "https://files.pythonhosted.org/packages/13/e3/3aa28a48a01169d0e20728bf0e5ebbea3ae2774285b40853ae49b3ef332a/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:761fdae6728ceb99ab182fad2f0cc1e262f610834dc891aea1d1a2a2e634776f", size = 496789, upload-time = "2026-10-04T16:29:01.011Z" }, + { url = "https://files.pythonhosted.org/packages/3e/a7/414e8d088602d66ad6f47c65bf07fe5090992a8574cdd4571c1a83b647ed/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6b723eb406dec5bc9ec516c73ab9c3239a3284e017f7eb89ee2b3258bd504fb7", size = 395929, upload-time = "2026-10-04T16:29:02.465Z" }, + { url = "https://files.pythonhosted.org/packages/2d/5c/aac3b960fd55d37b49f77c0eebbe043eaf36762ebd48e7b46951e59c1c05/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:136a1c3fe4402b7008bc81cb62ee538481795b61a7e83df88dff3b3f02b726ff", size = 375426, upload-time = "2026-10-04T16:29:03.944Z" }, + { url = "https://files.pythonhosted.org/packages/d3/80/77987a41c40be7202a78d9aed4a650bfb05792237cfd3b9e22638d990fe5/rpds_py-2026.9.1-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:839dde845559254f34885267c6878f60d61d5205180226d976fe488d45fa128e", size = 381497, upload-time = "2026-10-04T16:29:05.308Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d0/640b02eb1ecd71a913c1178b6235287b31006266bca9a26862e80e497b54/rpds_py-2026.9.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:56cd8b3f77d7b6812f533b662186a1f28316931166ddc00fb893b1b0db7e9888", size = 405740, upload-time = "2026-10-04T16:29:06.578Z" }, + { url = "https://files.pythonhosted.org/packages/3e/8c/d034195d45d215625a23f2ae85dbbdb86a49a93a51272061c97c80b9d316/rpds_py-2026.9.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:074a4d198bc34d9a8ea425114fc3ded6d11ec01f6a314a8db67454a5152d8834", size = 555850, upload-time = "2026-10-04T16:29:08.115Z" }, + { url = "https://files.pythonhosted.org/packages/de/f8/532f1addfc86207bc3cf23749638bb0d457bce891595012ed8a08d0543b8/rpds_py-2026.9.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6beb738155fe8ab8091afdfa5a3226b21c2b1593f1e50ebb90eb25b44dbc0391", size = 618661, upload-time = "2026-10-04T16:29:09.554Z" }, + { url = "https://files.pythonhosted.org/packages/50/5b/b0f2c9833a36a5b1f6ac5cb0e635fabacf713266f46ceefba2102da0bf51/rpds_py-2026.9.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:57492a550a1d88d29d003247e5f78dd8cf04a701fac0e4c8db8745a6d2504e0a", size = 583530, upload-time = "2026-10-04T16:29:11.072Z" }, + { url = "https://files.pythonhosted.org/packages/bf/20/464ed4b98d3c8c6a334f3b5e24d3886de04b05658a95fc40a37e0e6e4775/rpds_py-2026.9.1-cp311-cp311-win32.whl", hash = "sha256:d95a354e02393eada6d7351184671aced9d4cce109dabf927cb7aa99624352a1", size = 208125, upload-time = "2026-10-04T16:29:12.408Z" }, + { url = "https://files.pythonhosted.org/packages/df/c9/384a9b62f8cd89c8e3e609b8c9052c140e8c01d807261e0517c904b8bf29/rpds_py-2026.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:ce4d4f52e2a4324396caddbd45a97d8d7be5f42edd25d2355282a9c34f9b2f7f", size = 226045, upload-time = "2026-10-04T16:29:13.751Z" }, + { url = "https://files.pythonhosted.org/packages/4f/3b/0a691f0dcb2301fa9c736346da2d0d4991778373283301c5c12b8f952f3d/rpds_py-2026.9.1-cp311-cp311-win_arm64.whl", hash = "sha256:fdcd198979b4ecffcc1beba366a7fbcf4eb41243691a82fe52ceb0b902f09c12", size = 222636, upload-time = "2026-10-04T16:29:14.989Z" }, + { url = "https://files.pythonhosted.org/packages/5d/34/a828586ea3329fbb50895b50e9cf98ca3d924a9f41a0b26d443bff1b3794/rpds_py-2026.9.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:50906f5aea24b5a865cbd0a589698288631d9f3a54c3a937c83aefa95a0d14af", size = 346822, upload-time = "2026-10-04T16:29:16.258Z" }, + { url = "https://files.pythonhosted.org/packages/90/81/ac6a0d064982251856ce009c9e1dd51a34110b3c055aa7d1aad18b4899a3/rpds_py-2026.9.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e21c1429e205828ea886a2293a4a2c8e01f4c25d9893ca330e97a6cf73f52e7b", size = 341380, upload-time = "2026-10-04T16:29:17.706Z" }, + { url = "https://files.pythonhosted.org/packages/42/ff/bf7d54f362748fd6a49277b9d6531c394074110fedf50ad791ec59133fbb/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2693b2728bbcc48d09a981a356954b0c47c53ff25b545856f28a889ea619f69a", size = 373053, upload-time = "2026-10-04T16:29:19.063Z" }, + { url = "https://files.pythonhosted.org/packages/60/de/74b0ccdbbd28687b9b5fdb34c1cabed88352182facaced9d6f5486b8b9ed/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8601470267d938bcb7f3ab1a336100af51a4fd5b6ed030ef52461bb3ef5e7e07", size = 377742, upload-time = "2026-10-04T16:29:20.543Z" }, + { url = "https://files.pythonhosted.org/packages/c2/6d/b979775a3057b2a5c26d75ecbff60a20a24ef081db6dd76836fca3f20247/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3890a6aa36e6baa53d5258a2a25d3ef8b37ad165a6ab27a892d7c3e3a432cd69", size = 487289, upload-time = "2026-10-04T16:29:21.958Z" }, + { url = "https://files.pythonhosted.org/packages/1d/5d/7c34734ce3ece943d9d6ee0122a74a21bbc75b47acb5b7053c83f6efb1ed/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6b5b393eda5ea42cca1c1a6665f2a4882b4fd5d1777e41ce0545a107fb008c9d", size = 390710, upload-time = "2026-10-04T16:29:23.378Z" }, + { url = "https://files.pythonhosted.org/packages/1d/6d/b26eb1e75395925b3a142ed351cbed2ec8212f5c9d937aee3df32c701baa/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:addeda51556dac7c1a2f14cda62db8b621cd12afba3091d03a96c72932387eab", size = 371261, upload-time = "2026-10-04T16:29:25.1Z" }, + { url = "https://files.pythonhosted.org/packages/7c/98/b2fdfe10301a9e27af21e634337fbba7baac0bc17fe44619a17c26bba2cb/rpds_py-2026.9.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:d9edf30457d74eebfd76b045535e36f1cd89062566a128a0db2145ca042d787e", size = 376075, upload-time = "2026-10-04T16:29:26.748Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b1/c4b8d954e49c3c69cf8063c0c9e0919c99f3d2cecc46311606a1cbe835dc/rpds_py-2026.9.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:815d26356930846a40c7bc1366e7b1b0320ab8a063e66c11298a208bed0fd237", size = 399358, upload-time = "2026-10-04T16:29:28.282Z" }, + { url = "https://files.pythonhosted.org/packages/b3/59/9559a7293c97dff0cbb293efffbd36644103883ecc27741af8ef90c84ca8/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b5a6f40f0a1486b4b36c888123afc67acdbd9f33235927acf5ff295429a0ba3", size = 550267, upload-time = "2026-10-04T16:29:29.851Z" }, + { url = "https://files.pythonhosted.org/packages/6c/9d/6dc60e49511de4c8b00e74f9c1d43aed4d27f712cdb1ade92f2c199cbf64/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b5b8b0753718d258fd454283fbd57e14545d3b40583fa672e27cb4f987626bcc", size = 614018, upload-time = "2026-10-04T16:29:31.54Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bc/92a4ecf27301b886232a413360f6b348a81d55ffef654f2ebda5970ebc3d/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:46d80bc76b51a6c24f9944368c28d38b8bcbcea1da4f2f8d3ebc31a67e8c6ec6", size = 578509, upload-time = "2026-10-04T16:29:33.01Z" }, + { url = "https://files.pythonhosted.org/packages/08/53/efb97f2e6589b7ab8394385a73870e0b2ab2a195281c117b21fd6d7d4855/rpds_py-2026.9.1-cp312-cp312-win32.whl", hash = "sha256:befc2d6a953e563f8a7bfd87a42c22ebf8a3e980dcb7b6a4d17b70b0e914e8a3", size = 205401, upload-time = "2026-10-04T16:29:34.451Z" }, + { url = "https://files.pythonhosted.org/packages/1c/84/1700cc748d0eaa747486e4e82882d2575224e6e9a3148df583058850c629/rpds_py-2026.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:5ce8943f79c2210f7abcc28e86367b03b28d95027fd01c46d2472373ae70c86f", size = 222923, upload-time = "2026-10-04T16:29:35.709Z" }, + { url = "https://files.pythonhosted.org/packages/c3/89/0302215373c2f8b4408b0fdfee955368cc378a98ff59508d47a7e0dc2dbf/rpds_py-2026.9.1-cp312-cp312-win_arm64.whl", hash = "sha256:501909f2e4a1e2dee528ef766fe3c469060ebc17e54a8383d404ba07a81a6f02", size = 218415, upload-time = "2026-10-04T16:29:37.263Z" }, + { url = "https://files.pythonhosted.org/packages/83/ea/ee88fd9e756ff93fb6b1182a47ec09504a242620e33ce1d20679efefe841/rpds_py-2026.9.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:a36b70596407634ca82d4b989a3729074a008537a0522e4c8046a67c729103e9", size = 346229, upload-time = "2026-10-04T16:29:38.82Z" }, + { url = "https://files.pythonhosted.org/packages/57/71/a097d6552f837500fc36e6b23d09cfb9890c3cc47531f9ca64e149799615/rpds_py-2026.9.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eba5d173f7d5708b22a93815017a4611873ed54db9f268077c0dd1ed99cfc858", size = 340514, upload-time = "2026-10-04T16:29:40.405Z" }, + { url = "https://files.pythonhosted.org/packages/bd/b7/497e85768bf4e0d8ddbaa096a4cac31d1509251dee2728a8490aa367e0b5/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:457866b85daf5034296666168b84a69e0b2e89dc4f1af102b46f6448a60b9063", size = 372611, upload-time = "2026-10-04T16:29:41.778Z" }, + { url = "https://files.pythonhosted.org/packages/52/4b/74ab4108916250b6e198e0d3af05bc6835eb046315f22f7a0ceb49667c5a/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a3a52a3ba86436ab3aef510fbe21512abc2ddd1993005dfe50514bd2284ef025", size = 377712, upload-time = "2026-10-04T16:29:43.242Z" }, + { url = "https://files.pythonhosted.org/packages/0c/8e/067e77d9d7b3cc793c9d909b7e97e7aadbbb1fb094093b6876902cc96d38/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d7841166b7fa64c9c56404617ae4341448847482d45933b13135d26c130519e5", size = 488106, upload-time = "2026-10-04T16:29:44.692Z" }, + { url = "https://files.pythonhosted.org/packages/3c/b4/c5aae6c2dde269bf955f6b7d35065c655a57e47750d9668052ad67e74dda/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:926bdd3e3b5998ddf70cc64bc8cf57209571f9044542913afb673799fec77dd0", size = 390795, upload-time = "2026-10-04T16:29:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/a0/36/76fab39973ee11e7f9f357c55138197bb01c86f6502cb76487e3b4f42db0/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7868b85224291c6cb6759f9b5adb9745f486d226f62b16a614dd5a2a5ab2b35b", size = 370794, upload-time = "2026-10-04T16:29:47.603Z" }, + { url = "https://files.pythonhosted.org/packages/3d/fe/cd2a80e6d7b871937a60e935c5d507aa390d143f4ff3636f640b9733d5df/rpds_py-2026.9.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:3cd182d7291d29b92c521a0069d9c01ba6193628a9a105531d11b40a6d731a33", size = 375673, upload-time = "2026-10-04T16:29:49.223Z" }, + { url = "https://files.pythonhosted.org/packages/6c/18/7464a9953724e55a3b3206062fa0ffdeaa519584c6aabf65d3956d94f131/rpds_py-2026.9.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e6ea1cda8d8c688278430e4268a42f5e5da3bdd74578dfadc0820c3f1766ce83", size = 398758, upload-time = "2026-10-04T16:29:50.601Z" }, + { url = "https://files.pythonhosted.org/packages/c0/86/1534b436700fd49ff411063b7c4d7e938adfabf90895b6cf1622d5a7d1f6/rpds_py-2026.9.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5943980471829f6de242a20b109de3111ba6b77e3af0ffc587028ac854b05e6c", size = 550370, upload-time = "2026-10-04T16:29:52.002Z" }, + { url = "https://files.pythonhosted.org/packages/57/1c/e1fa82a8a01e3c5820f3ba98a8b2673f642128eb368fa88871b01dd2c909/rpds_py-2026.9.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:76d3af9732d2dab69f28179b40ba2d87e2f1d5824b4a694780aa787d685e8f36", size = 613106, upload-time = "2026-10-04T16:29:53.655Z" }, + { url = "https://files.pythonhosted.org/packages/29/55/b20b8c4c3dde8755bfcd5b08492a02d0cd2e26929aedf6199ca2a377d42a/rpds_py-2026.9.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:78326f4cb4427a56ba4996c0762b63be45f06b85f086526420d2b3a66e40f84d", size = 578235, upload-time = "2026-10-04T16:29:55.157Z" }, + { url = "https://files.pythonhosted.org/packages/55/42/df3f7bbc3f7ab37a8a9db8d6c2ff2c985422899f1f7926afbf7ec3c0b8b4/rpds_py-2026.9.1-cp313-cp313-win32.whl", hash = "sha256:172e47169583f46ce118cbec68e6795d0da0f4606b488b6434f8276bca0a058c", size = 205293, upload-time = "2026-10-04T16:29:56.669Z" }, + { url = "https://files.pythonhosted.org/packages/31/9c/ba5a9569d719bfdd6ce863df4133ac6a1658cf1b07cc3534c31db729fbbc/rpds_py-2026.9.1-cp313-cp313-win_amd64.whl", hash = "sha256:3e93b2cd69a9830be33e03945cd7cda940a0a8bfcfbff41d6144f0cb0d3d8bd9", size = 222409, upload-time = "2026-10-04T16:29:58.049Z" }, + { url = "https://files.pythonhosted.org/packages/35/72/f28ca566f6c23c35bbf7445f65eb0364577b25a026305995e24f78b83d94/rpds_py-2026.9.1-cp313-cp313-win_arm64.whl", hash = "sha256:d151e148117294133bf8af7eeace085e7e87432db15ab6adf640330298a47f6f", size = 217681, upload-time = "2026-10-04T16:29:59.449Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f2/67b94be1532767803415c1c5a1fd88ea487643d74a673cec1ba140af77bb/rpds_py-2026.9.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c9d1aca01f49170fdcf5c92761b1fafe97f554b721ca4570c5949fff778f0d4b", size = 347178, upload-time = "2026-10-04T16:30:00.865Z" }, + { url = "https://files.pythonhosted.org/packages/04/37/b751de2b59b0197a1d92a5dd491de88e8a5e928c2e6562581974f1e85263/rpds_py-2026.9.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3f0e9ac28fc067d4d34b88ae43c48e9489455c97fee9633d851f7eeed5a05d35", size = 341878, upload-time = "2026-10-04T16:30:02.564Z" }, + { url = "https://files.pythonhosted.org/packages/72/e2/5873bc4643c250db9e05d48dc0c93763d4aa68bc3b81164cb1af3b45b284/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07deecbfce94c78473018bc7d10b337cc651d12df87a1eb2cb3e4024bc9c33d0", size = 373963, upload-time = "2026-10-04T16:30:04.026Z" }, + { url = "https://files.pythonhosted.org/packages/51/03/5acf7632158247f3f6386ff0af3a1ee48167d575037e8d0920594b76d92b/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:821b2755db9194409254012f429c56643416fb96ef9be090be82ec8826b7f477", size = 377975, upload-time = "2026-10-04T16:30:05.555Z" }, + { url = "https://files.pythonhosted.org/packages/e6/00/63fda451b8bffa5808fc8bb311ee7c073b340974b09f273c7b2a145d3d62/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c91c210ae7645626c608400e3519b4a642f837cce09ca830db3beb2e9f274d4", size = 490347, upload-time = "2026-10-04T16:30:07.156Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ae/c093ffd070ba0fb02f76c565d06fecc65ad6e4afdbae78f7031076d3cdac/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:54ac2158a6f96cfbabff0b2eedaf94b90c5ec7ca8317fcadc61e1c2b2e0ff6ef", size = 392962, upload-time = "2026-10-04T16:30:08.77Z" }, + { url = "https://files.pythonhosted.org/packages/22/9d/d08a1128ab199b2f0cf25bfeb0639bd05119fff4b7c47bec24ef9a8ec23f/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eac2f5dbafd585dfe31f86a23ebf0d3ba480a9d49ebc87947267b5608d4ea0cd", size = 371623, upload-time = "2026-10-04T16:30:10.501Z" }, + { url = "https://files.pythonhosted.org/packages/53/c7/4758ddcbb75609414bbccfcb11d612436f9b3ee821bd2f33f0f1604ee648/rpds_py-2026.9.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:8aa5dda18d39b6143eb24809d158f9252c88f402749b6f1b62a506cc7d96cc35", size = 376997, upload-time = "2026-10-04T16:30:12.124Z" }, + { url = "https://files.pythonhosted.org/packages/87/e4/947bd7f608ff60faf46dc9d389c3dffd0e3d767d78a0be19978448ef0ce7/rpds_py-2026.9.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5c90e7fa02e8f5de0d10c17595c568ada48c5302e749462c0ea1a4c362111a86", size = 400169, upload-time = "2026-10-04T16:30:13.804Z" }, + { url = "https://files.pythonhosted.org/packages/4b/35/fe93e020a0543b5670472c18d7e6af3197c08da571240ce1965c84f85c0f/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e6d198bad4e49dd6732fbd636e2fc5c082f45c8cad0b4acb756b00c82c76072e", size = 551615, upload-time = "2026-10-04T16:30:15.332Z" }, + { url = "https://files.pythonhosted.org/packages/0d/4f/5d2a0136bb03b2a56a39dc6ff92d58a6e3e53a2e17079238b86228882f16/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:96beca19ec79de272e8668585380ff9092c47077c1d7a1e098e00bbd921f4785", size = 614655, upload-time = "2026-10-04T16:30:16.92Z" }, + { url = "https://files.pythonhosted.org/packages/09/1c/3f1025aaf70d9bf7272cc41f8b64ee76b48bf01726248138430e16f23b38/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a5cf77eb04f20b720be95265a3e00eb2a14814074255cc27069c551b2db53118", size = 579602, upload-time = "2026-10-04T16:30:18.555Z" }, + { url = "https://files.pythonhosted.org/packages/53/0d/5c72e6204f76610608706da32b6b7e11ef7e10317558a7bb15a122e008dc/rpds_py-2026.9.1-cp314-cp314-win32.whl", hash = "sha256:a03d57b86d2a51d0a66c92177e2be154ad015f357791d306e714569999cdb4cc", size = 206315, upload-time = "2026-10-04T16:30:20.05Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0b/489d48abbcc7d70cf3fbf662d9d22abf1f4650761c0a9ae05260800800d3/rpds_py-2026.9.1-cp314-cp314-win_amd64.whl", hash = "sha256:837c6b305e26fe0f75b15c92cf3b2ba29e0ae19dc40b1c557b026cb426347d0c", size = 222852, upload-time = "2026-10-04T16:30:21.604Z" }, + { url = "https://files.pythonhosted.org/packages/91/16/bbb05a7e6a10cf79ba639be7f799d770ee15f64175cc61d081b218dd402a/rpds_py-2026.9.1-cp314-cp314-win_arm64.whl", hash = "sha256:fce4b85234a0cbad67bf8e6e1201ee815d172c9aebad75f25645bc4d834f8e31", size = 218709, upload-time = "2026-10-04T16:30:23.036Z" }, + { url = "https://files.pythonhosted.org/packages/22/ac/ac507a0a4ec478ca470440a09583db4be5259ba7670aeba0620822f1e57a/rpds_py-2026.9.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3a72c11530d71abfb66c8d7696a2f86c43e63fca8b948f1a784ac490f4ec688e", size = 349596, upload-time = "2026-10-04T16:30:24.558Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f2/817a46b658d5070f477f722c298ee9a24525b0e4017347964146ef5fdd0e/rpds_py-2026.9.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:068c37bba854ec2fe42f7365c640af11dd9895890ccbf2df5070d0c059bd7f96", size = 343114, upload-time = "2026-10-04T16:30:26.048Z" }, + { url = "https://files.pythonhosted.org/packages/6c/42/6ade976b13ac1b4cb3bf2eb603f1be2fe74df19a29988d4c2b386be59d6f/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fca4eb6df565e2a928f1c7dad92d27db8f9df0f449e76423ed5d7e713ed445", size = 374197, upload-time = "2026-10-04T16:30:27.699Z" }, + { url = "https://files.pythonhosted.org/packages/d9/70/77cdf1d3f1a07faabe936016ae623aec7981f73108a8fe7a203ed2e21998/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c933c6678c6f116ff8af47a4c6db0868b8ace74af0343016c0ef00f00272ea69", size = 378025, upload-time = "2026-10-04T16:30:29.451Z" }, + { url = "https://files.pythonhosted.org/packages/3f/6b/18a44a3beaa9b7931acb04af7bd9539836477c630a794452d4826d6185d4/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:028ad274ea951dac64491b5d1e65712a4aeabfdbdb9fccf797b57bd899b0c495", size = 491969, upload-time = "2026-10-04T16:30:30.995Z" }, + { url = "https://files.pythonhosted.org/packages/73/27/fb39cfd6bddaf741b024f813890374578ff8ac1f1adc473659c048b03b05/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:740d0a99cf9de0b17a3943388e9294a59becf75e7c43421f387bd3c7a9901f7c", size = 393905, upload-time = "2026-10-04T16:30:32.628Z" }, + { url = "https://files.pythonhosted.org/packages/ed/71/0fa7bb77b57af0d710273964180d11b503f62b8a5c358eb2d8c3f62feff6/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0da298fb372dc192610a4b9ecbc68a0cd8b675bbbd1fc519d01b41cfd658333e", size = 374569, upload-time = "2026-10-04T16:30:34.257Z" }, + { url = "https://files.pythonhosted.org/packages/54/22/f41cfac269af3b449513ef1bc3d7f32fde52abbbd2d01c7e76b47743acd9/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:eb61be926bb81567c1f48bdc8aa22b9855048dc2efd53871f9f7e6e9a5632346", size = 377482, upload-time = "2026-10-04T16:30:35.997Z" }, + { url = "https://files.pythonhosted.org/packages/b4/fc/312b49006e7f8f9ca5f96647577b8aa6f3df30519c46bd448f5c425af0b2/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:42e75466f83cd43f6026c81eab74246efb2bdadafb307b85700632d06c68f299", size = 400040, upload-time = "2026-10-04T16:30:37.76Z" }, + { url = "https://files.pythonhosted.org/packages/cf/a6/18cca7a878dc7fa95165a83343fd4d7b65643fd22e54e47340a451121d5c/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:617f59cde379b4f648a09797b7f683d04b90a46344cddab85639da5aff0f5531", size = 551849, upload-time = "2026-10-04T16:30:39.443Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6d/1f5685e20f39604691bdc3c05aaa6b8bd2f954e9e996477adf2376768e33/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3edae8c5ddfdb6985d49ae9d150516e5076888879022f91a26c2de9276ce0bdb", size = 614842, upload-time = "2026-10-04T16:30:41.231Z" }, + { url = "https://files.pythonhosted.org/packages/c6/25/98652109fd9f7e10268dd4571aa52b81987001b806f37ef1a18260de714a/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0f045bb053c9057720d72c56dffe30dffdc05997b2897a827b9325f0ab6623fa", size = 582606, upload-time = "2026-10-04T16:30:43.345Z" }, + { url = "https://files.pythonhosted.org/packages/19/03/11ca09099bab5f53373a80a334c424ec917c13d050760a59499d2af5171e/rpds_py-2026.9.1-cp314-cp314t-win32.whl", hash = "sha256:bf35d0568abda97233239ce32896d3ad53fccc537832c104e30c94aa5fb93569", size = 203341, upload-time = "2026-10-04T16:30:44.954Z" }, + { url = "https://files.pythonhosted.org/packages/6f/8a/88909e3ffd9f47f5b58211473875d8c3c09079f0058c46fb72c55a702a26/rpds_py-2026.9.1-cp314-cp314t-win_amd64.whl", hash = "sha256:1e8d4d79d828299bf44a55db22a9388ab967b49d17132c88eab0f4360b48da8e", size = 222843, upload-time = "2026-10-04T16:30:46.486Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b7/a662f367d4896dd0a10cef2fc91f10b7f08af1c10858e287e019563f338d/rpds_py-2026.9.1-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:1d77b649e6f7cdf12ca5c2a98dad0ad37f9ea9b6f960408a92f0cb12bb3d04d9", size = 347463, upload-time = "2026-10-04T16:30:48.203Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3f/ad45d03df4f84ebae5439577ee81f3999d182711e82235c8037b6528890e/rpds_py-2026.9.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:00ba2d8c7dd4ee537978ddf4b3fbd712bef2d8751603f7f3146b3f4287768e25", size = 342051, upload-time = "2026-10-04T16:30:49.872Z" }, + { url = "https://files.pythonhosted.org/packages/7e/31/3dcd68c13d4bcc59c1f7eb33ac8e80698f06f3eb0d8a1e06419836071c20/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec450527cbf485e13c8d3602a54f428ab0432fdade0ede75efd74b735421c871", size = 374143, upload-time = "2026-10-04T16:30:51.508Z" }, + { url = "https://files.pythonhosted.org/packages/cf/0d/68c1f058a250fbd1380ebda9fc227cbf50117adf8ffe8161ef383ea79f68/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:306ee1850d8105b5baf977e78d45fcadd12c1a54678d614c9baf217708446e91", size = 378604, upload-time = "2026-10-04T16:30:53.206Z" }, + { url = "https://files.pythonhosted.org/packages/d7/d6/2d4c59b85397cb4800594fadf692688ccf5ce556adc930e7a5bf21061a5e/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ef6b65b03247c54692ad4fd9ee97cb772781927db72e3cb05e70b3db6d1ff14f", size = 490790, upload-time = "2026-10-04T16:30:54.925Z" }, + { url = "https://files.pythonhosted.org/packages/da/04/7e05dc3aebaf52f4e026766bd668fdd09a9d0e23f64a14686b36b3501892/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a575404ebc9cf2e91edd32eaf570ec1430eb900d4f56724ba7dd4bc1fc9c176d", size = 393146, upload-time = "2026-10-04T16:30:56.625Z" }, + { url = "https://files.pythonhosted.org/packages/57/ca/e2e9a0a46a74ed51a0498ba1fe10f4ea6b2d9a155f372a2f91e51f18cf10/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c16ab111bc27c646ba8aa005d0527754edc538ebb636f0b1bf8e244b48d1945", size = 372022, upload-time = "2026-10-04T16:30:58.295Z" }, + { url = "https://files.pythonhosted.org/packages/ba/cb/8f8774df5134e23424372838bcc5c7ed4127d723e1ff52f7bebd4dcb2563/rpds_py-2026.9.1-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:7664419f27db41d4f1c43a78dccda7dd6e8ef2428df3ee01d0c2a07a6b071297", size = 377092, upload-time = "2026-10-04T16:30:59.984Z" }, + { url = "https://files.pythonhosted.org/packages/90/02/8d7095d73bf9114219be40230baa5df00611e0a82ed9517779ff9c19f82b/rpds_py-2026.9.1-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4b26b03d9d2658ee2fa234f8f4f19f38a09773fe5261028025032e26d4d35af0", size = 400830, upload-time = "2026-10-04T16:31:01.721Z" }, + { url = "https://files.pythonhosted.org/packages/ec/02/8206856f8f363cd042a8315dc86b3912f5dcb6d3c66bbb24b69c2bfb0775/rpds_py-2026.9.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:be3e47e2d91aa3942ff9bf4077a505226005abfc39b6f7554a91c1b9393986b9", size = 551478, upload-time = "2026-10-04T16:31:03.472Z" }, + { url = "https://files.pythonhosted.org/packages/41/6b/36211f1bb1f0b0313f496d92f5905b74ea107f27cb16fa3355a82f04575e/rpds_py-2026.9.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:6307a0da524939decb8ca4a3933b8ab62525794411d6984fca6726e732804af6", size = 615173, upload-time = "2026-10-04T16:31:05.281Z" }, + { url = "https://files.pythonhosted.org/packages/35/77/cda0c4a6f055446b692f0ed5692f73707cafd3dff82672ede31b1a9b59de/rpds_py-2026.9.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:159a7aab5c5e8b112c8830f54717ce56da1252ebdbb526f5be2df2309280b9e7", size = 579653, upload-time = "2026-10-04T16:31:07.065Z" }, + { url = "https://files.pythonhosted.org/packages/74/ec/d8385f446240aed643b9e92a5055cff3015cc04a13c61f73b2883c478ed5/rpds_py-2026.9.1-cp315-cp315-win32.whl", hash = "sha256:dbc2673f9223d420c91145599b3ba45a8a50c207d1976908e5fb5ddb0c9b9429", size = 206498, upload-time = "2026-10-04T16:31:08.989Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a5/71b5cd00e0521e3b6b81828baea368c62b6b700ebdd9554cd7d41ddf12fa/rpds_py-2026.9.1-cp315-cp315-win_amd64.whl", hash = "sha256:75c38c50ab9aca840225d9a9a3810bf11d04bd5c1f186cabbb8aee56db3e9b15", size = 223121, upload-time = "2026-10-04T16:31:10.84Z" }, + { url = "https://files.pythonhosted.org/packages/33/58/dba857c3bc8221b31b62eb170a3080f4f191de79f047200389b7ed1b06a7/rpds_py-2026.9.1-cp315-cp315-win_arm64.whl", hash = "sha256:a431156bb41865fc14cd5d79bb9d7bbed83110b0159e34e62ae30951f96c0009", size = 218741, upload-time = "2026-10-04T16:31:12.592Z" }, + { url = "https://files.pythonhosted.org/packages/5b/d0/320ab28ccc1415eeb509d68682b0014fb74690cd49f1c2d29a232475af50/rpds_py-2026.9.1-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ef0d8c843e2827d6c120ab4687e9423fb1d893db1df27b7c1506615bcb9734a0", size = 349719, upload-time = "2026-10-04T16:31:14.48Z" }, + { url = "https://files.pythonhosted.org/packages/56/88/f5b12f1358f443c08b7ce3cc8391d82d335f4872580e2e097847fd36087a/rpds_py-2026.9.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:45bc6bccf78b20fd834237d18db64965d7ee68ba7f60440a26c7ab71e7b8d51a", size = 343267, upload-time = "2026-10-04T16:31:16.827Z" }, + { url = "https://files.pythonhosted.org/packages/f7/0c/c765b0059d532acb3b9c45d781ccc22f15a96dbe443d00903f643ba9df10/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d55198263bb51f557550c6ed2e6d1cb6a6fed6eb5c9120b741c5926bef8a45d", size = 374555, upload-time = "2026-10-04T16:31:18.931Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8a/cafddfda77564a10cd21184640c3bffda6a2b8d20972fb5dedd5e0166328/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a8763f20692da7df39b0afdd1ba3042b004c50a45994f76c2d9a25641f7673db", size = 378346, upload-time = "2026-10-04T16:31:20.75Z" }, + { url = "https://files.pythonhosted.org/packages/b9/01/5e626016eff72c183bf6c96539240cace15d402468647a453ec08415b2fc/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e43d4a1f673e8a1cbd8533e809e02b4bf9d4f2280269bb640436556312121250", size = 492373, upload-time = "2026-10-04T16:31:22.614Z" }, + { url = "https://files.pythonhosted.org/packages/3b/9c/15a2469e9389242f46896b3f0a01d68caea8a5a35c011fcb05ef333aae73/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea394a937f17a54c51239348bdbe2e3518124c8d4a8951ba04a311d3095bd18f", size = 394371, upload-time = "2026-10-04T16:31:24.768Z" }, + { url = "https://files.pythonhosted.org/packages/63/f5/c100ff77e1e6366e947c75969c258fdfc4b7bc5bbfe7351e62ffbf2a1228/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdeaa99ce822dca76cfb1b993e9120c5ea212f2eb66d48950ad63c349668a018", size = 375132, upload-time = "2026-10-04T16:31:26.588Z" }, + { url = "https://files.pythonhosted.org/packages/dd/f4/fe0269c9de253e99c81cabc12b8971a5feaa083debdaff1221e06264d9e3/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:b4f062343e7ad3fa94f2c66e5ae667dee47ee74dd41a9057c4fbe163236a123d", size = 377642, upload-time = "2026-10-04T16:31:28.677Z" }, + { url = "https://files.pythonhosted.org/packages/05/65/b34a7b257baccff8f4a24a722933166d4941d5ebdc9c3f4bc4ffcd5ce4f4/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:22ffd29a63d71fb1b81552c21f2c2b734949b7ac751a9be70675a939a900839b", size = 400855, upload-time = "2026-10-04T16:31:30.802Z" }, + { url = "https://files.pythonhosted.org/packages/98/32/844e54176b6071b90b38a564e6940bc6eb8f97b2890dc709c19db9dec0f4/rpds_py-2026.9.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:08dae4a4095150a7c4545a1fb40b98e1ab1744fbc2770d92c977b9dadaa49ab6", size = 552573, upload-time = "2026-10-04T16:31:32.709Z" }, + { url = "https://files.pythonhosted.org/packages/17/73/6041d20729dffbfdf155c02d65be58bc225a1c1fb548fd87c23ef306138f/rpds_py-2026.9.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:9a0460d43603d1fd9ef59c30278531e15d78581721ddb538fa560aa7817ea4ad", size = 615840, upload-time = "2026-10-04T16:31:34.565Z" }, + { url = "https://files.pythonhosted.org/packages/af/9e/418094adaee6b056ce199051b255448ed872829051e341b2294c80da0977/rpds_py-2026.9.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:1c2d1f6da5128eabf34e963d7163a818846075a52568250d006c4c953b40f903", size = 582764, upload-time = "2026-10-04T16:31:36.665Z" }, + { url = "https://files.pythonhosted.org/packages/3d/9b/1698ebf6b840ddfe8b472198abbed6ace35c6e398faeecd6c47dae742a4e/rpds_py-2026.9.1-cp315-cp315t-win32.whl", hash = "sha256:5c6ee90dee3e85e055ddfd502d611643d9b0fd94c818220bda84ec3dacd9b27b", size = 203795, upload-time = "2026-10-04T16:31:38.53Z" }, + { url = "https://files.pythonhosted.org/packages/8a/e2/91f70d804c61f8eac39a417e82aa1024f655ab9e8bdd7393b196242bc41b/rpds_py-2026.9.1-cp315-cp315t-win_amd64.whl", hash = "sha256:fe5ad0664ec772b02c45859041aa17655709cced7a31005817fbbbd988c25567", size = 223037, upload-time = "2026-10-04T16:31:40.468Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ad/f0eba548e297987964ef8088403db875b333963bf8189f3f563b6936b20e/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c0d2cb595a420b34d5086db0add011e26e2c09d6a024afbac4228bf8f863a30", size = 384112, upload-time = "2026-10-04T16:31:42.643Z" }, + { url = "https://files.pythonhosted.org/packages/33/89/891fa4d81f1eae7fe262ecd0aba2c51de94a116c7f3ccfb034fa75f4c537/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f3d6ed6a98cfd19155996605474982cc470d7601746a6439078f1a5a3fa8b050", size = 389662, upload-time = "2026-10-04T16:31:44.642Z" }, + { url = "https://files.pythonhosted.org/packages/4f/46/912cf623b2263f9c9c2d04e8508fc38dbe89983faeba5fb36ca4558ae550/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8171b44a054e5c67fd748ada04187f1250bf35b95f85e52ab64bcf3331a923bb", size = 504637, upload-time = "2026-10-04T16:31:46.667Z" }, + { url = "https://files.pythonhosted.org/packages/9b/5c/7f3cfa7c5f01765c9f47c8398210ed0c1eb686c0e99c223cb1c6776d3fb9/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fda1d96e542c37b6c804547dbf489c129fe7c97183a76a5ec275909ba1a063df", size = 402607, upload-time = "2026-10-04T16:31:49.068Z" }, + { url = "https://files.pythonhosted.org/packages/f0/76/2e915d4115ee453c33e6af17bc1df7e22ddc52974e178eeeb92a78c09983/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:270bdcdaac5d5b6f73c5e22e7e135c7f2a50e789f71d9e241d5be8d90026e19a", size = 382762, upload-time = "2026-10-04T16:31:51.24Z" }, + { url = "https://files.pythonhosted.org/packages/0d/e5/844382925d7b84a4074ad79fc2b9d05eb6231ea243b34f247e68e519b606/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:84a6ecc0c940169190d2c23bd969debd48c94dbc855acd60188a68d71d421608", size = 389563, upload-time = "2026-10-04T16:31:53.549Z" }, + { url = "https://files.pythonhosted.org/packages/81/82/d6e3951602dc2e24790ef9d299aa24bf5e2493df5043f2c6da00480cc447/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ab4b2fda7c2b542f7f9d886cc6a838c5079d2b76f72e6081411faba11adde2c9", size = 416842, upload-time = "2026-10-04T16:31:55.76Z" }, + { url = "https://files.pythonhosted.org/packages/ea/1c/8d248e4ea671d525ab7daa0703ad81cc2175076fa0ccd7ef088facbd2b01/rpds_py-2026.9.1-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:44b32a7c4f0da3d28af31c259e38ddcff096f855e205ed0671d02fcf44f1ea1c", size = 562434, upload-time = "2026-10-04T16:31:57.874Z" }, + { url = "https://files.pythonhosted.org/packages/ad/38/e7ac0db898120d15dea25e3661b1242a40cb3b23d1f4db86c15a01fe0ba1/rpds_py-2026.9.1-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:a3dbc5ed9514908d5046107d7b1346bde71eea61de6e0e4919c19354f97e769f", size = 627909, upload-time = "2026-10-04T16:31:59.977Z" }, + { url = "https://files.pythonhosted.org/packages/03/c8/37e555514932cb1ee36c75349187aa882a98dee5d5654c761699ed8a0d27/rpds_py-2026.9.1-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:6eae33003518fd4cb4f83a218d5371469dd3001aa3b87128c005b07762f7fe5e", size = 593658, upload-time = "2026-10-04T16:32:02.14Z" }, + { url = "https://files.pythonhosted.org/packages/12/5c/5f69a91add94a8a33acea68a2b034a464946a654ff7296af9d957ef0d6a5/rpds_py-2026.9.1-pp311-pypy311_pp80-macosx_10_12_x86_64.whl", hash = "sha256:0483515261947e4e8b8e1375bf7463e7eb6ccfb3d86e7b554d90cd5285f20f32", size = 351769, upload-time = "2026-10-04T16:32:04.675Z" }, + { url = "https://files.pythonhosted.org/packages/41/22/35ea8bf3b2682b90ab687c74e4828418ff12298a5b0b0a0c033f61c86a30/rpds_py-2026.9.1-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:4cfaf02209061880210819934de2f4f6aa83dc04dafe6770276acc240a56da31", size = 351719, upload-time = "2026-10-04T16:32:06.834Z" }, + { url = "https://files.pythonhosted.org/packages/a3/90/a9ba81408369d34c1aca73319c15adc91ed45c1d21e9e8ee4832ffb7fe0b/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6cdc537c8633d7fd92a82e2e0d2ab74320a3f63d5e59fb9cf08711e08fe151c4", size = 381109, upload-time = "2026-10-04T16:32:08.994Z" }, + { url = "https://files.pythonhosted.org/packages/b3/44/5192a0bed94cec86bd2a5e1cc5a1bb8f7eec3aec907eacd2deb13e87e326/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:10e208f2425d973938afcd56e28a7c4be32e27b6a60b5d381f49fb9d8acf9759", size = 388222, upload-time = "2026-10-04T16:32:11.321Z" }, + { url = "https://files.pythonhosted.org/packages/92/cd/5356549711448f18b52f7a11ffbe90f1228774996fbf3c6cd1b18d22abaf/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c0dbbcc19735fe5f8b0a54c07659d154a9e69f47e15d0a6ab7299215daf62cb", size = 500384, upload-time = "2026-10-04T16:32:13.478Z" }, + { url = "https://files.pythonhosted.org/packages/7a/88/ddda9d28adfe33119c75b2e733d4ba7e326f41d7e1b1093951771768ca48/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:684fd492fff4fead00587544e059be2bbcb6f93454f21fa2a91b66fc7508be82", size = 401661, upload-time = "2026-10-04T16:32:15.653Z" }, + { url = "https://files.pythonhosted.org/packages/32/c3/bb59ba16a6b4a57995d15b8c04c00f28df3b938c4dcdde48fbe0f73edb0c/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1028417bb44037eb3069c1009bd7b7277212876cda22fbe565b0bca9fab6d2c", size = 380748, upload-time = "2026-10-04T16:32:17.727Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b8/0580faa1c5a32ddc160130dd7ae54d2d007272a89a40a6b5db3e50f9f585/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:492e5e428cbe126221611f47e068f01660352feec4ad18bc0f5ea9b2ae88fb14", size = 385731, upload-time = "2026-10-04T16:32:19.855Z" }, + { url = "https://files.pythonhosted.org/packages/bf/70/f73564642bbe3322c7eeef2c2f258cf040b71c41a430b532e59d04a1b838/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:88b5268892fde430d5531f95bc560b6efbbd67c929662c586afd729a96e7461c", size = 412131, upload-time = "2026-10-04T16:32:22.285Z" }, + { url = "https://files.pythonhosted.org/packages/8c/5d/17fff2e1f8f68721a2afb5cb48f7e442c751bb6b4883157c5abaf618cd08/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:01445c8d194aa032a08e944f16567672da1c62dbdbefd8b6d0693032e290cf68", size = 558193, upload-time = "2026-10-04T16:32:24.4Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d0/869ab6fb08531f97aae4a780b4d61b4190d8aef48fe054ee28bcf7114466/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_i686.whl", hash = "sha256:eef6a03b0b6d08d0835ccfa8ec8d1bc70525e3801387567137b50c557695e6da", size = 622448, upload-time = "2026-10-04T16:32:26.844Z" }, + { url = "https://files.pythonhosted.org/packages/ba/a5/9672e532fe02cd3b92c78cfd8743abca939d96853006c63835009a1ecb6f/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:6b9bf3135b4ad5981df9a73d71a35272d650a2985ae9c2746357b24d59de2448", size = 591126, upload-time = "2026-10-04T16:32:29.274Z" }, + { url = "https://files.pythonhosted.org/packages/80/ef/3a9f8e4279c7920af561deed4cb7ebebed2794a8f608cf404d5eef14a105/rpds_py-2026.9.1-pp312-pypy312_pp80-macosx_10_12_x86_64.whl", hash = "sha256:56c6952a9b15047466d0c2347c446a761d4527f89976156341e68f0ce5cc08b0", size = 357108, upload-time = "2026-10-04T16:32:31.641Z" }, + { url = "https://files.pythonhosted.org/packages/b7/55/4b2fa381a583760aea5c92841e4e928a358e1c6511b129219e9c2826a226/rpds_py-2026.9.1-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:b242c27c8f836305a4a72df9cdd564386ac57b807bd252a063223331c9316b37", size = 346877, upload-time = "2026-10-04T16:32:34.061Z" }, +] + [[package]] name = "rq" version = "2.7.0" @@ -9207,7 +9944,8 @@ name = "scikit-learn" version = "1.8.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", @@ -9323,7 +10061,8 @@ name = "scipy" version = "1.17.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", @@ -9422,7 +10161,7 @@ dependencies = [ { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' or python_full_version >= '3.14'" }, { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and python_full_version < '3.14'" }, { name = "openai" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" } }, { name = "pyyaml" }, { name = "regex" }, { name = "tiktoken" }, @@ -9703,7 +10442,8 @@ name = "sphinx" version = "9.1.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", ] @@ -9791,7 +10531,8 @@ version = "2.0.49" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/09/45/461788f35e0364a8da7bda51a1fe1b09762d0c32f12f63727998d85a873b/sqlalchemy-2.0.49.tar.gz", hash = "sha256:d15950a57a210e36dd4cec1aac22787e2a4d57ba9318233e2ef8b2daf9ff2d5f", size = 9898221, upload-time = "2026-04-03T16:38:11.704Z" } wheels = [ @@ -9907,7 +10648,7 @@ version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } wheels = [ @@ -9929,7 +10670,7 @@ version = "0.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "exceptiongroup" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/f0/8d/e218e0160cc1b692e6e0e5ba34e8865dbb171efeb5fc9a704544b3020605/taskgroup-0.2.2.tar.gz", hash = "sha256:078483ac3e78f2e3f973e2edbf6941374fbea81b9c5d0a96f51d297717f4752d", size = 11504, upload-time = "2025-01-03T09:24:13.761Z" } wheels = [ @@ -9956,63 +10697,70 @@ wheels = [ [[package]] name = "tiktoken" -version = "0.12.0" +version = "0.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "regex" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7d/ab/4d017d0f76ec3171d469d80fc03dfbb4e48a4bcaddaa831b31d526f05edc/tiktoken-0.12.0.tar.gz", hash = "sha256:b18ba7ee2b093863978fcb14f74b3707cdc8d4d4d3836853ce7ec60772139931", size = 37806, upload-time = "2025-10-06T20:22:45.419Z" } +sdist = { url = "https://files.pythonhosted.org/packages/66/62/167a842aa0429d45f5e797354fd4343a96f6043d67d0513c675c7b8d36e6/tiktoken-0.14.0.tar.gz", hash = "sha256:231dec90efcdccf1b565a1416107736f1e09b1a08fe736ef9d6363e626d03874", size = 38898, upload-time = "2026-08-17T19:49:49.514Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/89/b3/2cb7c17b6c4cf8ca983204255d3f1d95eda7213e247e6947a0ee2c747a2c/tiktoken-0.12.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3de02f5a491cfd179aec916eddb70331814bd6bf764075d39e21d5862e533970", size = 1051991, upload-time = "2025-10-06T20:21:34.098Z" }, - { url = "https://files.pythonhosted.org/packages/27/0f/df139f1df5f6167194ee5ab24634582ba9a1b62c6b996472b0277ec80f66/tiktoken-0.12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b6cfb6d9b7b54d20af21a912bfe63a2727d9cfa8fbda642fd8322c70340aad16", size = 995798, upload-time = "2025-10-06T20:21:35.579Z" }, - { url = "https://files.pythonhosted.org/packages/ef/5d/26a691f28ab220d5edc09b9b787399b130f24327ef824de15e5d85ef21aa/tiktoken-0.12.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:cde24cdb1b8a08368f709124f15b36ab5524aac5fa830cc3fdce9c03d4fb8030", size = 1129865, upload-time = "2025-10-06T20:21:36.675Z" }, - { url = "https://files.pythonhosted.org/packages/b2/94/443fab3d4e5ebecac895712abd3849b8da93b7b7dec61c7db5c9c7ebe40c/tiktoken-0.12.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:6de0da39f605992649b9cfa6f84071e3f9ef2cec458d08c5feb1b6f0ff62e134", size = 1152856, upload-time = "2025-10-06T20:21:37.873Z" }, - { url = "https://files.pythonhosted.org/packages/54/35/388f941251b2521c70dd4c5958e598ea6d2c88e28445d2fb8189eecc1dfc/tiktoken-0.12.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6faa0534e0eefbcafaccb75927a4a380463a2eaa7e26000f0173b920e98b720a", size = 1195308, upload-time = "2025-10-06T20:21:39.577Z" }, - { url = "https://files.pythonhosted.org/packages/f8/00/c6681c7f833dd410576183715a530437a9873fa910265817081f65f9105f/tiktoken-0.12.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:82991e04fc860afb933efb63957affc7ad54f83e2216fe7d319007dab1ba5892", size = 1255697, upload-time = "2025-10-06T20:21:41.154Z" }, - { url = "https://files.pythonhosted.org/packages/5f/d2/82e795a6a9bafa034bf26a58e68fe9a89eeaaa610d51dbeb22106ba04f0a/tiktoken-0.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:6fb2995b487c2e31acf0a9e17647e3b242235a20832642bb7a9d1a181c0c1bb1", size = 879375, upload-time = "2025-10-06T20:21:43.201Z" }, - { url = "https://files.pythonhosted.org/packages/de/46/21ea696b21f1d6d1efec8639c204bdf20fde8bafb351e1355c72c5d7de52/tiktoken-0.12.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:6e227c7f96925003487c33b1b32265fad2fbcec2b7cf4817afb76d416f40f6bb", size = 1051565, upload-time = "2025-10-06T20:21:44.566Z" }, - { url = "https://files.pythonhosted.org/packages/c9/d9/35c5d2d9e22bb2a5f74ba48266fb56c63d76ae6f66e02feb628671c0283e/tiktoken-0.12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c06cf0fcc24c2cb2adb5e185c7082a82cba29c17575e828518c2f11a01f445aa", size = 995284, upload-time = "2025-10-06T20:21:45.622Z" }, - { url = "https://files.pythonhosted.org/packages/01/84/961106c37b8e49b9fdcf33fe007bb3a8fdcc380c528b20cc7fbba80578b8/tiktoken-0.12.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:f18f249b041851954217e9fd8e5c00b024ab2315ffda5ed77665a05fa91f42dc", size = 1129201, upload-time = "2025-10-06T20:21:47.074Z" }, - { url = "https://files.pythonhosted.org/packages/6a/d0/3d9275198e067f8b65076a68894bb52fd253875f3644f0a321a720277b8a/tiktoken-0.12.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:47a5bc270b8c3db00bb46ece01ef34ad050e364b51d406b6f9730b64ac28eded", size = 1152444, upload-time = "2025-10-06T20:21:48.139Z" }, - { url = "https://files.pythonhosted.org/packages/78/db/a58e09687c1698a7c592e1038e01c206569b86a0377828d51635561f8ebf/tiktoken-0.12.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:508fa71810c0efdcd1b898fda574889ee62852989f7c1667414736bcb2b9a4bd", size = 1195080, upload-time = "2025-10-06T20:21:49.246Z" }, - { url = "https://files.pythonhosted.org/packages/9e/1b/a9e4d2bf91d515c0f74afc526fd773a812232dd6cda33ebea7f531202325/tiktoken-0.12.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a1af81a6c44f008cba48494089dd98cccb8b313f55e961a52f5b222d1e507967", size = 1255240, upload-time = "2025-10-06T20:21:50.274Z" }, - { url = "https://files.pythonhosted.org/packages/9d/15/963819345f1b1fb0809070a79e9dd96938d4ca41297367d471733e79c76c/tiktoken-0.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:3e68e3e593637b53e56f7237be560f7a394451cb8c11079755e80ae64b9e6def", size = 879422, upload-time = "2025-10-06T20:21:51.734Z" }, - { url = "https://files.pythonhosted.org/packages/a4/85/be65d39d6b647c79800fd9d29241d081d4eeb06271f383bb87200d74cf76/tiktoken-0.12.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b97f74aca0d78a1ff21b8cd9e9925714c15a9236d6ceacf5c7327c117e6e21e8", size = 1050728, upload-time = "2025-10-06T20:21:52.756Z" }, - { url = "https://files.pythonhosted.org/packages/4a/42/6573e9129bc55c9bf7300b3a35bef2c6b9117018acca0dc760ac2d93dffe/tiktoken-0.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2b90f5ad190a4bb7c3eb30c5fa32e1e182ca1ca79f05e49b448438c3e225a49b", size = 994049, upload-time = "2025-10-06T20:21:53.782Z" }, - { url = "https://files.pythonhosted.org/packages/66/c5/ed88504d2f4a5fd6856990b230b56d85a777feab84e6129af0822f5d0f70/tiktoken-0.12.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:65b26c7a780e2139e73acc193e5c63ac754021f160df919add909c1492c0fb37", size = 1129008, upload-time = "2025-10-06T20:21:54.832Z" }, - { url = "https://files.pythonhosted.org/packages/f4/90/3dae6cc5436137ebd38944d396b5849e167896fc2073da643a49f372dc4f/tiktoken-0.12.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:edde1ec917dfd21c1f2f8046b86348b0f54a2c0547f68149d8600859598769ad", size = 1152665, upload-time = "2025-10-06T20:21:56.129Z" }, - { url = "https://files.pythonhosted.org/packages/a3/fe/26df24ce53ffde419a42f5f53d755b995c9318908288c17ec3f3448313a3/tiktoken-0.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:35a2f8ddd3824608b3d650a000c1ef71f730d0c56486845705a8248da00f9fe5", size = 1194230, upload-time = "2025-10-06T20:21:57.546Z" }, - { url = "https://files.pythonhosted.org/packages/20/cc/b064cae1a0e9fac84b0d2c46b89f4e57051a5f41324e385d10225a984c24/tiktoken-0.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:83d16643edb7fa2c99eff2ab7733508aae1eebb03d5dfc46f5565862810f24e3", size = 1254688, upload-time = "2025-10-06T20:21:58.619Z" }, - { url = "https://files.pythonhosted.org/packages/81/10/b8523105c590c5b8349f2587e2fdfe51a69544bd5a76295fc20f2374f470/tiktoken-0.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffc5288f34a8bc02e1ea7047b8d041104791d2ddbf42d1e5fa07822cbffe16bd", size = 878694, upload-time = "2025-10-06T20:21:59.876Z" }, - { url = "https://files.pythonhosted.org/packages/00/61/441588ee21e6b5cdf59d6870f86beb9789e532ee9718c251b391b70c68d6/tiktoken-0.12.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:775c2c55de2310cc1bc9a3ad8826761cbdc87770e586fd7b6da7d4589e13dab3", size = 1050802, upload-time = "2025-10-06T20:22:00.96Z" }, - { url = "https://files.pythonhosted.org/packages/1f/05/dcf94486d5c5c8d34496abe271ac76c5b785507c8eae71b3708f1ad9b45a/tiktoken-0.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a01b12f69052fbe4b080a2cfb867c4de12c704b56178edf1d1d7b273561db160", size = 993995, upload-time = "2025-10-06T20:22:02.788Z" }, - { url = "https://files.pythonhosted.org/packages/a0/70/5163fe5359b943f8db9946b62f19be2305de8c3d78a16f629d4165e2f40e/tiktoken-0.12.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:01d99484dc93b129cd0964f9d34eee953f2737301f18b3c7257bf368d7615baa", size = 1128948, upload-time = "2025-10-06T20:22:03.814Z" }, - { url = "https://files.pythonhosted.org/packages/0c/da/c028aa0babf77315e1cef357d4d768800c5f8a6de04d0eac0f377cb619fa/tiktoken-0.12.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:4a1a4fcd021f022bfc81904a911d3df0f6543b9e7627b51411da75ff2fe7a1be", size = 1151986, upload-time = "2025-10-06T20:22:05.173Z" }, - { url = "https://files.pythonhosted.org/packages/a0/5a/886b108b766aa53e295f7216b509be95eb7d60b166049ce2c58416b25f2a/tiktoken-0.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:981a81e39812d57031efdc9ec59fa32b2a5a5524d20d4776574c4b4bd2e9014a", size = 1194222, upload-time = "2025-10-06T20:22:06.265Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f8/4db272048397636ac7a078d22773dd2795b1becee7bc4922fe6207288d57/tiktoken-0.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9baf52f84a3f42eef3ff4e754a0db79a13a27921b457ca9832cf944c6be4f8f3", size = 1255097, upload-time = "2025-10-06T20:22:07.403Z" }, - { url = "https://files.pythonhosted.org/packages/8e/32/45d02e2e0ea2be3a9ed22afc47d93741247e75018aac967b713b2941f8ea/tiktoken-0.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:b8a0cd0c789a61f31bf44851defbd609e8dd1e2c8589c614cc1060940ef1f697", size = 879117, upload-time = "2025-10-06T20:22:08.418Z" }, - { url = "https://files.pythonhosted.org/packages/ce/76/994fc868f88e016e6d05b0da5ac24582a14c47893f4474c3e9744283f1d5/tiktoken-0.12.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d5f89ea5680066b68bcb797ae85219c72916c922ef0fcdd3480c7d2315ffff16", size = 1050309, upload-time = "2025-10-06T20:22:10.939Z" }, - { url = "https://files.pythonhosted.org/packages/f6/b8/57ef1456504c43a849821920d582a738a461b76a047f352f18c0b26c6516/tiktoken-0.12.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b4e7ed1c6a7a8a60a3230965bdedba8cc58f68926b835e519341413370e0399a", size = 993712, upload-time = "2025-10-06T20:22:12.115Z" }, - { url = "https://files.pythonhosted.org/packages/72/90/13da56f664286ffbae9dbcfadcc625439142675845baa62715e49b87b68b/tiktoken-0.12.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:fc530a28591a2d74bce821d10b418b26a094bf33839e69042a6e86ddb7a7fb27", size = 1128725, upload-time = "2025-10-06T20:22:13.541Z" }, - { url = "https://files.pythonhosted.org/packages/05/df/4f80030d44682235bdaecd7346c90f67ae87ec8f3df4a3442cb53834f7e4/tiktoken-0.12.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:06a9f4f49884139013b138920a4c393aa6556b2f8f536345f11819389c703ebb", size = 1151875, upload-time = "2025-10-06T20:22:14.559Z" }, - { url = "https://files.pythonhosted.org/packages/22/1f/ae535223a8c4ef4c0c1192e3f9b82da660be9eb66b9279e95c99288e9dab/tiktoken-0.12.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:04f0e6a985d95913cabc96a741c5ffec525a2c72e9df086ff17ebe35985c800e", size = 1194451, upload-time = "2025-10-06T20:22:15.545Z" }, - { url = "https://files.pythonhosted.org/packages/78/a7/f8ead382fce0243cb625c4f266e66c27f65ae65ee9e77f59ea1653b6d730/tiktoken-0.12.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0ee8f9ae00c41770b5f9b0bb1235474768884ae157de3beb5439ca0fd70f3e25", size = 1253794, upload-time = "2025-10-06T20:22:16.624Z" }, - { url = "https://files.pythonhosted.org/packages/93/e0/6cc82a562bc6365785a3ff0af27a2a092d57c47d7a81d9e2295d8c36f011/tiktoken-0.12.0-cp313-cp313t-win_amd64.whl", hash = "sha256:dc2dd125a62cb2b3d858484d6c614d136b5b848976794edfb63688d539b8b93f", size = 878777, upload-time = "2025-10-06T20:22:18.036Z" }, - { url = "https://files.pythonhosted.org/packages/72/05/3abc1db5d2c9aadc4d2c76fa5640134e475e58d9fbb82b5c535dc0de9b01/tiktoken-0.12.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:a90388128df3b3abeb2bfd1895b0681412a8d7dc644142519e6f0a97c2111646", size = 1050188, upload-time = "2025-10-06T20:22:19.563Z" }, - { url = "https://files.pythonhosted.org/packages/e3/7b/50c2f060412202d6c95f32b20755c7a6273543b125c0985d6fa9465105af/tiktoken-0.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:da900aa0ad52247d8794e307d6446bd3cdea8e192769b56276695d34d2c9aa88", size = 993978, upload-time = "2025-10-06T20:22:20.702Z" }, - { url = "https://files.pythonhosted.org/packages/14/27/bf795595a2b897e271771cd31cb847d479073497344c637966bdf2853da1/tiktoken-0.12.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:285ba9d73ea0d6171e7f9407039a290ca77efcdb026be7769dccc01d2c8d7fff", size = 1129271, upload-time = "2025-10-06T20:22:22.06Z" }, - { url = "https://files.pythonhosted.org/packages/f5/de/9341a6d7a8f1b448573bbf3425fa57669ac58258a667eb48a25dfe916d70/tiktoken-0.12.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:d186a5c60c6a0213f04a7a802264083dea1bbde92a2d4c7069e1a56630aef830", size = 1151216, upload-time = "2025-10-06T20:22:23.085Z" }, - { url = "https://files.pythonhosted.org/packages/75/0d/881866647b8d1be4d67cb24e50d0c26f9f807f994aa1510cb9ba2fe5f612/tiktoken-0.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:604831189bd05480f2b885ecd2d1986dc7686f609de48208ebbbddeea071fc0b", size = 1194860, upload-time = "2025-10-06T20:22:24.602Z" }, - { url = "https://files.pythonhosted.org/packages/b3/1e/b651ec3059474dab649b8d5b69f5c65cd8fcd8918568c1935bd4136c9392/tiktoken-0.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8f317e8530bb3a222547b85a58583238c8f74fd7a7408305f9f63246d1a0958b", size = 1254567, upload-time = "2025-10-06T20:22:25.671Z" }, - { url = "https://files.pythonhosted.org/packages/80/57/ce64fd16ac390fafde001268c364d559447ba09b509181b2808622420eec/tiktoken-0.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:399c3dd672a6406719d84442299a490420b458c44d3ae65516302a99675888f3", size = 921067, upload-time = "2025-10-06T20:22:26.753Z" }, - { url = "https://files.pythonhosted.org/packages/ac/a4/72eed53e8976a099539cdd5eb36f241987212c29629d0a52c305173e0a68/tiktoken-0.12.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:c2c714c72bc00a38ca969dae79e8266ddec999c7ceccd603cc4f0d04ccd76365", size = 1050473, upload-time = "2025-10-06T20:22:27.775Z" }, - { url = "https://files.pythonhosted.org/packages/e6/d7/0110b8f54c008466b19672c615f2168896b83706a6611ba6e47313dbc6e9/tiktoken-0.12.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cbb9a3ba275165a2cb0f9a83f5d7025afe6b9d0ab01a22b50f0e74fee2ad253e", size = 993855, upload-time = "2025-10-06T20:22:28.799Z" }, - { url = "https://files.pythonhosted.org/packages/5f/77/4f268c41a3957c418b084dd576ea2fad2e95da0d8e1ab705372892c2ca22/tiktoken-0.12.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:dfdfaa5ffff8993a3af94d1125870b1d27aed7cb97aa7eb8c1cefdbc87dbee63", size = 1129022, upload-time = "2025-10-06T20:22:29.981Z" }, - { url = "https://files.pythonhosted.org/packages/4e/2b/fc46c90fe5028bd094cd6ee25a7db321cb91d45dc87531e2bdbb26b4867a/tiktoken-0.12.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:584c3ad3d0c74f5269906eb8a659c8bfc6144a52895d9261cdaf90a0ae5f4de0", size = 1150736, upload-time = "2025-10-06T20:22:30.996Z" }, - { url = "https://files.pythonhosted.org/packages/28/c0/3c7a39ff68022ddfd7d93f3337ad90389a342f761c4d71de99a3ccc57857/tiktoken-0.12.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:54c891b416a0e36b8e2045b12b33dd66fb34a4fe7965565f1b482da50da3e86a", size = 1194908, upload-time = "2025-10-06T20:22:32.073Z" }, - { url = "https://files.pythonhosted.org/packages/ab/0d/c1ad6f4016a3968c048545f5d9b8ffebf577774b2ede3e2e352553b685fe/tiktoken-0.12.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5edb8743b88d5be814b1a8a8854494719080c28faaa1ccbef02e87354fe71ef0", size = 1253706, upload-time = "2025-10-06T20:22:33.385Z" }, - { url = "https://files.pythonhosted.org/packages/af/df/c7891ef9d2712ad774777271d39fdef63941ffba0a9d59b7ad1fd2765e57/tiktoken-0.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f61c0aea5565ac82e2ec50a05e02a6c44734e91b51c10510b084ea1b8e633a71", size = 920667, upload-time = "2025-10-06T20:22:34.444Z" }, + { url = "https://files.pythonhosted.org/packages/5e/82/d60a7a5d7bff7b4641d556ea68ea5914ea6edc3774a12eb1c0d444701382/tiktoken-0.14.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3b12e54f8bec91433e41aff65d8d1f209a4f678081163747079806e5361f6c91", size = 1095817, upload-time = "2026-08-17T19:48:31.788Z" }, + { url = "https://files.pythonhosted.org/packages/18/e2/d39ae33d3dc30a0c229ff0cb683df961ebb5e7b8691feb2d08b3ee6ac327/tiktoken-0.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:94f77b60a8ab23580db19ae822744c9716c1720020d2179ca5605112d12326f1", size = 1043064, upload-time = "2026-08-17T19:48:33.138Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e9/8e18cbee0c3ae8321c7e9696bef6090a24eed99a4a75a4c4a7f5115e5a2f/tiktoken-0.14.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:f3d6cf93fbe2e7117eb7bedca684216fbe328a41f0843ce34245451d8eb2df1c", size = 1190381, upload-time = "2026-08-17T19:48:34.386Z" }, + { url = "https://files.pythonhosted.org/packages/af/c8/051e7b72a816ff50eb34a1c7c5b185cd2429ffdf59a497baea35b2b6b2dd/tiktoken-0.14.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:18a1b651c4b032004bf7b4f1713391a54b2a341a52c6e8a2b59acae9d16e13c7", size = 1206869, upload-time = "2026-08-17T19:48:35.581Z" }, + { url = "https://files.pythonhosted.org/packages/c3/b3/7795db206adb6a57d6137fe48ef2cca6b9707e90b86ee8244671592ddc33/tiktoken-0.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4d8d91d68353bd167fdf26467e5ff9e56aaa5f87d6410c0238608629e4dc0d33", size = 1255197, upload-time = "2026-08-17T19:48:36.832Z" }, + { url = "https://files.pythonhosted.org/packages/c8/39/5234783af6b81af645ccdf9438f2f02af472f14e91d876ca2079af641841/tiktoken-0.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:10f31e63e40313f2e518d87f7086cfa44e45f64cc14d8ae14103b41220c30a14", size = 1319329, upload-time = "2026-08-17T19:48:37.944Z" }, + { url = "https://files.pythonhosted.org/packages/88/cf/f2d955c8c5c6c67cc86ba6fb132c47c710465ebe6a6dcec1c3b6e250660e/tiktoken-0.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:c6cb9896a82b9ee44e15ba0b5c8044072f2e4d48acaa704c8d3feeef5ad9487c", size = 944146, upload-time = "2026-08-17T19:48:39.011Z" }, + { url = "https://files.pythonhosted.org/packages/8f/c5/9d848b7f408241171e1f843deb8bfa626086452bc9c78beee500829583e3/tiktoken-0.14.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:c2edf09b381fafbc014ae8e018ed25087abb9a3dafa8465a0ea63c6558c47a79", size = 1094971, upload-time = "2026-08-17T19:48:40.347Z" }, + { url = "https://files.pythonhosted.org/packages/2d/a9/d94302340304328961d6f0c35ca4e60617fbb57a5cf667e2ed1692cb9e57/tiktoken-0.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cd8ca1305c1c902fe42c486165f2e4808d9997625c98ffb05b9e0366d99d3948", size = 1042916, upload-time = "2026-08-17T19:48:41.541Z" }, + { url = "https://files.pythonhosted.org/packages/c8/b6/31da98ee871383509cae2ba96a9ddef1965e3c4f8cb6dc7bcda3379398db/tiktoken-0.14.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:1f83081065ee5833d35b49e9180f3d8d15622a603dd1c435da0da6cc12b3662f", size = 1188650, upload-time = "2026-08-17T19:48:42.729Z" }, + { url = "https://files.pythonhosted.org/packages/24/65/8c5dddd7cb67f6571d154a58d7c6e2f07da54bf84c49b6a1839965b7c35e/tiktoken-0.14.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:f5e7665f6624e052e5e7f6a36919ab69279decdc976d7b16b4fa15e1897d0513", size = 1206378, upload-time = "2026-08-17T19:48:44.013Z" }, + { url = "https://files.pythonhosted.org/packages/d1/04/522ec59d30dd9a2f3ab837011cd4fc5d1178dc4a2fa07c9fa4b90af6ba9d/tiktoken-0.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:144a3fc369f92b7d548995217c5d6e84038d3572157a0f6f34080d65291d0f78", size = 1253694, upload-time = "2026-08-17T19:48:45.597Z" }, + { url = "https://files.pythonhosted.org/packages/69/84/9019e272bad188a1c61ecf44f25a9ba2368744644e3ac1f3d6516f3c9e80/tiktoken-0.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:151d37a150c8f3dfc5f4345597b10e101876bd1bd13494e0185af6b508758d2e", size = 1317873, upload-time = "2026-08-17T19:48:46.792Z" }, + { url = "https://files.pythonhosted.org/packages/24/7f/fff1217240343c0c11b5938b98aeae0e3a266cacfac25f86f91cdcd748f0/tiktoken-0.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:c77d4a3e1deb2707819df92046b89aad1ac81d27e07616b797cbff3f62c037da", size = 944395, upload-time = "2026-08-17T19:48:48.028Z" }, + { url = "https://files.pythonhosted.org/packages/8c/da/e273746b9d24a63c776bc60fba914351573ad9c575b52601eb5e60632564/tiktoken-0.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8e947aefe98ef74cce94923f90e48c98fe34eb1ec0a6bfdfadfc5a96359bfc36", size = 1094408, upload-time = "2026-08-17T19:48:49.269Z" }, + { url = "https://files.pythonhosted.org/packages/69/9f/fe6b1aca23331aa5271df5a4bd07bf68a7059254d47faee1b8272592a777/tiktoken-0.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d6cebe67765569df3dafac8474e4eccf5c19d24140492567a5e58a11445732a4", size = 1038499, upload-time = "2026-08-17T19:48:50.666Z" }, + { url = "https://files.pythonhosted.org/packages/0b/35/e9f47647c9e163bd1de30fe1a491669b7248cfc67b7404c35c009a701e1a/tiktoken-0.14.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:7db45b98e94adf4173a5cd7422b150999a7ee11ff847783a14f6e1b80cc38cb6", size = 1186355, upload-time = "2026-08-17T19:48:51.93Z" }, + { url = "https://files.pythonhosted.org/packages/51/11/9976ad86980a00cdef05e730a0127a2578a1bc6d11644d8d47246de2eb26/tiktoken-0.14.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:7896eea257fe497a2b7134474d909156c6744ce8da35bce88011a960e008aa0d", size = 1204197, upload-time = "2026-08-17T19:48:53.18Z" }, + { url = "https://files.pythonhosted.org/packages/d4/9c/7035b0bcfaa68d1ee4803fc5be5214ad865669b05bd20e7105ae8a18afc6/tiktoken-0.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b950248272f1b303dc32986396e2dccfa10cf6d1e83ec8f0bba1776660305482", size = 1250635, upload-time = "2026-08-17T19:48:54.392Z" }, + { url = "https://files.pythonhosted.org/packages/bc/1d/69cabf18bed7f4366da076735816abce0d4db3fae491ae338a6612128777/tiktoken-0.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3de75343041a1c57333b1e707ac8a9769738241d7d6a55d39e12cf84548337c6", size = 1316085, upload-time = "2026-08-17T19:48:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/bd/bd/a2e884fb1402cba5be08836590320012b2d8ada0e2eef9911a64df4bcd2d/tiktoken-0.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:087538c080e5ff421abd3a0785ed63c5111d06af98e6cd0d374dbe5969147ca3", size = 941208, upload-time = "2026-08-17T19:48:56.938Z" }, + { url = "https://files.pythonhosted.org/packages/50/53/ee1453623bf65f019328721ccb6587846d2c5b7b82f34e73ca09101f072e/tiktoken-0.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9c5fe393aab56469f04e432ff851216d3def3436cf5f07e442a240164bf500f", size = 1094198, upload-time = "2026-08-17T19:48:57.955Z" }, + { url = "https://files.pythonhosted.org/packages/ad/5f/6448cfe278c3664ba9ec5b5ac08344341f7dc3d42888476e215a14eda2be/tiktoken-0.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cbe2cc3bba939bcdaf103e03df9d5039d33887080b315624be28ec69059e5f94", size = 1038820, upload-time = "2026-08-17T19:48:59.015Z" }, + { url = "https://files.pythonhosted.org/packages/69/3b/d67eac1bcce9dee3abe23aff5e3ded3116bbebaf67b80a0811c06d3806fc/tiktoken-0.14.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:2157f52e4b4d7ac5ecc7457b3716834706e7ef9a46f5144029bfeb7cf71f4e06", size = 1186175, upload-time = "2026-08-17T19:49:00.068Z" }, + { url = "https://files.pythonhosted.org/packages/37/62/cae690d9783146b0f81f564ada0f8f611de68178c0c9c7e1e969f0516b48/tiktoken-0.14.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:26e60f6a956ee171ab728b37b8439905d7ea1db435c30f9822f291e9861c861d", size = 1203884, upload-time = "2026-08-17T19:49:01.163Z" }, + { url = "https://files.pythonhosted.org/packages/b9/1e/633e30237b94e383cf814145499079f3bb9cdd4aeafc1bc42e01b0f810a6/tiktoken-0.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:380873f330b741c4435574f37edb20813d04603ace2d53e0a63560e1fec83010", size = 1250980, upload-time = "2026-08-17T19:49:02.274Z" }, + { url = "https://files.pythonhosted.org/packages/cb/56/4c12f07b812f84206f38d723eb1ebfdd34bad9309b5dbc0bee6bbcff4cbf/tiktoken-0.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3fd7c14b1cb45b486c39fc9b3443bb341f3e2fc7e6f31247f3435a5836651632", size = 1315434, upload-time = "2026-08-17T19:49:03.434Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e0/c65603f0c44811def666d3fbf611bf2af3b5e1ef613e06c19411419830b3/tiktoken-0.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:90a762670c7f968184723769a06ed51f5cf5ce5dcd1e30164f25c72d85c2d1f1", size = 940883, upload-time = "2026-08-17T19:49:04.583Z" }, + { url = "https://files.pythonhosted.org/packages/59/b0/1cf129f4af8fc513931f931023def596b7c4bfc77026513cd9d851da9e88/tiktoken-0.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e067f4cbcc5d036e8aff7fe7a6b530a8f4de2e4616ad9005a24a1879e24e6450", size = 1096273, upload-time = "2026-08-17T19:49:05.807Z" }, + { url = "https://files.pythonhosted.org/packages/62/85/2ae74575e321148484147e10b53c3b1717c59ebaa9edb4fe18b1f5c055f8/tiktoken-0.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f2af4a336ea56d6c14f27741a0e1d8294a35dd0b038bcf990d232ebb54eb994b", size = 1040269, upload-time = "2026-08-17T19:49:06.943Z" }, + { url = "https://files.pythonhosted.org/packages/89/29/92a1120a12e4bcf2d5464350d1a91b68a433d63ce656bb7f806c27aec09c/tiktoken-0.14.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f702e0aeeb6506e57687e881c59e844ebe8f0a6a097ddafe20e3ab25f387be4e", size = 1186101, upload-time = "2026-08-17T19:49:08.102Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7d/144af98dc5ad68108451a82e2f5a17f80e2663f5115058b8dfd215c1ad02/tiktoken-0.14.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:e3442bbb2f0c588cec876061e37ae67b455b9df9978b003c8fe30e45f2ef5b42", size = 1204457, upload-time = "2026-08-17T19:49:09.28Z" }, + { url = "https://files.pythonhosted.org/packages/e6/1f/be7cb06ab2108f612f3e92e7b76cf391e192db0db37a984616f0cc32aafc/tiktoken-0.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:979c1524f753b662b0f3cd261b135afe6659cce33caaa7a5ea00dd1756b3055c", size = 1251716, upload-time = "2026-08-17T19:49:10.509Z" }, + { url = "https://files.pythonhosted.org/packages/ab/6b/81f158d0f90adb826cd704069c2129a046cb784a2a09861009519fc41cf4/tiktoken-0.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2cc19ac87b41c9493c9778ff5847f0c8bbcf5bd0ec6b87ce06c1c802adc8a771", size = 1315432, upload-time = "2026-08-17T19:49:11.844Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ec/f5fa35ec13f07279fdcaf3cc9c04bbb154ea591d23978651f2b672593e8a/tiktoken-0.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:eceeff0c62419bc78d4b6e70a4762a4d25df3ae8f2d5946e3853ce93e7a57098", size = 988046, upload-time = "2026-08-17T19:49:13.282Z" }, + { url = "https://files.pythonhosted.org/packages/68/c9/7756717408d3d0dfea3f046c9466144b28afde39ff69d5808f2475dcd7f5/tiktoken-0.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:6eb94895c45f26bb8f5546e5fd8a069efcf6e3f108ea9d5cbe3bf6f7f3983438", size = 1096261, upload-time = "2026-08-17T19:49:14.351Z" }, + { url = "https://files.pythonhosted.org/packages/79/29/46ad8061f57bd9f8b2ea0aa82bf574e0f2aa040b0857a1582adba9957899/tiktoken-0.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:86951a971c53979ec857bd8c4a32dc227ab0fd33f6c12a3bd62d3fbf5f0bfcaa", size = 1040183, upload-time = "2026-08-17T19:49:15.707Z" }, + { url = "https://files.pythonhosted.org/packages/5a/7c/3184d17b868456f17b60b1a75f5ec0405618a43aa753336df341d8f11781/tiktoken-0.14.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:e2eca764c53490f8930dbce329e0769f11108d87d908282a80c5c130e26e7037", size = 1186719, upload-time = "2026-08-17T19:49:16.84Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e8/46de4400d5bf859f640feee85bd7e32235f68ddf25db53c63be78e581e3a/tiktoken-0.14.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:26cc4b4840fa0e9f4b72ed489883e12f57e00d1021ca794720e3c29a12f0edef", size = 1204660, upload-time = "2026-08-17T19:49:17.987Z" }, + { url = "https://files.pythonhosted.org/packages/29/ce/af8964c38bc8226dd8950305b7a255fa33345d5572f78af7275a313d28e0/tiktoken-0.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2fc834fbe3f6a0736905c36ab709537e6840dbd63b982dc9e0216ae7d305ba1a", size = 1250932, upload-time = "2026-08-17T19:49:19.28Z" }, + { url = "https://files.pythonhosted.org/packages/1d/4b/323631116fc986d9cc5bbeb2b8223c7c85e61a8bb94ea5ab4951023b149b/tiktoken-0.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ca4db6ff5c5bf600f9b7761a0070ed44dfe5797a76bd432fb978bc480ef40c58", size = 1315190, upload-time = "2026-08-17T19:49:20.467Z" }, + { url = "https://files.pythonhosted.org/packages/18/8b/ba48a73729c9270989b36f37ab2ed5525e52690d715097c9fa791aaa5d05/tiktoken-0.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:7aab286a020660a039097912a088236b985d18a3090d73f136c4413d29d37ca0", size = 987717, upload-time = "2026-08-17T19:49:21.704Z" }, + { url = "https://files.pythonhosted.org/packages/1d/10/b73b7e319179e0f60b32475f783b044f9cece872c53b6662664e9084b0d0/tiktoken-0.14.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:14b47e3674f2624803a8acc8fb367b7e24fc53055f9df3296482fe9a3a34a232", size = 1096280, upload-time = "2026-08-17T19:49:22.779Z" }, + { url = "https://files.pythonhosted.org/packages/c2/6b/09999a9bf1d559670d1680e8f8e419ac0e2c5f6aac82e9bfdf70f260b30a/tiktoken-0.14.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:19d643d701fdaa70e5b9c7f8f96abcaffe77ca5e482a3a1a7dde46feb4284695", size = 1040433, upload-time = "2026-08-17T19:49:23.998Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7b/8537be0836f3df99b2a636b44399bfa43cd757f2b8b4097dacb794cf24a7/tiktoken-0.14.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:e4ddf863b59347deaa92302dcd90e5eb003cdc9be06ec2b692c38d1bdd9efd49", size = 1186989, upload-time = "2026-08-17T19:49:25.021Z" }, + { url = "https://files.pythonhosted.org/packages/7c/9d/f9c56d7a943a4468abf9ef37661bb9b8e0cd3aa8aa87368c7146cc3f3222/tiktoken-0.14.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:60c47ca69ddda0dea8256fffd12e1b86f4b59734a20e4a70c61f63cc5f021df4", size = 1204615, upload-time = "2026-08-17T19:49:26.37Z" }, + { url = "https://files.pythonhosted.org/packages/4b/d2/98a38579db25c4a8a84e31dd95d9072ec5f21f7e70de591da0412e29b25b/tiktoken-0.14.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:728303a072163130c5b477b1f20d6211895569c1d5302c24ffc93a3009160871", size = 1251828, upload-time = "2026-08-17T19:49:27.423Z" }, + { url = "https://files.pythonhosted.org/packages/0c/83/467be424746c039c5493c0f4102feab16b9b48eb6f5c089b2a2438e3cde2/tiktoken-0.14.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:3c5349c9f916283bba32bec8af69b763e4faa304dc004d0eaaea66a3cf004c1f", size = 1316260, upload-time = "2026-08-17T19:49:29.101Z" }, + { url = "https://files.pythonhosted.org/packages/02/ee/ddf46ca78e371f5890e96b6e7d089a85b3536432be219851eb0481786ca8/tiktoken-0.14.0-cp315-cp315-win_amd64.whl", hash = "sha256:1b6e4adcfd285c44502aed51df98aaaca4f0fea028165dbf8a9e857b9f98d8ea", size = 988230, upload-time = "2026-08-17T19:49:30.246Z" }, + { url = "https://files.pythonhosted.org/packages/2a/00/5162e90c851a28da18ed382d34898b79a8022548e5619a64e14c03ce7c3d/tiktoken-0.14.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:11d8211b290855d2721334ff17dd9b3a17bfb26872be01f25d73612ef7ece890", size = 1096186, upload-time = "2026-08-17T19:49:31.656Z" }, + { url = "https://files.pythonhosted.org/packages/65/97/a5a7bfccf25b1bb65e82bae8edff11ac3c9c041c374b7b4a823d60c38133/tiktoken-0.14.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:d0781223705199b289faa59601bb9c2441712d4c600dd13c43d8fd6a33d22cd5", size = 1039947, upload-time = "2026-08-17T19:49:32.848Z" }, + { url = "https://files.pythonhosted.org/packages/fb/ba/ef427fc638f1439181c5e12dd26b70e881861f89c007aa7e5b36300f8342/tiktoken-0.14.0-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:2ea70afba6b9eddbf22c165142e5f0a2ad7aa36a452873c48b57bb2aeb8492ae", size = 1186997, upload-time = "2026-08-17T19:49:34.121Z" }, + { url = "https://files.pythonhosted.org/packages/3e/88/2f3f85a968cdc514152129af0a060ebcccb067005a2f29b0d5ef3c838514/tiktoken-0.14.0-cp315-cp315t-manylinux_2_28_x86_64.whl", hash = "sha256:78571efc311c30b73f31eb949a921d6dac39a5d9dc42d1cfa8f8db157b3447b1", size = 1205211, upload-time = "2026-08-17T19:49:35.284Z" }, + { url = "https://files.pythonhosted.org/packages/4e/f6/80760e98a08e6649d2d68afb6035af713121dfb615acce8c4f73810ec438/tiktoken-0.14.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:86f66c85e796f5d05d5c4a60ec1d40cbfebc47a32464053528c797163fa9ab89", size = 1251479, upload-time = "2026-08-17T19:49:36.419Z" }, + { url = "https://files.pythonhosted.org/packages/c5/84/50966fb6918a0fb9b32721277e5342bf729a2d74350074d662fbedf9772e/tiktoken-0.14.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:149d97453c4c98c04b081d64a85e635921269b532710d6faf81e9e82b790e7d3", size = 1316673, upload-time = "2026-08-17T19:49:37.756Z" }, + { url = "https://files.pythonhosted.org/packages/35/5e/9b01afd037bfa22a0033963fa091e0f75b6fb15cd85bffb42ff86e697323/tiktoken-0.14.0-cp315-cp315t-win_amd64.whl", hash = "sha256:561e7580f84a79859af1ef6f676968e9030fcc3fe195700b15235bca64f009c9", size = 987929, upload-time = "2026-08-17T19:49:38.947Z" }, ] [[package]] @@ -10179,7 +10927,8 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "opentelemetry-semantic-conventions-ai" }, { name = "posthog" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "tenacity" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0a/b1/fd7360d97c651098da505e95600e067a7eedb1b78635b2f1d23545ee4a46/traceloop_sdk-0.34.0.tar.gz", hash = "sha256:4aa26003dfa2e417f73728bd847284a12d6da43a946dd588603a0966e753b3e6", size = 19808, upload-time = "2024-12-12T21:03:41.647Z" } @@ -10212,7 +10961,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore-stubs" }, { name = "types-s3transfer" }, - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bd/9c/904b71c1ffb9ddbfe0367e36ddd142c12a192b958cc10701d09888fb8beb/types_boto3-1.43.30.tar.gz", hash = "sha256:f4d9295a136325f5086f3967e33ec769555004b299bd11173875772393d5d907", size = 103364, upload-time = "2026-06-15T21:23:31.718Z" } wheels = [ @@ -10247,7 +10996,7 @@ name = "types-boto3-bedrock" version = "1.43.26" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/99/d7/22e117e8077f51b704d67a4c48deca60a893fc6c6efd13a1e582ab8b4049/types_boto3_bedrock-1.43.26.tar.gz", hash = "sha256:55c338ae47aef6f98ba1f188bc2e9f02794efbc346b68606bbe9751d4e1405a5", size = 67312, upload-time = "2026-06-09T20:33:02.407Z" } wheels = [ @@ -10259,7 +11008,7 @@ name = "types-boto3-bedrock-agent" version = "1.43.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ea/d0/7a4111691706006ba3e9ad9ddd1cd7bb562ade4138171126e0c27d2e7901/types_boto3_bedrock_agent-1.43.0.tar.gz", hash = "sha256:a3f5d8404e31c8315318e6149a6714930cdbddae84c610bb2483a13cac0a89fa", size = 53500, upload-time = "2026-04-29T22:59:28.167Z" } wheels = [ @@ -10271,7 +11020,7 @@ name = "types-boto3-bedrock-runtime" version = "1.43.30" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a2/28/dd863429fbcc7a38389b5d287836e40d9df20398d6c215387122b3453779/types_boto3_bedrock_runtime-1.43.30.tar.gz", hash = "sha256:0e79ec50a26b12b2da17a203983c81b60982abe7e17c464a5cf74c3a6637f504", size = 31282, upload-time = "2026-06-15T21:23:19.526Z" } wheels = [ @@ -10283,7 +11032,7 @@ name = "types-boto3-kms" version = "1.43.12" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0d/46/7343b52e16eaa9dec7099cdd6a901317df583473b1658d04cb42885c8d03/types_boto3_kms-1.43.12.tar.gz", hash = "sha256:f9a06ca5a1cbf02f820208f1e84983a750daa1bce305bd11231961a9d770d9cd", size = 30696, upload-time = "2026-05-20T20:01:12.294Z" } wheels = [ @@ -10295,7 +11044,7 @@ name = "types-boto3-s3" version = "1.43.14" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3c/79/ddd397734d7c6368492447c95be54e76158e7dc0d4e616117bf2b2430af0/types_boto3_s3-1.43.14.tar.gz", hash = "sha256:50d1fc0082f07be097184cf647e2dec6101fd1f8378a6c353100ccd067b95e4d", size = 76899, upload-time = "2026-05-22T20:48:17.311Z" } wheels = [ @@ -10307,7 +11056,7 @@ name = "types-boto3-sagemaker-runtime" version = "1.43.29" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/99/57/cc95a58135f2e1ec7af94e4b29f79ad6bdb6a44da9e4983c8e545f4693c1/types_boto3_sagemaker_runtime-1.43.29.tar.gz", hash = "sha256:a7efd7828f52f2d6b2656ea2d99eb1de56b846304ac7ad1b5d603770ad27b789", size = 15771, upload-time = "2026-06-12T20:09:00.99Z" } wheels = [ @@ -10319,7 +11068,7 @@ name = "types-boto3-sts" version = "1.43.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4d/a7/ea448e34f9b519b68505df256e8cc185d60ef8aeb41552553f66da5a7b35/types_boto3_sts-1.43.0.tar.gz", hash = "sha256:d8e0061fed51bb246bd966b9968104bc44411450faa8848f26170bf271913ab1", size = 16823, upload-time = "2026-04-29T23:07:24.448Z" } wheels = [ @@ -10407,18 +11156,38 @@ wheels = [ name = "typing-extensions" version = "4.15.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.14.*'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", + "python_full_version < '3.11'", +] sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, ] +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15'", +] +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + [[package]] name = "typing-inspect" version = "0.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mypy-extensions" }, - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/dc/74/1789779d91f1961fa9438e9a8710cdae6bd138c80d7303996933d117264a/typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78", size = 13825, upload-time = "2023-05-24T20:25:47.612Z" } wheels = [ @@ -10429,14 +11198,36 @@ wheels = [ name = "typing-inspection" version = "0.4.2" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.14.*'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", + "python_full_version < '3.11'", +] dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] +[[package]] +name = "typing-inspection" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15'", +] +dependencies = [ + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, +] + [[package]] name = "tzdata" version = "2026.2" @@ -10596,7 +11387,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "h11" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a2/65/b7c6c443ccc58678c91e1e973bbe2a878591538655d6e1d47f24ba1c51f3/uvicorn-0.51.0.tar.gz", hash = "sha256:f6f4b69b657c312f516dd2d268ab9ae6f254b11e4bac504f37b2ab58b24dd0b0", size = 94412, upload-time = "2026-07-08T10:59:05.962Z" } wheels = [ @@ -11165,6 +11956,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b0/aa/157b940428da80c104ca09666a740e51c94963df65d5b112e06b52e4d7a8/yarl-1.25.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65b5b2066651b7432d389e9799d979c703bcc6ef44266bb8153ef54e91e4aab3", size = 115822, upload-time = "2026-09-15T19:33:26.886Z" }, { url = "https://files.pythonhosted.org/packages/7e/af/19fbdce41412e1b96825544cc52cd7029d3724655d0988237972f078bd29/yarl-1.25.1-cp314-cp314t-win_amd64.whl", hash = "sha256:734f6e5400352ac4254456003d462866c684703570929cff7a7bde015d0cb371", size = 107386, upload-time = "2026-09-15T19:33:29.009Z" }, { url = "https://files.pythonhosted.org/packages/2a/99/f6431c8968e89be608d74b28ae2d024521b2953f27dd44e0dece5e04f67a/yarl-1.25.1-cp314-cp314t-win_arm64.whl", hash = "sha256:287e99ff5aa4dc1c7630bfc683ded6f106d756c99dec432a2d7f197a784f51c6", size = 102094, upload-time = "2026-09-15T19:33:31.151Z" }, + { url = "https://files.pythonhosted.org/packages/c7/3b/4f51eab40c2eabea6c3d5b121dff4b8988dc35087732ffede12d2be8b8dd/yarl-1.25.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:9b1bdaae98bc016825dd3c9d8ee1832f829b3341f9cc6ebd1a1b0a7fef7367cc", size = 143875, upload-time = "2026-09-15T19:33:33.52Z" }, + { url = "https://files.pythonhosted.org/packages/41/05/bbd58fc063f5f299a883f810760b265ca26c8167c91cb9a494d0fe2387e1/yarl-1.25.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:e7011b8fb8c4054bf0c12e5edc6cd83778b0028e99ce59b18586ed036f92cfdc", size = 104028, upload-time = "2026-09-15T19:33:36.22Z" }, + { url = "https://files.pythonhosted.org/packages/12/ee/2fba0aecb52e7020e189f684148783aa0b9cfa3b3bfb0b400646eef70ad4/yarl-1.25.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:f074e8d4aa0a5798920ddb6de3d08b228c614ff3724c3e8bd7577f4bafea867b", size = 104041, upload-time = "2026-09-15T19:33:38.86Z" }, + { url = "https://files.pythonhosted.org/packages/38/35/884beab53ed88c7247d1671972b5ef116f7351fe0c7e6de8c3558372cb16/yarl-1.25.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7d42e7e3ca399555578b4d617e3a6ecf13371b3743a115995fa010c7bf341459", size = 116018, upload-time = "2026-09-15T19:33:44.265Z" }, + { url = "https://files.pythonhosted.org/packages/e2/cc/1a91b685afb55cc18608443ace95280e96e263a97565811732b6788d3269/yarl-1.25.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:aa4ed3dd308548f9e707d9caaf005d2d7f8c1e7868f858dfeb47fe76e16b391d", size = 107033, upload-time = "2026-09-15T19:33:46.45Z" }, + { url = "https://files.pythonhosted.org/packages/c5/c1/58b379fcb1d68d907b7fcf75200c44321896509b2a6a74abbb4b19d864d2/yarl-1.25.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:42a66563d8cc056ee32e6191e05097a7b2b3bc302e0bc3133daf8710eb18bd26", size = 123257, upload-time = "2026-09-15T19:33:48.631Z" }, + { url = "https://files.pythonhosted.org/packages/d7/2d/1fe96cf5c2aeab10095e48f38585cf5a8451fb7253234822398e52aa5336/yarl-1.25.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:98d370568f393215d605304cdb77b3d5539bd192c75b623c7304c42c8d6d8273", size = 126745, upload-time = "2026-09-15T19:33:50.999Z" }, + { url = "https://files.pythonhosted.org/packages/ad/60/8674394ce43f4dadae573a1d6f451716438e9eab7d7fe8d643c673a32d85/yarl-1.25.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23bf5b403c879a54964e0feac7285688e04bb220074878d737d331522da0a5bf", size = 117255, upload-time = "2026-09-15T19:33:53.456Z" }, + { url = "https://files.pythonhosted.org/packages/2f/72/0faa30e02605d56127d42bb987dcc97da3863b7bf70b9bfbf5f739c05e30/yarl-1.25.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d45673badd08456d0340e9364eddafe1c53a9d2896424294de4d7dd71ad3ee57", size = 115166, upload-time = "2026-09-15T19:33:55.669Z" }, + { url = "https://files.pythonhosted.org/packages/8c/90/9a46eac564c437e128285c5c1d7bb385d268394e9209d84f6bf4a14471ef/yarl-1.25.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:0136d640dfa9b0523853e411430a99f8a91eca85774c6420285a33b755bc6de3", size = 116082, upload-time = "2026-09-15T19:33:57.78Z" }, + { url = "https://files.pythonhosted.org/packages/ad/38/4b1a686a3758878f93d2f1ea943f5a165f3555769cd16e761cfd0efdca17/yarl-1.25.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:59ba3a6e1aa8cfe5adf4bd270fd965db21955401b7ca6f1696010c55ed4daec2", size = 108048, upload-time = "2026-09-15T19:34:00.25Z" }, + { url = "https://files.pythonhosted.org/packages/da/14/f348eb967f31a58348612a2b93bd8a2ab664548e2b5e879cac7f592f7201/yarl-1.25.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:87796fedc3ba97ec14fab55acb48584276e6c1e4c1e89c422bda62c838e754a9", size = 122775, upload-time = "2026-09-15T19:34:02.455Z" }, + { url = "https://files.pythonhosted.org/packages/ab/e9/4f7b79700f88cb9e8bb66f8b54f9bce1844c013a2c39fdc47112e9334c95/yarl-1.25.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:bd0912757081f89b107d6c00b2ff8a194401b0b87eadcf4481de2b865a8fd44f", size = 115096, upload-time = "2026-09-15T19:34:05.283Z" }, + { url = "https://files.pythonhosted.org/packages/cf/37/f9cb020331997d3eb887bd28d5410ecfd3d80bf163c23d7cec490d78dade/yarl-1.25.1-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:b51c159a9794633f5e0db7ecec7b2b6e3734eca1f5d17dc989ff3552a43ff78b", size = 120655, upload-time = "2026-09-15T19:34:07.382Z" }, + { url = "https://files.pythonhosted.org/packages/12/83/52fceb22891a41f168db7ec22fd1d81e06b6a0b8d9f70921bd3e785defd0/yarl-1.25.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:319e070a01db9920fb63761843f96a104c8e2b9427266731810dc1e22595b17c", size = 117488, upload-time = "2026-09-15T19:34:09.988Z" }, + { url = "https://files.pythonhosted.org/packages/f5/5c/ce6c4ff1247fcbe4b33d462c23a097106d909b173fde7042bc52290466e2/yarl-1.25.1-cp315-cp315-win_amd64.whl", hash = "sha256:a2059a2d891bd156bc5184e7ab7a56e78a84dfcfdeac8c501b552533ad1c36ee", size = 103434, upload-time = "2026-09-15T19:34:12.56Z" }, + { url = "https://files.pythonhosted.org/packages/3c/a1/766a906b0704fb26d52b19dc22bed48a8ba0544203b70dcf44da350e8194/yarl-1.25.1-cp315-cp315-win_arm64.whl", hash = "sha256:a78b50b4f7918a3de71105d5c0b93bbc57bb8339a4d03a9dfd449f9068e76f3d", size = 99155, upload-time = "2026-09-15T19:34:15.132Z" }, + { url = "https://files.pythonhosted.org/packages/bd/d3/a1d09b32cb6ab14f66b44939f5b4255b8b9e747aef3974af1d5d80ccc2fd/yarl-1.25.1-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:b5402a340723fa7da00b5cff987ddab61276be6d11251ea71ae02bcac54890d8", size = 149284, upload-time = "2026-09-15T19:34:17.452Z" }, + { url = "https://files.pythonhosted.org/packages/7f/3b/fe554d879692650bca70bfbc0df124e82e4d2bb7456f698c7756f1279a96/yarl-1.25.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:eda19ea5ee88742f47a2340816e6f2d40b53bed3ab5b69794769f36af9f35bb4", size = 106399, upload-time = "2026-09-15T19:34:21.474Z" }, + { url = "https://files.pythonhosted.org/packages/e1/4b/e7af56177ac8d40094c82d7728224c0b8472157d50d362e5fb3b014b2bc8/yarl-1.25.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:75baa6cf9b6d1c52f3e111a130e202fd8cf0a5b3a066c3f73d615e885092e4ec", size = 106968, upload-time = "2026-09-15T19:34:23.619Z" }, + { url = "https://files.pythonhosted.org/packages/da/4f/2df41fd738d46f23ef829ae8b4468d94bb6070038fc6dfab6165ed44fea8/yarl-1.25.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbcef5a9119ef653653132cccaf999b30a0af6f33bb0a4ba80bec30056868487", size = 114717, upload-time = "2026-09-15T19:34:25.879Z" }, + { url = "https://files.pythonhosted.org/packages/69/ea/002b66df53bbd1aed1c23358ff99c9bdc744fe3f4d2740e1b2fdd7192885/yarl-1.25.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7efc9f082dfed77c316edffa9deb52888e1bc6789171887cc1f68e06d65465c8", size = 105198, upload-time = "2026-09-15T19:34:28.204Z" }, + { url = "https://files.pythonhosted.org/packages/b1/7c/95c8bc0c8f97d71e59c94525ad60d76f5c57d3f2820f08137ca8b9f0542a/yarl-1.25.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe01645169a2112aa1d4ebc3e4c5f029c5c8f97adfc32e5d37c993b39a994d75", size = 120271, upload-time = "2026-09-15T19:34:30.5Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7a/6fe9da56ec77927baa669fd86c39c567ce6205bab53d581082c6744c8ae7/yarl-1.25.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1c557dfd5e3db046053a0bdc72261ade790ebe8e2c7a41b36b0ca1f14cb95f3", size = 123572, upload-time = "2026-09-15T19:34:32.73Z" }, + { url = "https://files.pythonhosted.org/packages/8b/83/35f222d17fa70a14c7c74fdf112ccf5515e0c2a87082b1f9b99f7693bf57/yarl-1.25.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ce4d6ccafb33d39bd78444612d14938ead674c25702ded2ee9c54a47735d225", size = 115228, upload-time = "2026-09-15T19:34:35.344Z" }, + { url = "https://files.pythonhosted.org/packages/da/6f/fbaaf619423578a7d898d0f226ae47bc1906c293865c416d83c17b828b0e/yarl-1.25.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:80a063f8297fc796296f00f100be520f209b23dc98f93ce8eba6ee7122598209", size = 111618, upload-time = "2026-09-15T19:34:37.656Z" }, + { url = "https://files.pythonhosted.org/packages/ba/78/7383278f1b3cf8e0496bd95b3281a7b09b89217b6b428db24c6b99b3deca/yarl-1.25.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:7cb414a73e21a7ab58254926073f2930cb22f5b4314ea4260a687e2b3fd4dce3", size = 114839, upload-time = "2026-09-15T19:34:40.099Z" }, + { url = "https://files.pythonhosted.org/packages/62/49/5506e5b6d29aab91bd845cc9016d88c3d3f81b81bc242b8100bdd5737825/yarl-1.25.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:85a18376073f8a39aa07be34f9fc77e2869aa72c55c441efdd2cf79a0407504d", size = 106212, upload-time = "2026-09-15T19:34:42.768Z" }, + { url = "https://files.pythonhosted.org/packages/46/8a/19877c193b7c5929f4b07118c18bbe390f3fadd0f59dd98c0cd12b31fa5c/yarl-1.25.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:77716e245c90f058466a05e6a465bb8600f767a8f4b18b4d40f3aff958e5f73c", size = 119985, upload-time = "2026-09-15T19:34:44.976Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6c/0a46fbbf9ecbcbd0cc20d2193394254b9e19817f22c814aab60f99847400/yarl-1.25.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:1e80dcf1446e1b080b1932b0d103c464a04112f5bc31f0f983ad418172063cde", size = 112081, upload-time = "2026-09-15T19:34:47.45Z" }, + { url = "https://files.pythonhosted.org/packages/ef/30/93f5d471230c74ccd06255d0842739f86551f937f9e63a5e947853c6244a/yarl-1.25.1-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:bdc8d8b8c22e9e43ac68316b5e6cf083dec537f4ec213cb4aa967b583bc3fa64", size = 116995, upload-time = "2026-09-15T19:34:49.972Z" }, + { url = "https://files.pythonhosted.org/packages/2b/80/c386593035ee3f9c6c6af0847b5578f2830c674794a9d7701b744a3ebd42/yarl-1.25.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:dfbf531053a0935f2e871bcd4753f90313688772ff8c017f5ea402e315a78c1f", size = 115570, upload-time = "2026-09-15T19:34:52.628Z" }, + { url = "https://files.pythonhosted.org/packages/38/02/eef443559563ef8f2e10469387b8b1e97cb5efee95b288a56da60801f7ee/yarl-1.25.1-cp315-cp315t-win_amd64.whl", hash = "sha256:b13b88747769537f3d32e89e3a735da10c0a9e35d7322928c701b5f93d3afffd", size = 106811, upload-time = "2026-09-15T19:34:54.935Z" }, + { url = "https://files.pythonhosted.org/packages/88/91/41e284ca2cf5211e05dae031d126a3668aea88fa759df56e7e35c6ad25ba/yarl-1.25.1-cp315-cp315t-win_arm64.whl", hash = "sha256:783dd1467083f4d3f7722ad6a313f24c173e7571372738fcb7a6e6d1ba48df25", size = 101804, upload-time = "2026-09-15T19:34:57.231Z" }, { url = "https://files.pythonhosted.org/packages/54/22/318c7980066769c6bcd9221ed2248294f5698811da099013098c670565ed/yarl-1.25.1-py3-none-any.whl", hash = "sha256:681c758b0490f9e96b78e5fa8e8dc6e648e9185bb6eaebe73183c33ea0c445f3", size = 63617, upload-time = "2026-09-15T19:34:59.616Z" }, ] From e0982d62485bc0bbec67de3ebe58c16c465fd7c2 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 26 Oct 2026 10:10:57 -0700 Subject: [PATCH 049/155] build(deps): bump webpki-roots and webpki-root-certs to 1.0.11 (#48301) 1.0.11 drops emSign Root CA - C1 and emSign ECC Root CA - C3 from the built-in roots --- litellm-rust/Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index c868d3e4b8a..cb50324e1dd 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -8202,18 +8202,18 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" +checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" dependencies = [ "rustls-pki-types", ] [[package]] name = "webpki-roots" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" dependencies = [ "rustls-pki-types", ] From 96792d09138deecd5bc9eb5a746e040ea6f49beb Mon Sep 17 00:00:00 2001 From: Arsh Goyal Date: Mon, 26 Oct 2026 23:33:37 +0530 Subject: [PATCH 050/155] test(streaming): pin that a stream stuck on one chunk closes its provider stream and logs one failure (#38753) * fix(streaming): stop re-raising and re-logging a failed stream on every next() A stream that raised keeps its provider iterator alive, so a consumer that iterates past the error re-detected, re-raised and re-logged the same failure on every call, flooding failure callbacks for one request. Latch the failure and re-raise it without logging again. * fix(streaming): drop the retained traceback before re-raising a failed stream * test(streaming): count failures through a real logging callback and require a clean stop on later reads * test(streaming): require the stuck stream to close early and release its socket The provider stream now records a GeneratorExit and caps its chunks, so draining it or detecting the loop late fails. Two wire cases check a sync and an async stuck stream release their TCP session. * test(streaming): avoid failure recorder name collision --------- Co-authored-by: Devin AI Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: moe-berri --- ...est_sync_stream_connection_release_wire.py | 33 +++++ .../test_streaming_handler.py | 118 ++++++++++++++++++ 2 files changed, 151 insertions(+) diff --git a/tests/integration/sdk/test_sync_stream_connection_release_wire.py b/tests/integration/sdk/test_sync_stream_connection_release_wire.py index a8b49d285c3..21774df66d0 100644 --- a/tests/integration/sdk/test_sync_stream_connection_release_wire.py +++ b/tests/integration/sdk/test_sync_stream_connection_release_wire.py @@ -212,3 +212,36 @@ def test_sync_stream_that_raises_mid_stream_releases_the_connection() -> None: assert _open_sessions_settling_to(wire, at_most=0) == 0, ( "a stream that raised mid-way left its TCP session open" ) + + +_STUCK: Final = _Shape( + "hosted_vllm", + "hosted_vllm/stuck-model", + _OPENAI_OPENING, + (_openai_delta("the model is stuck on this", None),) * (10 * litellm.REPEATED_STREAMING_CHUNK_LIMIT), +) + + +def _still_generating_peer(request: Request) -> Reply: + return Reply(content_type="text/event-stream", chunks=(_STUCK.opening, *_STUCK.closing), pause_between_chunks=0.01) + + +def test_sync_stream_stuck_repeating_one_chunk_releases_the_connection() -> None: + with wire_server(_still_generating_peer, keep_alive=True) as wire: + stuck: Final = litellm.completion(**_request_kwargs(_STUCK, wire)) + with pytest.raises(Exception, match="repeating the same chunk"): + for _ in stuck: + pass + assert _open_sessions_settling_to(wire, at_most=0) == 0, "a stream stuck on one chunk left its TCP session open" + + +@pytest.mark.asyncio +async def test_async_stream_stuck_repeating_one_chunk_releases_the_connection() -> None: + with wire_server(_still_generating_peer, keep_alive=True) as wire: + stuck: Final = await litellm.acompletion(**_request_kwargs(_STUCK, wire)) + with pytest.raises(Exception, match="repeating the same chunk"): + async for _ in stuck: + pass + assert await _open_sessions_settling_to_while_the_loop_runs(wire, at_most=0) == 0, ( + "a stream stuck on one chunk left its TCP session open" + ) diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index c13cea51183..cc1fa1b8803 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -1,6 +1,7 @@ import asyncio import datetime import json +import queue import sys import threading import time @@ -2819,6 +2820,123 @@ def test_raise_on_model_repetition_tolerates_empty_choices( wrapper.raise_on_model_repetition() +_STUCK_CHUNK: Final = "the model is stuck on this" +_STUCK_PROVIDER_CHUNK_CAP: Final = 10 * litellm.REPEATED_STREAMING_CHUNK_LIMIT + + +class _StuckProvider: + def __init__(self) -> None: + self.closed_early = threading.Event() + self.chunks_sent = 0 + + def _next_chunk(self) -> ModelResponseStream: + self.chunks_sent += 1 + return _make_chunk(_STUCK_CHUNK) + + def sync_stream(self) -> Iterator[ModelResponseStream]: + try: + yield from (self._next_chunk() for _ in range(_STUCK_PROVIDER_CHUNK_CAP)) + except GeneratorExit: + self.closed_early.set() + raise + + async def async_stream(self) -> AsyncIterator[ModelResponseStream]: + try: + for _ in range(_STUCK_PROVIDER_CHUNK_CAP): + yield self._next_chunk() + except GeneratorExit: + self.closed_early.set() + raise + + +class _StuckStreamFailureRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.failures: Final = queue.SimpleQueue[str]() + + def log_failure_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None: + self.failures.put(str(cast(Mapping[str, object], kwargs)["exception"])) + + async def async_log_failure_event( + self, kwargs: object, response_obj: object, start_time: object, end_time: object + ) -> None: + self.failures.put(str(cast(Mapping[str, object], kwargs)["exception"])) + + +def _stuck_stream_wrapper( + provider_stream: Iterator[ModelResponseStream] | AsyncIterator[ModelResponseStream], + call_type: Literal["completion", "acompletion"], + recorder: _StuckStreamFailureRecorder, +) -> CustomStreamWrapper: + logging_obj: Final = Logging( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type=call_type, + start_time=datetime.datetime(2026, 1, 1), + litellm_call_id=f"stuck-stream-{call_type}", + function_id=f"stuck-stream-{call_type}", + dynamic_failure_callbacks=[recorder], + dynamic_async_failure_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model="test-model", + optional_params={}, + litellm_params=get_litellm_params(custom_llm_provider="openai", acompletion=call_type == "acompletion"), + ) + return CustomStreamWrapper( + completion_stream=provider_stream, + model="test-model", + logging_obj=logging_obj, + custom_llm_provider="openai", + ) + + +@pytest.mark.asyncio +async def test_async_stream_stuck_on_one_chunk_closes_the_provider_stream_and_logs_one_failure(): + """https://github.com/BerriAI/litellm/issues/19428 and https://github.com/BerriAI/litellm/issues/13786""" + provider: Final = _StuckProvider() + recorder: Final = _StuckStreamFailureRecorder() + wrapper: Final = _stuck_stream_wrapper(provider.async_stream(), "acompletion", recorder) + + with pytest.raises(Exception, match="repeating the same chunk"): + async for _ in wrapper: + pass + assert provider.closed_early.is_set() + assert provider.chunks_sent <= litellm.REPEATED_STREAMING_CHUNK_LIMIT + 1 + chunks_sent_at_failure: Final = provider.chunks_sent + + with pytest.raises(StopAsyncIteration): + await wrapper.__anext__() + await wrapper.wait_for_failure_logging(timeout=5) + + assert provider.chunks_sent == chunks_sent_at_failure + assert "repeating the same chunk" in recorder.failures.get_nowait() + assert recorder.failures.empty() + + +def test_sync_stream_stuck_on_one_chunk_closes_the_provider_stream_and_logs_one_failure(): + """Sync counterpart of the async test above""" + provider: Final = _StuckProvider() + recorder: Final = _StuckStreamFailureRecorder() + wrapper: Final = _stuck_stream_wrapper(provider.sync_stream(), "completion", recorder) + + with pytest.raises(Exception, match="repeating the same chunk"): + for _ in wrapper: + pass + assert provider.closed_early.is_set() + assert provider.chunks_sent <= litellm.REPEATED_STREAMING_CHUNK_LIMIT + 1 + chunks_sent_at_failure: Final = provider.chunks_sent + assert "repeating the same chunk" in recorder.failures.get(timeout=5) + + with pytest.raises(StopIteration): + next(wrapper) + + assert provider.chunks_sent == chunks_sent_at_failure + with pytest.raises(queue.Empty): + recorder.failures.get(timeout=0.5) + + def test_usage_chunk_after_finish_reason_updates_hidden_params(logging_obj): """ Test that provider-reported usage from a post-finish_reason chunk From be3a7be1db6c4ee38d0d61957a2ae5179446c874 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 26 Oct 2026 14:12:11 -0400 Subject: [PATCH 051/155] fix(ui): restore clipboard copying in key and log details (#48425) --- ui/litellm-dashboard/eslint-suppressions.json | 5 - .../AuditLogDrawer/AuditLogDrawer.test.tsx | 23 +- .../audit/AuditLogDrawer/AuditLogDrawer.tsx | 19 +- .../logs/detail/DrawerHeader.test.tsx | 25 ++ .../components/logs/detail/DrawerHeader.tsx | 6 +- .../components/logs/detail/InputCard.test.tsx | 42 +-- .../src/components/logs/detail/InputCard.tsx | 5 +- .../LogDetailContent.integration.test.tsx | 41 +++ .../logs/detail/LogDetailContent.tsx | 6 +- .../logs/detail/LogDetailsDrawer.test.tsx | 22 ++ .../logs/detail/LogDetailsDrawer.tsx | 6 +- .../logs/detail/OutputCard.test.tsx | 35 +-- .../src/components/logs/detail/OutputCard.tsx | 5 +- .../logs/detail/RealtimePrettyView.test.tsx | 22 +- .../logs/detail/RealtimePrettyView.tsx | 3 +- .../src/components/shared/CopyButton.test.tsx | 55 +--- .../src/components/shared/CopyButton.tsx | 12 +- .../src/utils/dataUtils.test.ts | 277 +++++++----------- ui/litellm-dashboard/src/utils/dataUtils.ts | 23 +- 19 files changed, 287 insertions(+), 345 deletions(-) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 38d4c435e9a..32777b43504 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -2175,11 +2175,6 @@ "count": 1 } }, - "src/utils/dataUtils.test.ts": { - "max-nested-callbacks": { - "count": 1 - } - }, "tests/CreateKeyPage.expiredToken.test.tsx": { "@typescript-eslint/no-require-imports": { "count": 3 diff --git a/ui/litellm-dashboard/src/components/logs/audit/AuditLogDrawer/AuditLogDrawer.test.tsx b/ui/litellm-dashboard/src/components/logs/audit/AuditLogDrawer/AuditLogDrawer.test.tsx index da158fb8450..8ac99760c01 100644 --- a/ui/litellm-dashboard/src/components/logs/audit/AuditLogDrawer/AuditLogDrawer.test.tsx +++ b/ui/litellm-dashboard/src/components/logs/audit/AuditLogDrawer/AuditLogDrawer.test.tsx @@ -1,10 +1,16 @@ +import { copyToClipboard } from "@/utils/dataUtils"; import { describe, it, expect, vi, beforeEach } from "vitest"; import userEvent from "@testing-library/user-event"; -import { render, screen, waitFor, within } from "@testing-library/react"; +import { render, screen, within } from "@testing-library/react"; import moment from "moment"; import { AuditLogDrawer } from "./AuditLogDrawer"; import { AuditLogEntry } from "../AuditLogsTableColumns"; +vi.mock("@/utils/dataUtils", async (importOriginal) => ({ + ...(await importOriginal()), + copyToClipboard: vi.fn(), +})); + vi.mock("../../../common_components/DefaultProxyAdminTag", () => ({ default: ({ userId }: { userId: string }) => {userId}, })); @@ -120,16 +126,17 @@ describe("AuditLogDrawer", () => { expect(screen.queryByText(/"spend"/)).not.toBeInTheDocument(); }); - it("should copy the json of a block to the clipboard", async () => { + it.each([true, false])("confirms copied JSON only when copying succeeds (%s)", async (success) => { const user = userEvent.setup(); - const writeText = vi.fn().mockResolvedValue(undefined); - Object.defineProperty(navigator, "clipboard", { value: { writeText }, configurable: true }); - Object.defineProperty(window, "isSecureContext", { value: true, configurable: true }); - + Object.defineProperty(navigator, "clipboard", { value: undefined, configurable: true }); + vi.mocked(copyToClipboard).mockResolvedValue(success); render(); - await user.click(within(blockNamed("Before")).getByTitle("Copy JSON")); - await waitFor(() => expect(writeText).toHaveBeenCalledWith(JSON.stringify({ max_budget: 10 }, null, 2))); + const button = within(blockNamed("Before")).getByRole("button", { name: "Copy JSON" }); + await user.click(button); + + expect(copyToClipboard).toHaveBeenCalledWith(JSON.stringify({ max_budget: 10 }, null, 2)); + expect(button.querySelector(".lucide-check") !== null).toBe(success); }); describe("width", () => { diff --git a/ui/litellm-dashboard/src/components/logs/audit/AuditLogDrawer/AuditLogDrawer.tsx b/ui/litellm-dashboard/src/components/logs/audit/AuditLogDrawer/AuditLogDrawer.tsx index fae3c35b80c..eaddf1e507f 100644 --- a/ui/litellm-dashboard/src/components/logs/audit/AuditLogDrawer/AuditLogDrawer.tsx +++ b/ui/litellm-dashboard/src/components/logs/audit/AuditLogDrawer/AuditLogDrawer.tsx @@ -1,3 +1,4 @@ +import { copyToClipboard } from "@/utils/dataUtils"; import { Check, Copy } from "lucide-react"; import { useState, useCallback } from "react"; import moment from "moment"; @@ -27,25 +28,9 @@ function CopyableJsonBlock({ label, value }: { label: string; value: Record { - try { - const text = JSON.stringify(value, null, 2); - if (navigator.clipboard && window.isSecureContext) { - await navigator.clipboard.writeText(text); - } else { - const el = document.createElement("textarea"); - el.value = text; - el.style.position = "fixed"; - el.style.opacity = "0"; - document.body.appendChild(el); - el.focus(); - el.select(); - document.execCommand("copy"); - document.body.removeChild(el); - } + if (await copyToClipboard(JSON.stringify(value, null, 2))) { setCopied(true); setTimeout(() => setCopied(false), 2000); - } catch (e) { - console.error("Copy failed:", e); } }, [value]); diff --git a/ui/litellm-dashboard/src/components/logs/detail/DrawerHeader.test.tsx b/ui/litellm-dashboard/src/components/logs/detail/DrawerHeader.test.tsx index 564862acccd..b24667c6a2e 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/DrawerHeader.test.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/DrawerHeader.test.tsx @@ -6,6 +6,12 @@ import { describe, expect, it, vi } from "vitest"; import { render } from "../../../../tests/test-utils"; import type { LogEntry } from "../types"; import { DrawerHeader } from "./DrawerHeader"; +import { copyToClipboard } from "@/utils/dataUtils"; + +vi.mock("@/utils/dataUtils", async (importOriginal) => ({ + ...(await importOriginal()), + copyToClipboard: vi.fn(), +})); const logEntry = (overrides: Partial): LogEntry => ({ @@ -118,3 +124,22 @@ describe("DrawerHeader back link", () => { expect(screen.queryByRole("button", { name: /^Back to/ })).not.toBeInTheDocument(); }); }); + +describe("DrawerHeader copying", () => { + it.each([true, false])("confirms both ids only when copying succeeds (%s)", async (success) => { + const user = userEvent.setup(); + Object.defineProperty(navigator, "clipboard", { value: undefined, configurable: true }); + vi.mocked(copyToClipboard).mockReset().mockResolvedValue(success); + renderHeader(logEntry({ request_id: "request-copy", litellm_call_id: "call-copy" }), false); + + const requestButton = screen.getByRole("button", { name: "Copy Request ID" }); + const callButton = screen.getByRole("button", { name: "Copy x-litellm-call-id" }); + await user.click(requestButton); + await user.click(callButton); + + expect(copyToClipboard).toHaveBeenNthCalledWith(1, "request-copy"); + expect(copyToClipboard).toHaveBeenNthCalledWith(2, "call-copy"); + expect(requestButton).toHaveAccessibleName(success ? "Copied!" : "Copy Request ID"); + expect(callButton).toHaveAccessibleName(success ? "Copied!" : "Copy x-litellm-call-id"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/logs/detail/DrawerHeader.tsx b/ui/litellm-dashboard/src/components/logs/detail/DrawerHeader.tsx index 8ec615d444d..312a0797f1d 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/DrawerHeader.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/DrawerHeader.tsx @@ -1,3 +1,4 @@ +import { copyToClipboard } from "@/utils/dataUtils"; import { type ReactNode, useState } from "react"; import { ArrowLeft, Check, ChevronDown, ChevronUp, Copy, X } from "lucide-react"; import moment from "moment"; @@ -183,12 +184,9 @@ function CopyableId({ const [copied, setCopied] = useState(false); const handleCopy = async () => { - try { - await navigator.clipboard.writeText(value); + if (await copyToClipboard(value)) { setCopied(true); setTimeout(() => setCopied(false), 1200); - } catch { - /* clipboard unavailable in non-secure contexts */ } }; diff --git a/ui/litellm-dashboard/src/components/logs/detail/InputCard.test.tsx b/ui/litellm-dashboard/src/components/logs/detail/InputCard.test.tsx index dfd9dd54dd7..87105d3f48f 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/InputCard.test.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/InputCard.test.tsx @@ -1,12 +1,15 @@ import React from "react"; -import { render, screen, waitFor, fireEvent, act } from "@testing-library/react"; +import { render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { describe, it, expect, vi, beforeEach } from "vitest"; import { InputCard } from "./InputCard"; import { ParsedMessage } from "./prettyMessagesTypes"; +import { copyToClipboard } from "@/utils/dataUtils"; +import { toast } from "@/lib/toast"; + +vi.mock("@/utils/dataUtils", () => ({ copyToClipboard: vi.fn() })); describe("InputCard", () => { - const mockWriteText = vi.fn().mockResolvedValue(undefined); const mockMessages: ParsedMessage[] = [ { role: "user", @@ -16,13 +19,7 @@ describe("InputCard", () => { beforeEach(() => { vi.clearAllMocks(); - Object.defineProperty(navigator, "clipboard", { - value: { - writeText: mockWriteText, - }, - writable: true, - configurable: true, - }); + vi.mocked(copyToClipboard).mockResolvedValue(true); }); it("should render the InputCard component", () => { @@ -91,27 +88,16 @@ describe("InputCard", () => { expect(screen.getByText(/Cost: \$0\.001500/)).toBeInTheDocument(); }); - it("should copy last message content when copy button is clicked", async () => { - const messages: ParsedMessage[] = [ - { - role: "user", - content: "Copy this text", - }, - ]; + it.each([true, false])("delegates copying and feedback to the helper (%s)", async (success) => { + const user = userEvent.setup(); + Object.defineProperty(navigator, "clipboard", { value: undefined, configurable: true }); + vi.mocked(copyToClipboard).mockResolvedValue(success); + render(); - render(); + await user.click(screen.getByRole("button", { name: /copy/i })); - const copyButton = screen.getByRole("button", { name: /copy/i }); - - expect(copyButton).toBeInTheDocument(); - - await act(async () => { - fireEvent.click(copyButton); - }); - - await waitFor(() => { - expect(mockWriteText).toHaveBeenCalledWith("Copy this text"); - }); + expect(copyToClipboard).toHaveBeenCalledWith("Copy this text", "Input copied"); + expect(toast.success).not.toHaveBeenCalled(); }); it("should toggle collapse state when header is clicked", async () => { diff --git a/ui/litellm-dashboard/src/components/logs/detail/InputCard.tsx b/ui/litellm-dashboard/src/components/logs/detail/InputCard.tsx index d367c69f263..0b0bde85b91 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/InputCard.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/InputCard.tsx @@ -4,7 +4,7 @@ */ import { useState } from "react"; -import { toast } from "@/lib/toast"; +import { copyToClipboard } from "@/utils/dataUtils"; import { ParsedMessage } from "./prettyMessagesTypes"; import { SectionHeader } from "./SectionHeader"; import { CollapsibleMessage } from "./CollapsibleMessage"; @@ -32,8 +32,7 @@ export function InputCard({ messages, promptTokens, inputCost }: InputCardProps) const handleCopy = () => { const content = lastMessage?.content || ""; - navigator.clipboard.writeText(content); - toast.success("Input copied"); + void copyToClipboard(content, "Input copied"); }; return ( diff --git a/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx b/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx index 7f39a0d8fe5..21ecf19235e 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx @@ -3,6 +3,7 @@ import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi } from "vitest"; import { GuardrailJumpLink, LogDetailContent } from "./LogDetailContent"; import type { LogEntry } from "../types"; +import { toast } from "@/lib/toast"; vi.mock("./sections/GuardrailViewer/GuardrailViewer", () => ({ default: ({ data }: { data: unknown }) =>
{JSON.stringify(data)}
, @@ -295,6 +296,46 @@ describe("LogDetailContent", () => { expect(screen.queryByRole("tab", { name: "Request" })).not.toBeInTheDocument(); }); + it.each([ + ["Request", true], + ["Response", true], + ["Metadata", true], + ["Request", false], + ] as const)("copies %s through the fallback and reports its outcome (%s)", async (section, success) => { + const user = userEvent.setup(); + const clipboardDescriptor = Object.getOwnPropertyDescriptor(navigator, "clipboard"); + const execCommandDescriptor = Object.getOwnPropertyDescriptor(document, "execCommand"); + const writeText = vi.fn<(value: string) => boolean>().mockReturnValue(success); + Object.defineProperty(navigator, "clipboard", { value: undefined, configurable: true }); + Object.defineProperty(document, "execCommand", { + configurable: true, + value: (command: string) => + command === "copy" && writeText((document.activeElement as HTMLTextAreaElement).value), + }); + vi.mocked(toast.success).mockClear(); + const log = createLogEntry(); + const expected = { Request: log.messages, Response: log.response, Metadata: log.metadata }[section]; + + try { + render(); + if (section !== "Metadata") { + await user.click(screen.getByRole("tab", { name: "JSON", exact: true })); + await user.click(screen.getByRole("tab", { name: section, exact: true })); + } + const label = section === "Metadata" ? "Copy Metadata" : "Copy JSON"; + const button = screen.getByRole("button", { name: label, exact: true }); + await user.click(button); + + expect(writeText).toHaveBeenCalledWith(JSON.stringify(expected, null, 2)); + expect(button).toHaveAccessibleName(success ? "Copied!" : label); + expect(toast.success).toHaveBeenCalledTimes(success ? 1 : 0); + } finally { + if (clipboardDescriptor) Object.defineProperty(navigator, "clipboard", clipboardDescriptor); + if (execCommandDescriptor) Object.defineProperty(document, "execCommand", execCommandDescriptor); + else Reflect.deleteProperty(document, "execCommand"); + } + }); + it.each(["object", "serialized", "messages only", "null captures"])( "preserves request inspection and copying for classifier logs with %s data", async (shape) => { diff --git a/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.tsx index f6c63582938..f740b6c586a 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.tsx @@ -1,3 +1,4 @@ +import { copyToClipboard } from "@/utils/dataUtils"; import { useState } from "react"; import { Check, ChevronDown, ChevronRight, CircleAlert, Copy, Info } from "lucide-react"; import moment from "moment"; @@ -304,12 +305,9 @@ function CopyButton({ const [copied, setCopied] = useState(false); const handleCopy = async () => { - try { - await navigator.clipboard.writeText(getText()); + if (await copyToClipboard(getText())) { setCopied(true); setTimeout(() => setCopied(false), 1200); - } catch { - /* clipboard unavailable in non-secure contexts */ } }; diff --git a/ui/litellm-dashboard/src/components/logs/detail/LogDetailsDrawer.test.tsx b/ui/litellm-dashboard/src/components/logs/detail/LogDetailsDrawer.test.tsx index 1321a4a152a..36ab70ecf2a 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/LogDetailsDrawer.test.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/LogDetailsDrawer.test.tsx @@ -1,3 +1,5 @@ +import userEvent from "@testing-library/user-event"; +import { copyToClipboard } from "@/utils/dataUtils"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { fireEvent, render, screen, waitFor } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; @@ -6,6 +8,11 @@ import { sessionSpendLogsCall } from "../../networking"; import { LogEntry } from "../types"; import { AutoRouterModelGroupsProvider } from "@/components/shared/table_cells"; +vi.mock("@/utils/dataUtils", async (importOriginal) => ({ + ...(await importOriginal()), + copyToClipboard: vi.fn(), +})); + vi.mock("../../networking", () => ({ sessionSpendLogsCall: vi.fn(), })); @@ -201,3 +208,18 @@ describe("LogDetailsDrawer session sidebar auto-router icon", () => { expect(directRow.querySelector(".lucide-waypoints")).toBeNull(); }); }); + +describe("LogDetailsDrawer copying", () => { + it.each([true, false])("confirms a session id only when copying succeeds (%s)", async (success) => { + const user = userEvent.setup(); + Object.defineProperty(navigator, "clipboard", { value: undefined, configurable: true }); + vi.mocked(copyToClipboard).mockReset().mockResolvedValue(success); + renderSessionDrawer(); + + const button = await screen.findByRole("button", { name: "Copy trace id" }); + await user.click(button); + + expect(copyToClipboard).toHaveBeenCalledWith("session-1"); + expect(button.querySelector(".lucide-check") !== null).toBe(success); + }); +}); diff --git a/ui/litellm-dashboard/src/components/logs/detail/LogDetailsDrawer.tsx b/ui/litellm-dashboard/src/components/logs/detail/LogDetailsDrawer.tsx index 48b89ee8ac1..23ad629d572 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/LogDetailsDrawer.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/LogDetailsDrawer.tsx @@ -1,3 +1,4 @@ +import { copyToClipboard } from "@/utils/dataUtils"; import { useEffect, useMemo, useState } from "react"; import { Bot, Check, Copy, Sparkles, Wrench } from "lucide-react"; import { Sheet, SheetContent, SheetTitle } from "@/components/ui/sheet"; @@ -289,12 +290,9 @@ export function LogDetailsDrawer({ const handleCopyLeftPanelId = async () => { if (!leftPanelId) return; - try { - await navigator.clipboard.writeText(leftPanelId); + if (await copyToClipboard(leftPanelId)) { setCopiedLeftPanelId(true); setTimeout(() => setCopiedLeftPanelId(false), 1200); - } catch { - /* clipboard unavailable in non-secure contexts */ } }; diff --git a/ui/litellm-dashboard/src/components/logs/detail/OutputCard.test.tsx b/ui/litellm-dashboard/src/components/logs/detail/OutputCard.test.tsx index 3dda7c5fa65..7b4c1a987a1 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/OutputCard.test.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/OutputCard.test.tsx @@ -1,12 +1,15 @@ import React from "react"; -import { render, screen, waitFor, fireEvent, act } from "@testing-library/react"; +import { render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { describe, it, expect, vi, beforeEach } from "vitest"; import { OutputCard } from "./OutputCard"; import { ParsedMessage } from "./prettyMessagesTypes"; +import { copyToClipboard } from "@/utils/dataUtils"; +import { toast } from "@/lib/toast"; + +vi.mock("@/utils/dataUtils", () => ({ copyToClipboard: vi.fn() })); describe("OutputCard", () => { - const mockWriteText = vi.fn().mockResolvedValue(undefined); const mockMessage: ParsedMessage = { role: "assistant", content: "This is a test response", @@ -14,13 +17,7 @@ describe("OutputCard", () => { beforeEach(() => { vi.clearAllMocks(); - Object.defineProperty(navigator, "clipboard", { - value: { - writeText: mockWriteText, - }, - writable: true, - configurable: true, - }); + vi.mocked(copyToClipboard).mockResolvedValue(true); }); it("should render the OutputCard component", () => { @@ -53,20 +50,16 @@ describe("OutputCard", () => { expect(screen.getByText(/Cost: \$0\.002500/)).toBeInTheDocument(); }); - it("should copy message content when copy button is clicked", async () => { + it.each([true, false])("delegates copying and feedback to the helper (%s)", async (success) => { + const user = userEvent.setup(); + Object.defineProperty(navigator, "clipboard", { value: undefined, configurable: true }); + vi.mocked(copyToClipboard).mockResolvedValue(success); render(); - const copyButton = screen.getByRole("button", { name: /copy/i }); + await user.click(screen.getByRole("button", { name: /copy/i })); - expect(copyButton).toBeInTheDocument(); - - await act(async () => { - fireEvent.click(copyButton); - }); - - await waitFor(() => { - expect(mockWriteText).toHaveBeenCalledWith("This is a test response"); - }); + expect(copyToClipboard).toHaveBeenCalledWith("This is a test response", "Output copied"); + expect(toast.success).not.toHaveBeenCalled(); }); it("should not copy when message is null and copy button is clicked", async () => { @@ -80,7 +73,7 @@ describe("OutputCard", () => { await user.click(copyButton); await waitFor(() => { - expect(mockWriteText).not.toHaveBeenCalled(); + expect(copyToClipboard).not.toHaveBeenCalled(); }); }); diff --git a/ui/litellm-dashboard/src/components/logs/detail/OutputCard.tsx b/ui/litellm-dashboard/src/components/logs/detail/OutputCard.tsx index 7d0ec62ffe2..7daa8b02546 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/OutputCard.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/OutputCard.tsx @@ -4,7 +4,7 @@ */ import { useState } from "react"; -import { toast } from "@/lib/toast"; +import { copyToClipboard } from "@/utils/dataUtils"; import { COLOR_BORDER } from "./constants"; import { ParsedMessage } from "./prettyMessagesTypes"; import { SectionHeader } from "./SectionHeader"; @@ -22,8 +22,7 @@ export function OutputCard({ message, completionTokens, outputCost }: OutputCard const handleCopy = () => { if (!message) return; - navigator.clipboard.writeText(message.content || ""); - toast.success("Output copied"); + void copyToClipboard(message.content || "", "Output copied"); }; return ( diff --git a/ui/litellm-dashboard/src/components/logs/detail/RealtimePrettyView.test.tsx b/ui/litellm-dashboard/src/components/logs/detail/RealtimePrettyView.test.tsx index 201c9e74ce8..d5280c35983 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/RealtimePrettyView.test.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/RealtimePrettyView.test.tsx @@ -1,9 +1,12 @@ import React from "react"; import { render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; +import { copyToClipboard } from "@/utils/dataUtils"; import { describe, it, expect, vi, beforeEach } from "vitest"; import { RealtimePrettyView, isRealtimeResponse } from "./RealtimePrettyView"; +vi.mock("@/utils/dataUtils", () => ({ copyToClipboard: vi.fn().mockResolvedValue(true) })); + const sampleRealtimeResponse = { usage: { total_tokens: 587, @@ -142,15 +145,20 @@ describe("isRealtimeResponse", () => { }); describe("RealtimePrettyView", () => { - const mockWriteText = vi.fn().mockResolvedValue(undefined); - beforeEach(() => { vi.clearAllMocks(); - Object.defineProperty(navigator, "clipboard", { - value: { writeText: mockWriteText }, - writable: true, - configurable: true, - }); + }); + + it("copies all transcript turns when the Clipboard API is unavailable", async () => { + const user = userEvent.setup(); + Object.defineProperty(navigator, "clipboard", { value: undefined, configurable: true }); + render(); + + await user.click(screen.getByRole("button", { name: /copy/i })); + + expect(copyToClipboard).toHaveBeenCalledWith( + "assistant: Hello! How's your day going?\nassistant: I'm here to help with information and general questions.", + ); }); it("should render the component successfully", () => { diff --git a/ui/litellm-dashboard/src/components/logs/detail/RealtimePrettyView.tsx b/ui/litellm-dashboard/src/components/logs/detail/RealtimePrettyView.tsx index 82f0d3e7fb2..04f0a178462 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/RealtimePrettyView.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/RealtimePrettyView.tsx @@ -1,3 +1,4 @@ +import { copyToClipboard } from "@/utils/dataUtils"; /** * RealtimePrettyView - Structured pretty view for OpenAI Realtime API logs * Displays session config, conversation turns, and token usage @@ -276,7 +277,7 @@ function ConversationCard({ (r.output || []).flatMap((o) => (o.content || []).map((c) => `${o.role}: ${c.transcript || c.text || ""}`)), ) .join("\n"); - navigator.clipboard.writeText(transcripts); + void copyToClipboard(transcripts); }; return ( diff --git a/ui/litellm-dashboard/src/components/shared/CopyButton.test.tsx b/ui/litellm-dashboard/src/components/shared/CopyButton.test.tsx index c2c2ee199b4..311b536d89c 100644 --- a/ui/litellm-dashboard/src/components/shared/CopyButton.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/CopyButton.test.tsx @@ -3,7 +3,7 @@ import userEvent from "@testing-library/user-event"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { ComponentProps } from "react"; import { copyToClipboard } from "@/utils/dataUtils"; -import { renderWithProviders, screen, waitFor } from "../../../tests/test-utils"; +import { renderWithProviders, screen } from "../../../tests/test-utils"; import CopyButton from "./CopyButton"; vi.mock("@/utils/dataUtils", () => ({ copyToClipboard: vi.fn() })); @@ -13,13 +13,9 @@ vi.mock("lucide-react", async (importOriginal) => ({ Check: (props: ComponentProps<"svg">) => , })); -const clipboardDescriptor = Object.getOwnPropertyDescriptor(navigator, "clipboard"); - afterEach(() => { vi.useRealTimers(); vi.resetAllMocks(); - if (clipboardDescriptor) Object.defineProperty(navigator, "clipboard", clipboardDescriptor); - else Reflect.deleteProperty(navigator, "clipboard"); }); describe("CopyButton", () => { @@ -28,55 +24,37 @@ describe("CopyButton", () => { expect(screen.queryByRole("button")).not.toBeInTheDocument(); }); - it("shows the confirmation checkmark only after a successful write", async () => { + it("shows the icon confirmation only after copying succeeds", async () => { const user = userEvent.setup(); - const writeText = vi.fn().mockResolvedValue(undefined); - Object.defineProperty(navigator, "clipboard", { value: { writeText }, configurable: true }); + const pendingCopy = Promise.withResolvers(); + vi.mocked(copyToClipboard).mockReturnValue(pendingCopy.promise); + renderWithProviders(); - renderWithProviders(); + await user.click(screen.getByRole("button", { name: "Copy value" })); - const button = screen.getByRole("button", { name: "Copy value" }); + expect(copyToClipboard).toHaveBeenCalledWith("secret-value", "Value copied"); expect(screen.getByRole("img", { name: "Copy icon" })).toBeInTheDocument(); + expect(screen.queryByRole("img", { name: "Copied icon" })).not.toBeInTheDocument(); - await user.click(button); - - expect(writeText).toHaveBeenCalledWith("secret-value"); - expect(await screen.findByRole("img", { name: "Copied icon" })).toBeInTheDocument(); + await act(async () => pendingCopy.resolve(true)); + expect(screen.getByRole("img", { name: "Copied icon" })).toBeInTheDocument(); }); - it("does not show the checkmark when the clipboard write is rejected", async () => { + it("keeps the icon unchanged when copying fails", async () => { const user = userEvent.setup(); - const writeText = vi.fn().mockRejectedValue(new Error("permission denied")); - Object.defineProperty(navigator, "clipboard", { value: { writeText }, configurable: true }); - + vi.mocked(copyToClipboard).mockResolvedValue(false); renderWithProviders(); - const button = screen.getByRole("button", { name: "Copy value" }); - await user.click(button); + await user.click(screen.getByRole("button", { name: "Copy value" })); - await waitFor(() => expect(writeText).toHaveBeenCalledWith("secret-value")); + expect(copyToClipboard).toHaveBeenCalledWith("secret-value", "Copied"); expect(screen.queryByRole("img", { name: "Copied icon" })).not.toBeInTheDocument(); expect(screen.getByRole("img", { name: "Copy icon" })).toBeInTheDocument(); }); - it("does not show the checkmark when the clipboard API is unavailable", async () => { - const user = userEvent.setup(); - Object.defineProperty(navigator, "clipboard", { value: undefined, configurable: true }); - - renderWithProviders(); - - const button = screen.getByRole("button", { name: "Copy value" }); - await user.click(button); - - expect(screen.queryByRole("img", { name: "Copied icon" })).not.toBeInTheDocument(); - expect(screen.getByRole("img", { name: "Copy icon" })).toBeInTheDocument(); - expect(copyToClipboard).not.toHaveBeenCalled(); - }); - it("keeps the default icon styling, custom classes, title and 1200ms confirmation", async () => { vi.useFakeTimers(); - const writeText = vi.fn().mockResolvedValue(undefined); - Object.defineProperty(navigator, "clipboard", { value: { writeText }, configurable: true }); + vi.mocked(copyToClipboard).mockResolvedValue(true); renderWithProviders( , ); @@ -87,8 +65,7 @@ describe("CopyButton", () => { expect(screen.getByRole("img", { name: "Copy icon" })).toHaveClass("size-3"); await act(async () => fireEvent.click(button)); - expect(writeText).toHaveBeenCalledWith("request-id"); - expect(copyToClipboard).not.toHaveBeenCalled(); + expect(copyToClipboard).toHaveBeenCalledWith("request-id", "Copied"); act(() => vi.advanceTimersByTime(1199)); expect(screen.getByRole("img", { name: "Copied icon" })).toBeInTheDocument(); act(() => vi.advanceTimersByTime(1)); diff --git a/ui/litellm-dashboard/src/components/shared/CopyButton.tsx b/ui/litellm-dashboard/src/components/shared/CopyButton.tsx index 57ae86a26d0..225154e4bcf 100644 --- a/ui/litellm-dashboard/src/components/shared/CopyButton.tsx +++ b/ui/litellm-dashboard/src/components/shared/CopyButton.tsx @@ -54,17 +54,7 @@ const CopyButton: React.FC = ({ if (value == null || (!value && !isAction)) return null; const handleCopy = async () => { - if (isAction) { - setCopied(await copyToClipboard(value, copiedLabel)); - return; - } - if (!navigator.clipboard) return; - try { - await navigator.clipboard.writeText(value); - setCopied(true); - } catch { - setCopied(false); - } + setCopied(await copyToClipboard(value, copiedLabel)); }; const button = ( diff --git a/ui/litellm-dashboard/src/utils/dataUtils.test.ts b/ui/litellm-dashboard/src/utils/dataUtils.test.ts index af249ca601a..23b425b47e0 100644 --- a/ui/litellm-dashboard/src/utils/dataUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/dataUtils.test.ts @@ -1,5 +1,6 @@ // @vitest-environment jsdom +import "@testing-library/jest-dom/vitest"; import { describe, it, expect, beforeEach, vi, afterEach } from "vitest"; import { copyToClipboard, @@ -16,8 +17,6 @@ const mockToast = vi.mocked(toast); describe("dataUtils", () => { beforeEach(() => { vi.clearAllMocks(); - // Reset document.execCommand mock - delete (document as any).execCommand; }); afterEach(() => { @@ -134,194 +133,114 @@ describe("dataUtils", () => { }); describe("copyToClipboard", () => { - describe("when Clipboard API is available (HTTPS scenario)", () => { - beforeEach(() => { - // Mock modern Clipboard API - Object.assign(navigator, { - clipboard: { - writeText: vi.fn().mockResolvedValue(undefined), - }, - }); - }); - - it("should use navigator.clipboard.writeText when available", async () => { - const result = await copyToClipboard("test text"); - - expect(navigator.clipboard.writeText).toHaveBeenCalledWith("test text"); - expect(mockToast.success).toHaveBeenCalledWith("Copied to clipboard"); - expect(result).toBe(true); - }); - - it("should use custom message when provided", async () => { - await copyToClipboard("test text", "Custom message"); - - expect(mockToast.success).toHaveBeenCalledWith("Custom message"); - }); - - it("should return false for null/undefined text", async () => { - expect(await copyToClipboard(null)).toBe(false); - expect(await copyToClipboard(undefined)).toBe(false); - expect(navigator.clipboard.writeText).not.toHaveBeenCalled(); - }); - - it("should fall back to execCommand when clipboard API fails", async () => { - // Make clipboard API fail - navigator.clipboard.writeText = vi.fn().mockRejectedValue(new Error("Permission denied")); - - // Mock successful execCommand - document.execCommand = vi.fn().mockReturnValue(true); - - // Mock DOM methods - const mockTextArea = { - value: "", - style: {} as Record, - setAttribute: vi.fn(), - focus: vi.fn(), - select: vi.fn(), - }; - document.createElement = vi.fn().mockReturnValue(mockTextArea); - document.body.appendChild = vi.fn(); - document.body.removeChild = vi.fn(); - - const result = await copyToClipboard("test text"); - - expect(navigator.clipboard.writeText).toHaveBeenCalledWith("test text"); - expect(document.execCommand).toHaveBeenCalledWith("copy"); - expect(mockToast.success).toHaveBeenCalledWith("Copied to clipboard"); - expect(result).toBe(true); + const clipboardDescriptor = Object.getOwnPropertyDescriptor(navigator, "clipboard"); + const execCommandDescriptor = Object.getOwnPropertyDescriptor(document, "execCommand"); + const writeText = vi.fn<(text: string) => Promise>(); + const execCommand = vi.fn<(command: string) => boolean>(); + const selectedText = vi.fn(() => { + const element = document.activeElement; + return element instanceof HTMLTextAreaElement + ? element.value.slice(element.selectionStart, element.selectionEnd) + : undefined; + }); + const trigger = document.createElement("button"); + const text = "Request α\nsecond line"; + + beforeEach(() => { + writeText.mockResolvedValue(undefined); + execCommand.mockImplementation(() => { + selectedText(); + return true; }); + Object.defineProperty(navigator, "clipboard", { configurable: true, value: { writeText } }); + Object.defineProperty(document, "execCommand", { configurable: true, value: execCommand }); + vi.spyOn(console, "error").mockImplementation(() => {}); + document.body.appendChild(trigger); + trigger.focus(); }); - describe("when Clipboard API is not available (HTTP scenario)", () => { - beforeEach(() => { - // Mock HTTP scenario - no clipboard API - Object.assign(navigator, { - clipboard: undefined, - }); - - // Mock successful execCommand - document.execCommand = vi.fn().mockReturnValue(true); - - // Mock DOM methods - const mockTextArea = { - value: "", - style: {} as Record, - setAttribute: vi.fn(), - focus: vi.fn(), - select: vi.fn(), - }; - document.createElement = vi.fn().mockReturnValue(mockTextArea); - document.body.appendChild = vi.fn(); - document.body.removeChild = vi.fn(); - }); - - it("should fall back to execCommand when clipboard API is not available", async () => { - const result = await copyToClipboard("test text"); - - expect(document.createElement).toHaveBeenCalledWith("textarea"); - expect(document.execCommand).toHaveBeenCalledWith("copy"); - expect(mockToast.success).toHaveBeenCalledWith("Copied to clipboard"); - expect(result).toBe(true); - }); - - it("should set textarea properties correctly", async () => { - const mockTextArea = { - value: "", - style: {} as Record, - setAttribute: vi.fn(), - focus: vi.fn(), - select: vi.fn(), - }; - document.createElement = vi.fn().mockReturnValue(mockTextArea); - - await copyToClipboard("test text"); - - expect(mockTextArea.value).toBe("test text"); - /* eslint-disable jest-dom/prefer-to-have-style -- the subject is a plain mock object, not a DOM node, so toHaveStyle cannot read it */ - expect(mockTextArea.style.position).toBe("fixed"); - expect(mockTextArea.style.left).toBe("-999999px"); - expect(mockTextArea.style.top).toBe("-999999px"); - /* eslint-enable jest-dom/prefer-to-have-style */ - expect(mockTextArea.setAttribute).toHaveBeenCalledWith("readonly", ""); - expect(mockTextArea.focus).toHaveBeenCalled(); - expect(mockTextArea.select).toHaveBeenCalled(); - }); - - it("should handle execCommand failure", async () => { - document.execCommand = vi.fn().mockReturnValue(false); - - const result = await copyToClipboard("test text"); - - expect(document.execCommand).toHaveBeenCalledWith("copy"); - expect(mockToast.fromError).toHaveBeenCalledWith("Failed to copy to clipboard"); - expect(result).toBe(false); - }); - - it("should handle DOM manipulation errors", async () => { - document.createElement = vi.fn().mockImplementation(() => { - throw new Error("DOM error"); - }); - - const result = await copyToClipboard("test text"); - - expect(mockToast.fromError).toHaveBeenCalledWith("Failed to copy to clipboard"); - expect(result).toBe(false); - }); - - it("should clean up textarea element after successful copy", async () => { - const mockTextArea = { - value: "", - style: {} as Record, - setAttribute: vi.fn(), - focus: vi.fn(), - select: vi.fn(), - }; - document.createElement = vi.fn().mockReturnValue(mockTextArea); - - await copyToClipboard("test text"); - - expect(document.body.appendChild).toHaveBeenCalledWith(mockTextArea); - expect(document.body.removeChild).toHaveBeenCalledWith(mockTextArea); - }); + afterEach(() => { + document.body.replaceChildren(); + if (clipboardDescriptor) Object.defineProperty(navigator, "clipboard", clipboardDescriptor); + else Reflect.deleteProperty(navigator, "clipboard"); + if (execCommandDescriptor) Object.defineProperty(document, "execCommand", execCommandDescriptor); + else Reflect.deleteProperty(document, "execCommand"); }); - describe("edge cases", () => { - beforeEach(() => { - // Mock scenario where navigator exists but clipboard is null - Object.assign(navigator, { - clipboard: null, - }); + it("confirms a modern clipboard write only after it succeeds", async () => { + const pendingWrite = Promise.withResolvers(); + writeText.mockReturnValue(pendingWrite.promise); + const result = copyToClipboard(text, "Request copied"); - document.execCommand = vi.fn().mockReturnValue(true); - const mockTextArea = { - value: "", - style: {} as Record, - setAttribute: vi.fn(), - focus: vi.fn(), - select: vi.fn(), - }; - document.createElement = vi.fn().mockReturnValue(mockTextArea); - document.body.appendChild = vi.fn(); - document.body.removeChild = vi.fn(); + expect(writeText).toHaveBeenCalledWith(text); + expect(mockToast.success).not.toHaveBeenCalled(); + pendingWrite.resolve(); + + expect(await result).toBe(true); + expect(mockToast.success).toHaveBeenCalledWith("Request copied"); + expect(execCommand).not.toHaveBeenCalled(); + expect(trigger).toHaveFocus(); + }); + + it.each([null, undefined, ""])("does not copy absent content %s", async (value) => { + expect(await copyToClipboard(value)).toBe(false); + expect(writeText).not.toHaveBeenCalled(); + expect(execCommand).not.toHaveBeenCalled(); + expect(mockToast.success).not.toHaveBeenCalled(); + }); + + it.each([undefined, null, {}])("copies the full selection when the Clipboard API is %s", async (clipboard) => { + Object.defineProperty(navigator, "clipboard", { configurable: true, value: clipboard }); + + expect(await copyToClipboard(text)).toBe(true); + + expect(execCommand).toHaveBeenCalledWith("copy"); + expect(selectedText).toHaveReturnedWith(text); + expect(mockToast.success).toHaveBeenCalledWith("Copied to clipboard"); + expect(document.querySelector("textarea")).not.toBeInTheDocument(); + expect(trigger).toHaveFocus(); + }); + + it("falls back after a rejected modern write", async () => { + writeText.mockRejectedValue(new Error("Permission denied")); + + expect(await copyToClipboard(text, "Request copied")).toBe(true); + + expect(writeText).toHaveBeenCalledWith(text); + expect(execCommand).toHaveBeenCalledWith("copy"); + expect(selectedText).toHaveReturnedWith(text); + expect(mockToast.success).toHaveBeenCalledExactlyOnceWith("Request copied"); + expect(document.querySelector("textarea")).not.toBeInTheDocument(); + expect(trigger).toHaveFocus(); + }); + + it.each(["false", "throw"])("cleans up and reports failure when the fallback returns %s", async (outcome) => { + Object.defineProperty(navigator, "clipboard", { configurable: true, value: undefined }); + execCommand.mockImplementation(() => { + selectedText(); + if (outcome === "throw") throw new Error("Copy denied"); + return false; }); - it("should handle navigator.clipboard being null", async () => { - const result = await copyToClipboard("test text"); + expect(await copyToClipboard(text)).toBe(false); - expect(document.execCommand).toHaveBeenCalledWith("copy"); - expect(result).toBe(true); + expect(selectedText).toHaveReturnedWith(text); + expect(mockToast.fromError).toHaveBeenCalledWith("Failed to copy to clipboard"); + expect(mockToast.success).not.toHaveBeenCalled(); + expect(document.querySelector("textarea")).not.toBeInTheDocument(); + expect(trigger).toHaveFocus(); + }); + + it("reports failure when the fallback cannot create its selection", async () => { + Object.defineProperty(navigator, "clipboard", { configurable: true, value: undefined }); + vi.spyOn(document, "createElement").mockImplementationOnce(() => { + throw new Error("DOM error"); }); - it("should handle navigator.clipboard.writeText being undefined", async () => { - Object.assign(navigator, { - clipboard: {}, // clipboard exists but writeText doesn't - }); - - const result = await copyToClipboard("test text"); - - expect(document.execCommand).toHaveBeenCalledWith("copy"); - expect(result).toBe(true); - }); + expect(await copyToClipboard(text)).toBe(false); + expect(mockToast.fromError).toHaveBeenCalledWith("Failed to copy to clipboard"); + expect(mockToast.success).not.toHaveBeenCalled(); + expect(trigger).toHaveFocus(); }); }); }); diff --git a/ui/litellm-dashboard/src/utils/dataUtils.ts b/ui/litellm-dashboard/src/utils/dataUtils.ts index c813a69b999..a05b8d6b0c2 100644 --- a/ui/litellm-dashboard/src/utils/dataUtils.ts +++ b/ui/litellm-dashboard/src/utils/dataUtils.ts @@ -103,19 +103,20 @@ const fallbackCopyToClipboard = (text: string, messageText: string): boolean => textArea.setAttribute("readonly", ""); document.body.appendChild(textArea); - textArea.focus(); - textArea.select(); + try { + textArea.focus(); + textArea.select(); - const successful = document.execCommand("copy"); - document.body.removeChild(textArea); - if (previouslyFocused instanceof HTMLElement) previouslyFocused.focus(); - - if (successful) { - toast.success(messageText); - return true; - } else { - throw new Error("execCommand failed"); + if (document.execCommand("copy")) { + toast.success(messageText); + return true; + } + } finally { + textArea.remove(); + if (previouslyFocused instanceof HTMLElement) previouslyFocused.focus(); } + toast.fromError("Failed to copy to clipboard"); + return false; } catch (err) { toast.fromError("Failed to copy to clipboard"); console.error("Failed to copy: ", err); From 4c674e8e3f0758531308b68f4d42399811cf3b8c Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 26 Oct 2026 14:18:23 -0400 Subject: [PATCH 052/155] fix(mcp): report oversized SSE catalog errors (#48424) --- .../mcp_server/faults/list_outcomes.py | 46 +++++++++++-- .../mcp_server/rest_endpoints.py | 2 +- .../test_mcp_client.py | 56 ++++++++++++++++ .../mcp_server/faults/test_list_outcomes.py | 65 ++++++++++++++++++- 4 files changed, 163 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py b/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py index a3e33929f6d..e148e28e826 100644 --- a/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py +++ b/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py @@ -3,8 +3,8 @@ The aggregate listing deliberately keeps serving the healthy subset when one server fails, but a failed server must contribute a classified outcome instead of silently shrinking the list: an empty contribution with no signal makes a broken upstream indistinguishable from a healthy server with no -tools. Outcomes carry only machine fields (category and status code) so nothing from an upstream -body crosses the trust boundary; classification is total, so any exception out of a server fetch +tools. Outcomes carry machine fields and controlled diagnostics so nothing from an upstream body +crosses the trust boundary; classification is total, so any exception out of a server fetch becomes an outcome, never a second failure. """ @@ -21,8 +21,10 @@ from urllib.request import parse_http_list, parse_keqv_list import anyio import httpx import httpx2 +from mcp import MCPError +from mcp.types import CONNECTION_CLOSED from mcp.types import Tool as MCPTool -from pydantic import ConfigDict, TypeAdapter +from pydantic import ConfigDict, PositiveInt, TypeAdapter from typing_extensions import assert_never from litellm._logging import verbose_logger @@ -65,6 +67,16 @@ class ServerListFault(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) tag: ListFaultCategory status_code: int | None = None + sse_event_limit_bytes: PositiveInt | None = None + + @property + def public_message(self) -> str | None: + if self.sse_event_limit_bytes is None: + return None + return ( + f"Upstream MCP SSE event exceeded the {self.sse_event_limit_bytes} byte per-event limit. " + "Paginate tools/list responses so each event stays below the limit." + ) ServerOutcome: TypeAlias = ServerListOk | ServerListFault @@ -155,6 +167,18 @@ def raise_classified_list_failure( raise MCPServerListError(classify_list_exception(exc), server_name) from exc +def _sse_event_limit(exc: BaseException) -> int | None: + """The SDK replaces decoder exceptions with CONNECTION_CLOSED and a fixed diagnostic.""" + if not isinstance(exc, (httpx2.SSEError, MCPError)): + return None + if isinstance(exc, MCPError) and exc.error.code != CONNECTION_CLOSED: + return None + prefix: Final = "SSE stream failed: " if isinstance(exc, MCPError) else "" + message: Final = exc.error.message if isinstance(exc, MCPError) else str(exc) + matched: Final = re.fullmatch(prefix + r"Server-sent event exceeded the ([1-9][0-9]{0,18}) byte limit\.", message) + return int(matched[1]) if matched is not None else None + + def classify_list_exception(exc: BaseException) -> ServerListFault: """Classify a per-server listing failure into exactly one outcome. Total: an exception this function cannot recognize is the gateway's own fault (``internal``), never a re-raise.""" @@ -179,6 +203,11 @@ def classify_list_exception(exc: BaseException) -> ServerListFault: response: Final = _find_upstream_response(exc) if response is not None: return ServerListFault(tag="upstream_error", status_code=response.status_code) + sse_event_limit: Final = next( + (limit for error in iter_exception_tree(exc) if (limit := _sse_event_limit(error)) is not None), None + ) + if sse_event_limit is not None: + return ServerListFault(tag="upstream_error", sse_event_limit_bytes=sse_event_limit) if isinstance(exc, (httpx.TimeoutException, httpx2.TimeoutException)): return ServerListFault(tag="timeout") if isinstance(exc, (httpx.TransportError, httpx2.TransportError)): @@ -295,7 +324,7 @@ def listing_failed_outright(outcomes: Mapping[str, ServerOutcome]) -> bool: def outcome_wire_value(outcome: ServerOutcome) -> dict[str, object]: """The client-visible form of one outcome, for the tools/list result ``_meta`` and the REST - response: category plus status code only, never upstream prose or URLs.""" + response: category, status code and controlled diagnostics, never upstream prose or URLs.""" match outcome.tag: case "ok": return {"status": "ok", "tool_count": outcome.tool_count} @@ -303,6 +332,15 @@ def outcome_wire_value(outcome: ServerOutcome) -> dict[str, object]: return { "status": outcome.tag, **({"http_status": outcome.status_code} if outcome.status_code is not None else {}), + **( + { + "reason": "sse_event_too_large", + "limit_bytes": outcome.sse_event_limit_bytes, + "message": outcome.public_message, + } + if outcome.sse_event_limit_bytes is not None + else {} + ), } case _: assert_never(outcome.tag) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index ee063b81ae4..62387b7cc28 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -957,7 +957,7 @@ if MCP_AVAILABLE: status_code=list_fault_http_status(fault), detail={ "error": fault.tag, - "message": f"Failed to list tools from server {get_server_prefix(server)}", + "message": fault.public_message or f"Failed to list tools from server {get_server_prefix(server)}", }, ) from e except Exception as e: diff --git a/tests/unit/experimental_mcp_client/test_mcp_client.py b/tests/unit/experimental_mcp_client/test_mcp_client.py index b4485bcc5f1..60600afab84 100644 --- a/tests/unit/experimental_mcp_client/test_mcp_client.py +++ b/tests/unit/experimental_mcp_client/test_mcp_client.py @@ -1921,6 +1921,62 @@ async def test_list_tools_classifies_upstream_status_for_post_failures(failure_m assert rate_limit is None +@pytest.mark.asyncio +@pytest.mark.parametrize("paginated", (False, True)) +async def test_sse_catalog_limit_reports_the_failure_and_allows_smaller_pages(paginated: bool) -> None: + from mcp.types import Tool + + limit: Final = 1024 + tools: Final = tuple( + Tool(name=f"tool_{index}", description="private-upstream-secret " * 25, input_schema={"type": "object"}) + for index in range(2) + ) + catalog: Final = tuple(tool.model_dump(by_alias=True, exclude_none=True) for tool in tools) + assert len(json.dumps({"tools": catalog}).encode()) > limit + + def respond(request: httpx2.Request) -> httpx2.Response: + if request.method != "POST": + return httpx2.Response(200) + payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content) + assert isinstance(payload, (JSONRPCRequest, JSONRPCNotification)) + if isinstance(payload, JSONRPCNotification): + return httpx2.Response(202) + if payload.method == "initialize": + return httpx2.Response(200, json={ + "jsonrpc": "2.0", "id": payload.id, "result": { + "protocolVersion": LATEST_HANDSHAKE_VERSION, + "capabilities": {"tools": {}}, "serverInfo": {"name": "catalog", "version": "1"}, + }, + }) + assert payload.method == "tools/list" + cursor: Final = (payload.params or {}).get("cursor") + page: Final = catalog[1:] if cursor == "next-page" else catalog[:1] if paginated else catalog + result: Final = { + "tools": page, + **({"nextCursor": "next-page"} if paginated and cursor is None else {}), + } + event: Final = f"data: {json.dumps({'jsonrpc': '2.0', 'id': payload.id, 'result': result})}\n\n" + assert (len(event.encode()) < limit) == paginated + return httpx2.Response(200, headers={"content-type": "text/event-stream"}, content=event) + + def transport_context(url: str, http_client: httpx2.AsyncClient) -> _TransportContext: + return streamable_http_client(url, http_client=http_client, max_sse_event_size=limit) + + client: Final = _MockTransportClient( + respond, server_url="https://upstream.example/mcp", transport_context=transport_context, + ) + if paginated: + assert await client.list_tools(raise_on_error=True) == list(tools) + return + with pytest.raises(MCPError) as caught: + await client.list_tools(raise_on_error=True) + fault: Final = classify_list_exception(caught.value) + assert fault == ServerListFault(tag="upstream_error", sse_event_limit_bytes=limit) + assert list_fault_http_status(fault) == 502 + assert fault.public_message is not None + assert "private-upstream-secret" not in fault.public_message + + @pytest.mark.asyncio async def test_http_status_check_allows_auth_refresh_before_rejecting() -> None: from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import ( diff --git a/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py b/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py index f6902c778e5..fdae575276d 100644 --- a/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py +++ b/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py @@ -9,9 +9,10 @@ if sys.version_info < (3, 11): # BaseExceptionGroup is a builtin only from 3.11 from exceptiongroup import BaseExceptionGroup import httpx +import httpx2 import pytest from mcp import MCPError -from mcp.types import ErrorData +from mcp.types import CONNECTION_CLOSED, ErrorData from litellm.proxy._experimental.mcp_server.exceptions import ( MCPServerListError, @@ -102,6 +103,68 @@ def test_unknown_exception_is_internal(): assert classify_list_exception(ValueError("who knows")).tag == "internal" +@pytest.mark.parametrize("sdk_wrapped", (False, True)) +@pytest.mark.parametrize("nested", (False, True)) +def test_sse_event_limit_is_a_safe_upstream_fault(sdk_wrapped: bool, nested: bool) -> None: + limit: Final = 1024 + decoder_message: Final = f"Server-sent event exceeded the {limit} byte limit." + failure: Final = ( + MCPError(code=CONNECTION_CLOSED, message=f"SSE stream failed: {decoder_message}") + if sdk_wrapped else httpx2.SSEError(decoder_message) + ) + wrapped: Final = BaseExceptionGroup("private-upstream-secret", [failure]) if nested else failure + fault: Final = classify_list_exception(wrapped) + message: Final = ( + f"Upstream MCP SSE event exceeded the {limit} byte per-event limit. " + "Paginate tools/list responses so each event stays below the limit." + ) + assert fault == ServerListFault(tag="upstream_error", sse_event_limit_bytes=limit) + assert fault.public_message == message + assert list_fault_http_status(fault) == 502 + assert outcome_wire_value(fault) == { + "status": "upstream_error", "reason": "sse_event_too_large", "limit_bytes": limit, "message": message, + } + + +@pytest.mark.parametrize( + "failure", + ( + MCPError(code=CONNECTION_CLOSED, message="Connection closed"), + MCPError(code=-32603, message="SSE stream failed: Server-sent event exceeded the 1024 byte limit."), + MCPError(code=CONNECTION_CLOSED, message="SSE stream failed: Server-sent event exceeded the 0 byte limit."), + MCPError(code=CONNECTION_CLOSED, message="SSE stream failed: Server-sent event exceeded the -1 byte limit."), + MCPError(code=CONNECTION_CLOSED, message=( + "SSE stream failed: Server-sent event exceeded the 10000000000000000000 byte limit." + )), + MCPError(code=CONNECTION_CLOSED, message=( + "SSE stream failed: Server-sent event exceeded the 1024 byte limit. private-secret" + )), + httpx2.SSEError("private-secret Server-sent event exceeded the 1024 byte limit."), + httpx2.SSEError("Expected response with content type 'text/event-stream', got 'private-secret'."), + RuntimeError("SSE stream failed: Server-sent event exceeded the 1024 byte limit."), + ), +) +def test_unrelated_errors_do_not_claim_an_sse_event_limit(failure: Exception) -> None: + fault: Final = classify_list_exception(failure) + assert fault.sse_event_limit_bytes is None + assert fault.public_message is None + assert "private-secret" not in str(outcome_wire_value(fault)) + + +@pytest.mark.parametrize("status_code", (401, 403)) +def test_upstream_auth_takes_precedence_over_sse_event_limit(status_code: int) -> None: + request: Final = httpx2.Request("POST", "https://upstream.example/mcp") + response: Final = httpx2.Response(status_code, request=request) + failure: Final = BaseExceptionGroup("upstream errors", [ + httpx2.SSEError("Server-sent event exceeded the 1024 byte limit."), + httpx2.HTTPStatusError("private-secret", request=request, response=response), + ]) + fault: Final = classify_list_exception(failure) + assert fault.tag == ("auth_required" if status_code == 401 else "forbidden") + assert list_fault_http_status(fault) == status_code + assert fault.sse_event_limit_bytes is None + + def test_wire_value_carries_no_prose(): fault = ServerListFault(tag="upstream_error", status_code=500) assert outcome_wire_value(fault) == {"status": "upstream_error", "http_status": 500} From 1c022d1d06dba1cb42f9190c7b2641a1402bd2d9 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 26 Oct 2026 11:31:26 -0700 Subject: [PATCH 053/155] chore(cost-map): add openai gpt-6.1-terra from the pricing page (#48429) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 102 ++++++++++++++++++ model_prices_and_context_window.json | 102 ++++++++++++++++++ 2 files changed, 204 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9a7d88b8eda..0a85d8a00cd 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -78652,5 +78652,107 @@ "prompt_cache_min_tokens": 512, "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing", "supports_web_search": true + }, + "gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_12hr": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-05, + "cache_creation_input_token_cost_above_12hr_batches": 1.25e-06, + "cache_creation_input_token_cost_above_12hr_flex": 1.25e-06, + "cache_creation_input_token_cost_above_12hr_priority": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-06, + "cache_creation_input_token_cost_batches": 6.25e-07, + "cache_creation_input_token_cost_flex": 6.25e-07, + "cache_creation_input_token_cost_priority": 2.5e-06, + "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-08, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 2e-07, + "cache_read_input_token_cost_batches": 2.5e-08, + "cache_read_input_token_cost_flex": 2.5e-08, + "cache_read_input_token_cost_priority": 1e-07, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_272k_tokens": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 1e-06, + "input_cost_per_token_above_272k_tokens_flex": 1e-06, + "input_cost_per_token_above_272k_tokens_priority": 4e-06, + "input_cost_per_token_batches": 5e-07, + "input_cost_per_token_flex": 5e-07, + "input_cost_per_token_priority": 2e-06, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-06, + "output_cost_per_token_above_272k_tokens": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 3.75e-06, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-06, + "output_cost_per_token_above_272k_tokens_priority": 1.5e-05, + "output_cost_per_token_batches": 2.5e-06, + "output_cost_per_token_flex": 2.5e-06, + "output_cost_per_token_priority": 1e-05, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01}, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_tool_search": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, + "chatgpt/gpt-6.1-terra": { + "litellm_provider": "chatgpt", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9a7d88b8eda..0a85d8a00cd 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -78652,5 +78652,107 @@ "prompt_cache_min_tokens": 512, "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing", "supports_web_search": true + }, + "gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_12hr": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-05, + "cache_creation_input_token_cost_above_12hr_batches": 1.25e-06, + "cache_creation_input_token_cost_above_12hr_flex": 1.25e-06, + "cache_creation_input_token_cost_above_12hr_priority": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-06, + "cache_creation_input_token_cost_batches": 6.25e-07, + "cache_creation_input_token_cost_flex": 6.25e-07, + "cache_creation_input_token_cost_priority": 2.5e-06, + "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-08, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 2e-07, + "cache_read_input_token_cost_batches": 2.5e-08, + "cache_read_input_token_cost_flex": 2.5e-08, + "cache_read_input_token_cost_priority": 1e-07, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_272k_tokens": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 1e-06, + "input_cost_per_token_above_272k_tokens_flex": 1e-06, + "input_cost_per_token_above_272k_tokens_priority": 4e-06, + "input_cost_per_token_batches": 5e-07, + "input_cost_per_token_flex": 5e-07, + "input_cost_per_token_priority": 2e-06, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-06, + "output_cost_per_token_above_272k_tokens": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 3.75e-06, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-06, + "output_cost_per_token_above_272k_tokens_priority": 1.5e-05, + "output_cost_per_token_batches": 2.5e-06, + "output_cost_per_token_flex": 2.5e-06, + "output_cost_per_token_priority": 1e-05, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01}, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_tool_search": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, + "chatgpt/gpt-6.1-terra": { + "litellm_provider": "chatgpt", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true } } From 38f318e73659c5ae73c7ce26f387e1933ba210e9 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 26 Oct 2026 11:44:30 -0700 Subject: [PATCH 054/155] fix(otel): decide per-team Arize sampling from the trace id so every pod agrees (#47468) * fix(otel): decide per-team Arize sampling from the trace id so every pod agrees * test(otel): cover per-team sampling across pods, workers, endpoints and restarts * fix(otel): remember only a failed tree's sampling verdict so a later failed call answers to the error rate A successful tree's verdict follows from the trace id alone, so remembering it only made a later failed call of the same trace follow the success rate on the pod that served both, while other pods judged it at the error rate * test(otel): flush every worker before the restart cell kills it and cover a failed call after a dropped one * fix(otel): decide team sampling per request, not per trace Calls sharing a caller's trace id are separate requests, keyed by their root span. Each is decided by the trace hash at the rate for its own outcome, and only its own late spans follow its verdict, so no pod applies an earlier call's verdict to a later call of the trace * fix(otel): forget a span's request root when it ends, not when newer spans start The root of each open span was kept in a first-in-first-out table of 16384 entries, so a span two levels below the root that stayed open while the worker started more spans than that lost its entry and ended under its parent's id. Its request then never closed and was held from every sampled destination until shutdown. Each open span now keeps its entry until it ends, and a span starting under a parent that has already ended reads the root from the parent span itself * fix(otel): decide a request as its spans end once it falls out of the open span table A request whose span was pushed out of the table of open spans, because the worker held more than 16384 open at once, ended that span under another key and was held from every sampled destination until shutdown. Pushing a span out now drops its request's open count too, so the request is decided as each of its spans ends, the way one pushed out of the open request table already is * test(otel): annotate the team sampling test locals as Final * test(otel): build the caller's remote parent in one helper instead of rebinding it per trace * fix(otel): keep each open span's request until it ends, bounded by the open requests The span table had its own bound of 16384, so a worker holding more spans open than that pushed out the oldest, and a failure in such a span no longer reached its request. Each request now holds the ids of its open spans, and a span's entry in the lookup goes when the span ends or when its request is pushed out of the open request table, the same bound main already applies. A failure anywhere in a tracked request reaches its verdict however many spans are open --------- Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- .../integrations/otel/model/destination.py | 2 +- .../integrations/otel/plumbing/providers.py | 160 ++-- .../test_otel_team_sampling_across_pods.py | 723 ++++++++++++++++++ .../otel/test_otel_v2_destinations.py | 403 ++++++++-- 4 files changed, 1180 insertions(+), 108 deletions(-) create mode 100644 tests/integration/observability/test_otel_team_sampling_across_pods.py diff --git a/litellm/integrations/otel/model/destination.py b/litellm/integrations/otel/model/destination.py index 4226b4361dd..e760ba3600b 100644 --- a/litellm/integrations/otel/model/destination.py +++ b/litellm/integrations/otel/model/destination.py @@ -42,7 +42,7 @@ class OtelDestination(LiteLLMBaseModel): success_sampling_rate: float | None = Field( default=None, description=( - "Share of the request trees forwarded, 0.0..1.0, drawn once per request when its root span ends; " + "Share of the request trees forwarded, 0.0..1.0, picked by a hash of the trace id; " "``None`` forwards every one." ), ) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 898eab5715b..fc1e9b7099d 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -1,13 +1,14 @@ """Provider / exporter factory + the Baggage span processor.""" import queue -import random import threading import time +import weakref from collections import OrderedDict from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from hashlib import sha256 from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal +from typing import TYPE_CHECKING, Final, Literal, TypeAlias from opentelemetry import _logs, baggage, metrics, trace from opentelemetry._logs import Logger, LoggerProvider, NoOpLoggerProvider @@ -486,15 +487,15 @@ def is_llm_call_span(span: ReadableSpan) -> bool: return GenAI.OPERATION_NAME in attributes and MCP.METHOD_NAME not in attributes -#: Request trees held back for a sampled destination until the last span of the trace +#: Request trees held back for a sampled destination until the last span of the request #: still open ends, the verdicts kept for the spans that start after that (a post-call -#: database write), and the spans in flight per trace. All bounded, so a span that never -#: ends, or a flood of requests, cannot hold spans for ever: the oldest tree is decided -#: on what it has. +#: database write), the spans in flight per request, and the request each open span +#: belongs to. All bounded, so a span that never ends, or a flood of requests, cannot +#: hold spans for ever: the oldest tree is decided on what it has. _MAX_PENDING_TREES: Final = 1024 _MAX_PENDING_SPANS_PER_TREE: Final = 512 _MAX_REMEMBERED_VERDICTS: Final = 4096 -_MAX_OPEN_TRACES: Final = 16384 +_MAX_OPEN_REQUESTS: Final = 16384 class _PendingTree: @@ -512,25 +513,35 @@ def _is_sampled(destination: "OtelDestination") -> bool: return destination.success_sampling_rate is not None or destination.error_sampling_rate is not None -#: A verdict is one request tree's draw for one destination at its rates: the exporter +#: A request is its trace id and the id of its root, the span of it whose parent is +#: remote or missing. Calls sharing a caller's trace are separate requests, each decided +#: on its own outcome. +_RequestKey: TypeAlias = tuple[int, int] + +#: A verdict is one request's keep or drop for one destination at its rates: the exporter #: ``cache_key`` leaves the rates out, so two destinations sharing an exporter with -#: different rates would otherwise share one draw. -_VerdictKey = tuple[int, object, float | None, float | None] +#: different rates would otherwise share one verdict. +_VerdictKey: TypeAlias = tuple[int, int, object, float | None, float | None] -def _verdict_key(trace_id: int, destination: "OtelDestination") -> _VerdictKey: - return (trace_id, destination.cache_key(), destination.success_sampling_rate, destination.error_sampling_rate) +def _verdict_key(request: _RequestKey, destination: "OtelDestination") -> _VerdictKey: + return (*request, destination.cache_key(), destination.success_sampling_rate, destination.error_sampling_rate) -def _keeps_tree(destination: "OtelDestination", failed: bool, draw: Callable[[], float]) -> bool: - """Whether one draw against the destination's rate keeps a request tree, read the - way the legacy Arize callback reads ``arize_success_sampling_rate`` / - ``arize_error_sampling_rate``: a tree with a failed span answers to the error - rate, an unset rate keeps everything and ``0.0`` keeps nothing.""" +def _keeps_tree(destination: "OtelDestination", failed: bool, trace_id: int) -> bool: + """Whether the destination's rate keeps a request tree of ``trace_id``: a tree with + a failed span answers to ``arize_error_sampling_rate``, the rest to + ``arize_success_sampling_rate``, an unset rate keeps everything and ``0.0`` keeps + nothing. + + The first eight bytes of a SHA-256 of the trace id are read against the rate, so + every process deciding a request of the trace with the same outcome reaches the same + verdict.""" rate: Final = destination.error_sampling_rate if failed else destination.success_sampling_rate if rate is None: return True - return rate > 0.0 and draw() <= rate + digest: Final = sha256(trace_id.to_bytes(16, "big")).digest() + return int.from_bytes(digest[:8], "big") < round(rate * 2**64) def _in_scope(span: ReadableSpan, scope: "OtelSpanScope") -> bool: @@ -667,10 +678,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): default_capture_by_backend: Mapping[str, str] = MappingProxyType({}), pending_drains: int = _MAX_PENDING_DRAINS, drain_pool: _DrainPool | None = None, - sampling_draw: Callable[[], float] | None = None, ) -> None: self._operator_sinks: Final = operator_sinks - self._draw: Final = sampling_draw if sampling_draw is not None else random.random self._excluded_db_systems: Final = excluded_db_systems self._default_capture: Final = default_capture self._default_capture_by_backend: Final = default_capture_by_backend @@ -682,24 +691,32 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._processors: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded LRU self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count - self._pending: OrderedDict[int, _PendingTree] = OrderedDict() # mutable-ok: bounded, by trace id - self._open: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded, spans in flight by trace id + self._pending: OrderedDict[_RequestKey, _PendingTree] = OrderedDict() # mutable-ok: bounded, by request + self._open: OrderedDict[_RequestKey, set[int]] = OrderedDict() # mutable-ok: bounded, open span ids by request + self._roots: dict[int, _RequestKey] = {} # mutable-ok: holds only the spans ``_open`` holds, request by span id + self._span_roots: weakref.WeakKeyDictionary[SDKSpan, int] = weakref.WeakKeyDictionary() self._verdicts: OrderedDict[_VerdictKey, bool] = OrderedDict() # mutable-ok: bounded LRU self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains) def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: context: Final = span.context - trace_id: Final = context.trace_id if context is not None else 0 with self._lock: - self._open[trace_id] = self._open.get(trace_id, 0) + 1 - while len(self._open) > _MAX_OPEN_TRACES: - self._open.popitem(last=False) + request: Final = self._request_of(span, trace.get_current_span(parent_context)) + if context is not None: + self._span_roots[span] = request[1] + self._roots[context.span_id] = request + self._open.setdefault(request, set()).add(context.span_id) + while len(self._open) > _MAX_OPEN_REQUESTS: + for forgotten in self._open.popitem(last=False)[1]: + self._roots.pop(forgotten, None) def on_end(self, span: ReadableSpan) -> None: suppressed: Final = suppressed_backends() attributes: Final = span.attributes or _NO_ATTRIBUTES - context: Final = span.context - trace_id: Final = context.trace_id if context is not None else 0 + with self._lock: + request: Final = self._request_of(span) + if span.context is not None: + self._roots.pop(span.context.span_id, None) failed: Final = span.status.status_code is StatusCode.ERROR for destination in request_destinations(): if ( @@ -716,11 +733,32 @@ class TenantFanOutSpanProcessor(SpanProcessor): if not _is_sampled(destination): self._forward(span, destination) continue - for held_span, held_destination in self._route(trace_id, span, destination, failed): + for held_span, held_destination in self._route(request, span, destination, failed): self._forward(held_span, held_destination) - for held_span, held_destination in self._settle(trace_id, failed): + for held_span, held_destination in self._settle(request, span, failed): self._forward(held_span, held_destination) + def _request_of(self, span: ReadableSpan, parent_span: Span | None = None) -> _RequestKey: + """The request ``span`` belongs to, read under ``_lock``: the request recorded while + it is open, else the span itself when its parent is remote or missing, else its + parent's request, recorded while the parent is open or on ``parent_span`` for a + child starting under a parent that has ended, else the parent itself.""" + context: Final = span.context + if context is None: + return (0, 0) + recorded: Final = self._roots.get(context.span_id) + if recorded is not None: + return recorded + parent: Final = span.parent + if parent is None or parent.is_remote: + return (context.trace_id, context.span_id) + parent_request: Final = self._roots.get(parent.span_id) + if parent_request is not None: + return parent_request + if isinstance(parent_span, SDKSpan): + return (context.trace_id, self._span_roots.get(parent_span, parent.span_id)) + return (context.trace_id, parent.span_id) + def _default_capture_for(self, destination: "OtelDestination") -> str: backend: Final = destination.callback_name return ( @@ -743,14 +781,14 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._release(processor) def _route( - self, trace_id: int, span: ReadableSpan, destination: "OtelDestination", failed: bool + self, request: _RequestKey, span: ReadableSpan, destination: "OtelDestination", failed: bool ) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": """The spans to forward now for ``span`` on a sampled ``destination``: the span itself when its tree's verdict is already known and keeps it, nothing while the - tree is held for the rest of its trace, or what the bounds made the fan-out decide + tree is held for the rest of its request, or what the bounds made the fan-out decide early (a tree that outgrew its cap is decided on what it has, and so is the oldest tree once too many are waiting, so a span that never ends holds nothing back for - ever). A failed span marks its tree before any such early draw. Once shutdown + ever). A failed span marks its tree before any such early decision. Once shutdown has started flushing, nothing is held: a span ending then is decided at once, since no later flush would reach it. @@ -758,58 +796,62 @@ class TenantFanOutSpanProcessor(SpanProcessor): another thread between the two would leave this span in a fresh tree. """ with self._lock: - verdict: Final = self._verdicts.get(_verdict_key(trace_id, destination)) + verdict: Final = self._verdicts.get(_verdict_key(request, destination)) if verdict is not None: return ((span, destination),) if verdict else () - tree: Final = self._pending.setdefault(trace_id, _PendingTree()) + tree: Final = self._pending.setdefault(request, _PendingTree()) tree.held.append((span, destination)) if failed: tree.failed = True if self._draining or len(tree.held) >= _MAX_PENDING_SPANS_PER_TREE: - return self._decide_locked(trace_id) + return self._decide_locked(request) if len(self._pending) > _MAX_PENDING_TREES: oldest: Final = next(iter(self._pending)) return self._decide_locked(oldest) return () - def _decide(self, trace_id: int) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": + def _decide(self, request: _RequestKey) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": with self._lock: - return self._decide_locked(trace_id) + return self._decide_locked(request) - def _settle(self, trace_id: int, failed: bool) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": - """What the trace's held tree exports once the span that just ended was the last - of its trace still open. The model-call span ends in the post-call callback, - after the server span, so a tree is decided when its trace goes quiet rather than + def _settle( + self, request: _RequestKey, span: ReadableSpan, failed: bool + ) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": + """What the request's held tree exports once the span that just ended was the last + of its request still open. The model-call span ends in the post-call callback, + after the server span, so a tree is decided when its request goes quiet rather than when its root ends, and a model call that fails late still answers to the error - rate. A trace whose starts this fan-out never saw (attached mid-flight, or forgotten - by the bound) is decided as each of its spans ends. + rate. A request whose starts this fan-out never saw (attached mid-flight, or + forgotten by the bound) is decided as each of its spans ends. """ with self._lock: - tree: Final = self._pending.get(trace_id) + tree: Final = self._pending.get(request) if tree is not None and failed: tree.failed = True - still_open: Final = self._open.pop(trace_id, 1) - 1 - if still_open > 0: - self._open[trace_id] = still_open + still_open: Final = self._open.pop(request, set()) + if span.context is not None: + still_open.discard(span.context.span_id) + if still_open: + self._open[request] = still_open return () - return self._decide_locked(trace_id) + return self._decide_locked(request) - def _decide_locked(self, trace_id: int) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": - """Draw once per destination for the held tree; the spans the draws keep, and a - verdict remembered for the spans of this trace that start later.""" - tree: Final = self._pending.pop(trace_id, None) + def _decide_locked(self, request: _RequestKey) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": + """Decide once per destination for the held tree; the spans the verdicts keep, and + a verdict remembered for the spans of this request that start later.""" + tree: Final = self._pending.pop(request, None) if tree is None: return () - verdicts: dict[_VerdictKey, bool] = {} # mutable-ok: one draw per destination of this tree + verdicts: dict[_VerdictKey, bool] = {} # mutable-ok: one verdict per destination of this tree for _, destination in tree.held: - key = _verdict_key(trace_id, destination) + key = _verdict_key(request, destination) if key not in verdicts: - verdicts[key] = _keeps_tree(destination, tree.failed, self._draw) + verdicts[key] = _keeps_tree(destination, tree.failed, request[0]) self._verdicts[key] = verdicts[key] while len(self._verdicts) > _MAX_REMEMBERED_VERDICTS: self._verdicts.popitem(last=False) return tuple( - (span, destination) for span, destination in tree.held if verdicts[_verdict_key(trace_id, destination)] + (span, destination) for span, destination in tree.held if verdicts[_verdict_key(request, destination)] ) def _operator_already_writes( @@ -850,8 +892,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): with self._lock: self._draining = True undecided: Final = tuple(self._pending) - for trace_id in undecided: - for held_span, held_destination in self._decide(trace_id): + for request in undecided: + for held_span, held_destination in self._decide(request): self._forward(held_span, held_destination) deadline: Final = time.monotonic() + self._drain_seconds with self._lock: diff --git a/tests/integration/observability/test_otel_team_sampling_across_pods.py b/tests/integration/observability/test_otel_team_sampling_across_pods.py new file mode 100644 index 00000000000..dc7b295febd --- /dev/null +++ b/tests/integration/observability/test_otel_team_sampling_across_pods.py @@ -0,0 +1,723 @@ +"""A team's Arize sampling rate decides once per trace, so every pod and worker serving a call of it agrees. + +Two proxies share one database, as two pods behind a load balancer do: the first runs one worker, the second +two. Calls of one trace carry the caller's ``traceparent`` and land on different pods and workers; the team's +Arize space must receive every call of a trace or none of them. +""" + +import asyncio +import json +import re +import uuid +from collections.abc import Callable, Generator, Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass, replace +from itertools import chain, product +from pathlib import Path +from typing import Final, Literal, TypeAlias +from urllib.parse import urlparse + +import httpx +import pytest +import yaml +from integration._support.client import ( + Scenario, + eventually, + gateway_from_environment, + list_value, + object_value, + until_asserted, +) +from integration._support.database import scratch_database +from integration._support.otlp_sink import Span, SpanSinks, configure_sink, recorded_requests, recorded_spans +from integration._support.process import OwnedProxy, WorkerHealthcheck, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +MODEL: Final = "team-sampling" +SERVER: Final = 2 +CALL_ID: Final = "litellm.call_id" +OPERATION: Final = "gen_ai.operation.name" +SPACE: Final = "space-sampled" +CREDENTIALS: Final = {"arize_space_key": SPACE, "arize_api_key": "arize-sampled"} +FAILING: Final = re.compile(rb"sampling-fail") +JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +TRACES: Final = 16 +Endpoint: TypeAlias = Literal["chat", "responses", "messages"] +Verdicts: TypeAlias = Mapping[str, tuple[bool, ...]] + + +def _chat_reply(stream: bool) -> Reply: + usage: Final[JsonValue] = {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9} + if not stream: + return Reply( + body=json.dumps( + { + "id": "chatcmpl-sampling", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} + ], + "usage": usage, + } + ).encode() + ) + chunk: Final = {"id": "chatcmpl-sampling", "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini"} + deltas: Final[tuple[JsonValue, ...]] = ( + {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]}, + {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {**chunk, "choices": [], "usage": usage}, + ) + return Reply( + content_type="text/event-stream", + chunks=(*(b"data: " + json.dumps(delta).encode() + b"\n\n" for delta in deltas), b"data: [DONE]\n\n"), + ) + + +def _responses_reply(stream: bool) -> Reply: + response: Final[Mapping[str, JsonValue]] = { + "id": "resp_sampling", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "id": "msg_sampling", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 7, "output_tokens": 2, "total_tokens": 9}, + } + if not stream: + return Reply(body=json.dumps(response).encode()) + events: Final[tuple[Mapping[str, JsonValue], ...]] = ( + { + "type": "response.created", + "sequence_number": 0, + "response": {**response, "status": "in_progress", "output": []}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_sampling", + "output_index": 0, + "content_index": 0, + "delta": "ok", + }, + {"type": "response.completed", "sequence_number": 2, "response": dict(response)}, + ) + return Reply( + content_type="text/event-stream", + chunks=tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events), + ) + + +def _upstream(request: Request) -> Reply: + if FAILING.search(request.body) is not None: + return Reply(status=500, body=b'{"error":{"message":"scripted upstream failure","type":"server_error"}}') + stream: Final = object_value(JSON.validate_json(request.body)).get("stream") is True + if request.target.endswith("/responses"): + return _responses_reply(stream) + return _chat_reply(stream) + + +@dataclass(frozen=True, slots=True) +class Sent: + trace_id: str + parent_span_id: str + call_id: str + + +@dataclass(frozen=True, slots=True) +class Call: + path: str + body: Mapping[str, JsonValue] + headers: Mapping[str, str] + sent: Sent + expected: int + + +def _call( + key: str, trace_id: str | None, endpoint: Endpoint = "chat", stream: bool = False, content: str = "hello" +) -> Call: + parent: Final = uuid.uuid4().hex[:16] + call_id: Final = str(uuid.uuid4()) + messages: Final[JsonValue] = [{"role": "user", "content": content}] + path, body = { + "chat": ("/v1/chat/completions", {"model": MODEL, "messages": messages, "stream": stream}), + "responses": ("/v1/responses", {"model": MODEL, "input": content, "stream": stream}), + "messages": ("/v1/messages", {"model": MODEL, "max_tokens": 16, "messages": messages, "stream": stream}), + }[endpoint] + headers: Final = { + "Authorization": f"Bearer {key}", + "x-litellm-call-id": call_id, + "connection": "close", + **({} if trace_id is None else {"traceparent": f"00-{trace_id}-{parent}-01"}), + } + expected: Final = 500 if FAILING.search(content.encode()) else 200 + return Call(path, body, headers, Sent(trace_id or "", parent if trace_id else "", call_id), expected) + + +@dataclass(frozen=True, slots=True) +class Pods: + """``first`` serves with one worker, ``second`` with two.""" + + first: OwnedProxy + second: OwnedProxy + scenario: Scenario + sinks: SpanSinks + + def team_key(self, variables: Mapping[str, str]) -> str: + team: Final = self.scenario.team() + attached: Final = self.first.gateway.request( + "POST", f"/team/{team}/callback", {"callback_name": "arize", "callback_vars": dict(variables)} + ) + assert attached.status_code == 200, attached.text + return self.scenario.key(team_id=team) + + def send(self, pod: OwnedProxy, call: Call) -> Sent: + with pod.gateway.client.stream("POST", call.path, json=call.body, headers=call.headers) as response: + text: Final = response.read().decode() + assert response.status_code == call.expected, text + return call.sent + + async def send_async(self, client: httpx.AsyncClient, call: Call) -> Sent: + async with client.stream("POST", call.path, json=call.body, headers=call.headers) as response: + text: Final = (await response.aread()).decode() + assert response.status_code == call.expected, text + return call.sent + + def on_each_worker(self, pod: OwnedProxy, call: Callable[[], Call]) -> tuple[Sent, ...]: + """One call served by each worker of ``pod``, the others suspended while it runs.""" + workers: Final = sorted(pod.worker_pids()) + assert len(workers) == pod.gateway.workers, workers + sent: Final[list[Sent]] = [] # mutable-ok: one call per worker, in worker order + for serving in workers: + with _suspended_all_but(pod, workers, serving): + sent.append(self.send(pod, call())) + return tuple(sent) + + def fence(self, key: str) -> tuple[Sent, ...]: + """Calls of an unsampled team on the sampled team's exporter, one per worker of both pods. + + Each worker sends spans to that exporter in the order they were handed to it, so all of these + arriving means every call any worker decided before them has been sent too. + """ + return (self.send(self.first, _call(key, uuid.uuid4().hex)),) + self.on_each_worker( + self.second, lambda: _call(key, uuid.uuid4().hex) + ) + + +@contextmanager +def _suspended_all_but(pod: OwnedProxy, workers: Sequence[int], serving: int) -> Generator[None]: + if len(workers) == 1: + yield + return + others: Final = [pid for pid in workers if pid != serving] + with pod.suspended(others[0]): + yield + + +def _space_id(request: Mapping[str, JsonValue]) -> JsonValue: + return {name.lower(): value for name, value in object_value(request.get("headers") or {}).items()}.get( + "arize-space-id" + ) + + +def _space_span_ids(url: str) -> frozenset[str]: + """Span ids the Arize sink received in export requests sent with the team's space id.""" + ours: Final = ( + list_value(request.get("span_ids") or []) for request in recorded_requests(url) if _space_id(request) == SPACE + ) + return frozenset(span_id for span_id in chain.from_iterable(ours) if isinstance(span_id, str)) + + +def _team_spans(pods: Pods, since: int) -> tuple[Span, ...]: + owned: Final = _space_span_ids(pods.sinks.arize) + return tuple(span for span in recorded_spans(pods.sinks.arize, since)[1] if span["span_id"] in owned) + + +def _delivered(spans: tuple[Span, ...], sent: Sequence[Sent]) -> frozenset[str]: + """Call ids of ``sent`` whose server span is among ``spans``.""" + by_parent: Final = {(call.trace_id, call.parent_span_id): call.call_id for call in sent} + return frozenset( + by_parent[(span["trace_id"], span["parent_span_id"])] + for span in spans + if span["kind"] == SERVER and (span["trace_id"], span["parent_span_id"]) in by_parent + ) + + +def _await_logged(pods: Pods, since: int, sent: Sequence[Sent]) -> None: + """Wait until every call of ``sent`` has its model span in the operator's unsampled copy.""" + expected: Final = {call.call_id for call in sent} + eventually( + lambda: ( + { + str(span["attributes"][CALL_ID]) + for span in recorded_spans(pods.sinks.operator, since)[1] + if OPERATION in span["attributes"] and CALL_ID in span["attributes"] + } + & expected + ), + lambda logged: logged == expected, + seconds=60, + ) + + +@dataclass(frozen=True, slots=True) +class Cursors: + operator: int + arize: int + + +def _cursors(pods: Pods) -> Cursors: + return Cursors(recorded_spans(pods.sinks.operator)[0], recorded_spans(pods.sinks.arize)[0]) + + +def _await_flushed(pods: Pods, cursors: Cursors, fence_key: str) -> None: + """Wait until every worker of both pods has sent the team's exporter everything it decided so far.""" + fences: Final = pods.fence(fence_key) + + def arrived() -> None: + delivered: Final = _delivered(_team_spans(pods, cursors.arize), fences) + assert {fence.call_id for fence in fences} <= delivered, "the unsampled team's calls have not arrived yet" + + until_asserted(arrived, seconds=30) + + +def _settled( + pods: Pods, + cursors: Cursors, + by_trace: Mapping[str, Sequence[Sent]], + fence_key: str, + holds: Callable[[tuple[bool, ...]], bool] = lambda kept: len(set(kept)) == 1, +) -> Verdicts: + """Per trace, which of its calls reached the team's space, once every worker has sent what it decided + and ``holds`` is true of every trace.""" + calls: Final = tuple(chain.from_iterable(by_trace.values())) + _await_logged(pods, cursors.operator, calls) + _await_flushed(pods, cursors, fence_key) + + def check() -> Verdicts: + delivered: Final = _delivered(_team_spans(pods, cursors.arize), calls) + verdicts: Final = { + trace_id: tuple(call.call_id in delivered for call in sent) for trace_id, sent in by_trace.items() + } + broken: Final = {trace_id: kept for trace_id, kept in verdicts.items() if not holds(kept)} + assert broken == {}, f"{len(broken)} of {len(verdicts)} traces were split (per call, kept): {broken}" + return verdicts + + return until_asserted(check, seconds=30) + + +def _trace_ids(cell: str, count: int = TRACES) -> tuple[str, ...]: + """Fixed RFC 4122 shaped trace ids, like the uuid4 hex a caller mints for ``traceparent``.""" + return tuple(uuid.uuid5(uuid.NAMESPACE_URL, f"team-sampling/{cell}/{index}").hex for index in range(count)) + + +def _kept(verdicts: Verdicts) -> int: + return sum(1 for kept in verdicts.values() if all(kept)) + + +@pytest.mark.parametrize( + ("variables", "content"), + [({"arize_success_sampling_rate": "0.5"}, "hello"), ({"arize_error_sampling_rate": "0.5"}, "sampling-fail")], + ids=["success-rate", "error-rate"], +) +def test_each_trace_reaches_the_team_space_whole_or_not_at_all( + pods: Pods, variables: Mapping[str, str], content: str +) -> None: + """The ticket's case: three calls per trace load balanced over the pods.""" + sampled: Final = pods.team_key({**CREDENTIALS, **variables}) + cursors: Final = _cursors(pods) + by_trace: Final = { + trace_id: tuple( + pods.send(pod, _call(sampled, trace_id, content=content)) for pod in (pods.first, pods.second, pods.first) + ) + for trace_id in _trace_ids("-".join(sorted(variables))) + } + + verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS)) + + assert 0 < _kept(verdicts) < TRACES, f"a rate of 0.5 kept {_kept(verdicts)} of {TRACES} traces" + + +@pytest.mark.parametrize("client", ["sync", "async"]) +def test_every_endpoint_follows_the_verdict_of_its_trace(pods: Pods, client: str) -> None: + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"}) + cursors: Final = _cursors(pods) + plan: Final[tuple[tuple[OwnedProxy, Endpoint, bool], ...]] = ( + (pods.first, "chat", False), + (pods.second, "chat", True), + (pods.first, "responses", False), + (pods.second, "messages", True), + ) + + async def send_trace(trace_id: str) -> tuple[Sent, ...]: + clients: Final = { + id(pod): httpx.AsyncClient(base_url=str(pod.gateway.client.base_url), timeout=15, trust_env=False) + for pod in (pods.first, pods.second) + } + try: + return tuple( + await asyncio.gather( + *( + pods.send_async(clients[id(pod)], _call(sampled, trace_id, endpoint, stream)) + for pod, endpoint, stream in plan + ) + ) + ) + finally: + for opened in clients.values(): + await opened.aclose() + + by_trace: Final = { + trace_id: ( + tuple(pods.send(pod, _call(sampled, trace_id, endpoint, stream)) for pod, endpoint, stream in plan) + if client == "sync" + else asyncio.run(send_trace(trace_id)) + ) + for trace_id in _trace_ids(f"endpoints-{client}") + } + + verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS)) + + assert 0 < _kept(verdicts) < TRACES + + +def test_the_workers_of_one_pod_agree(pods: Pods) -> None: + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"}) + cursors: Final = _cursors(pods) + by_trace: Final = { + trace_id: pods.on_each_worker(pods.second, lambda trace_id=trace_id: _call(sampled, trace_id)) + for trace_id in _trace_ids("workers") + } + + verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS)) + + assert 0 < _kept(verdicts) < TRACES + + +def test_a_replaced_worker_reaches_the_verdict_its_predecessor_reached(pods: Pods) -> None: + """A worker that restarts forgets the verdicts it remembered; the calls it serves next for the same + traces still answer the way the first ones did.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"}) + cursors: Final = _cursors(pods) + fence_key: Final = pods.team_key(CREDENTIALS) + traces: Final = _trace_ids("replaced") + before: Final = { + trace_id: pods.on_each_worker(pods.second, lambda t=trace_id: _call(sampled, t)) for trace_id in traces + } + _await_logged(pods, cursors.operator, tuple(chain.from_iterable(before.values()))) + _await_flushed(pods, cursors, fence_key) + for victim in sorted(pods.second.worker_pids()): + pods.second.kill_worker(victim) + by_trace: Final = { + trace_id: before[trace_id] + pods.on_each_worker(pods.second, lambda t=trace_id: _call(sampled, t)) + for trace_id in traces + } + + verdicts: Final = _settled(pods, cursors, by_trace, fence_key) + + assert 0 < _kept(verdicts) < TRACES + + +def test_a_failed_call_is_kept_wherever_a_successful_call_of_its_trace_is(pods: Pods) -> None: + """One call of the trace fails on one pod and answers to the error rate, another succeeds on the other + pod and answers to the success rate: with the error rate above the success rate, no trace keeps its + successful call and drops its failed one.""" + sampled: Final = pods.team_key( + {**CREDENTIALS, "arize_success_sampling_rate": "0.3", "arize_error_sampling_rate": "0.7"} + ) + cursors: Final = _cursors(pods) + by_trace: Final = { + trace_id: ( + pods.send(pods.first, _call(sampled, trace_id, content="sampling-fail")), + pods.send(pods.second, _call(sampled, trace_id)), + ) + for trace_id in _trace_ids("mixed", 32) + } + + verdicts: Final = _settled( + pods, cursors, by_trace, pods.team_key(CREDENTIALS), holds=lambda kept: kept[0] or not kept[1] + ) + + assert any(kept == (True, False) for kept in verdicts.values()), "no trace sat between the two rates" + assert any(kept == (True, True) for kept in verdicts.values()), "no trace sat under both rates" + + +def test_a_failed_call_answers_to_the_error_rate_on_a_pod_that_dropped_its_trace(pods: Pods) -> None: + """A pod that already dropped a successful call of the trace still keeps a later failed call of it at an + unset error rate, as the pod that only served the failed call does.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.3"}) + cursors: Final = _cursors(pods) + traces: Final = _trace_ids("failed-after-dropped") + succeeded: Final = {trace_id: pods.send(pods.first, _call(sampled, trace_id)) for trace_id in traces} + _await_logged(pods, cursors.operator, tuple(succeeded.values())) + by_trace: Final = { + trace_id: ( + succeeded[trace_id], + pods.send(pods.first, _call(sampled, trace_id, content="sampling-fail")), + pods.send(pods.second, _call(sampled, trace_id, content="sampling-fail")), + ) + for trace_id in traces + } + + verdicts: Final = _settled( + pods, cursors, by_trace, pods.team_key(CREDENTIALS), holds=lambda kept: kept[1] and kept[2] + ) + + assert any(not kept[0] for kept in verdicts.values()), "no successful call was dropped before its trace failed" + + +def test_a_successful_call_answers_to_the_success_rate_on_a_pod_that_kept_its_failed_call(pods: Pods) -> None: + """A pod that already kept a failed call of the trace drops a later successful call of it at a zero + success rate, as the pod that only served the successful call does.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.0"}) + cursors: Final = _cursors(pods) + traces: Final = _trace_ids("succeeded-after-kept") + failed: Final = { + trace_id: pods.send(pods.first, _call(sampled, trace_id, content="sampling-fail")) for trace_id in traces + } + _await_logged(pods, cursors.operator, tuple(failed.values())) + by_trace: Final = { + trace_id: ( + failed[trace_id], + pods.send(pods.first, _call(sampled, trace_id)), + pods.send(pods.second, _call(sampled, trace_id)), + ) + for trace_id in traces + } + + _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS), holds=lambda kept: kept == (True, False, False)) + + +def test_concurrent_calls_of_one_trace_on_one_worker_each_answer_to_their_own_outcome(pods: Pods) -> None: + """A failed and a successful call of the trace in flight at once on the single worker of the first pod + are decided apart: the failed one is kept at an unset error rate, the successful one dropped at a zero + success rate.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.0"}) + cursors: Final = _cursors(pods) + traces: Final = _trace_ids("concurrent-mixed") + plan: Final = tuple(product(traces, ("sampling-fail", "hello"))) + + def send(job: tuple[str, str]) -> Sent: + return pods.send(pods.first, _call(sampled, job[0], content=job[1])) + + with ThreadPoolExecutor(max_workers=8) as pool: + sent: Final = tuple(pool.map(send, plan)) + by_trace: Final = {trace_id: tuple(call for call in sent if call.trace_id == trace_id) for trace_id in traces} + + _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS), holds=lambda kept: kept == (True, False)) + + +@pytest.mark.parametrize( + ("variables", "expected"), + [({"arize_success_sampling_rate": "1.0"}, TRACES), ({"arize_success_sampling_rate": "0.0"}, 0), ({}, TRACES)], + ids=["one", "zero", "unset"], +) +def test_a_rate_at_either_end_keeps_every_trace_or_none( + pods: Pods, variables: Mapping[str, str], expected: int +) -> None: + sampled: Final = pods.team_key({**CREDENTIALS, **variables}) + cursors: Final = _cursors(pods) + by_trace: Final = { + trace_id: tuple(pods.send(pod, _call(sampled, trace_id)) for pod in (pods.first, pods.second)) + for trace_id in _trace_ids(f"edge-{'-'.join(f'{k}={v}' for k, v in sorted(variables.items()))}") + } + + verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS)) + + assert _kept(verdicts) == expected + + +def test_a_request_without_a_traceparent_is_sampled_whole_at_the_rate(pods: Pods) -> None: + """With no caller trace the proxy starts one per request: each request's server and model spans travel + together and the rate still keeps some requests and drops others.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"}) + cursors: Final = _cursors(pods) + sent: Final = tuple( + pods.send(pod, _call(sampled, None)) for _, pod in product(range(12), (pods.first, pods.second)) + ) + _await_logged(pods, cursors.operator, sent) + trace_of: Final = { + str(span["attributes"][CALL_ID]): span["trace_id"] + for span in recorded_spans(pods.sinks.operator, cursors.operator)[1] + if CALL_ID in span["attributes"] + } + fences: Final = pods.fence(pods.team_key(CREDENTIALS)) + + def check() -> Mapping[str, tuple[bool, bool]]: + spans: Final = _team_spans(pods, cursors.arize) + assert {fence.call_id for fence in fences} <= _delivered(spans, fences), ( + "the unsampled team's calls have not arrived yet" + ) + kinds: Final = { + call.call_id: ( + any(span["kind"] == SERVER for span in spans if span["trace_id"] == trace_of[call.call_id]), + any(OPERATION in span["attributes"] for span in spans if span["trace_id"] == trace_of[call.call_id]), + ) + for call in sent + } + partial: Final = {call_id: kept for call_id, kept in kinds.items() if kept[0] != kept[1]} + assert partial == {}, f"requests reached the space without their whole tree (server, model): {partial}" + return kinds + + kept: Final = sum(1 for server, _ in until_asserted(check, seconds=30).values() if server) + assert 0 < kept < len(sent), f"a rate of 0.5 kept {kept} of {len(sent)} requests" + + +def test_a_key_level_destination_follows_the_verdict_of_its_trace(pods: Pods) -> None: + team: Final = pods.scenario.team() + sampled: Final = pods.scenario.key( + team_id=team, + metadata={ + "logging": [ + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": {**CREDENTIALS, "arize_success_sampling_rate": "0.5"}, + } + ] + }, + ) + cursors: Final = _cursors(pods) + by_trace: Final = { + trace_id: tuple(pods.send(pod, _call(sampled, trace_id)) for pod in (pods.first, pods.second, pods.first)) + for trace_id in _trace_ids("key-level") + } + + verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS)) + + assert 0 < _kept(verdicts) < TRACES + + +def test_a_concurrent_burst_keeps_every_trace_whole(pods: Pods) -> None: + """Four calls per trace, all in flight at once across both pods.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"}) + cursors: Final = _cursors(pods) + traces: Final = _trace_ids("burst", 24) + plan: Final = tuple(product(traces, (pods.first, pods.second, pods.first, pods.second))) + + def send(job: tuple[str, OwnedProxy]) -> Sent: + return pods.send(job[1], _call(sampled, job[0])) + + with ThreadPoolExecutor(max_workers=16) as pool: + sent: Final = list(pool.map(send, plan)) + by_trace: Final = {trace_id: tuple(call for call in sent if call.trace_id == trace_id) for trace_id in traces} + + verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS)) + + assert 0 < _kept(verdicts) < len(traces) + + +def test_a_stalled_team_space_during_a_burst_keeps_serving_and_each_trace_whole(pods: Pods) -> None: + """The team's Arize endpoint stops answering while a burst of calls runs over both pods: every call is + still served, and once the endpoint answers again each trace arrives whole or not at all.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"}) + cursors: Final = _cursors(pods) + traces: Final = _trace_ids("stalled") + plan: Final = tuple(product(traces, (pods.first, pods.second))) + + def send(job: tuple[str, OwnedProxy]) -> Sent: + return pods.send(job[1], _call(sampled, job[0])) + + configure_sink(pods.sinks.arize, paused=True) + try: + with ThreadPoolExecutor(max_workers=8) as pool: + sent: Final = tuple(pool.map(send, plan)) + finally: + configure_sink(pods.sinks.arize, paused=False) + by_trace: Final = {trace_id: tuple(call for call in sent if call.trace_id == trace_id) for trace_id in traces} + + verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS)) + + assert 0 < _kept(verdicts) < len(traces) + + +def _write_config(directory: Path, sinks: SpanSinks, provider: Wire) -> Path: + shared: Final = object_value( + JSON.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + ) + config: Final = { + **shared, + "model_list": [ + *list_value(shared.get("model_list") or []), + { + "model_name": MODEL, + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "integration-provider-key", + "api_base": provider.url + "/v1", + }, + }, + ], + "litellm_settings": { + **object_value(shared.get("litellm_settings") or {}), + "callbacks": ["langfuse_otel"], + "otel_tenant_destination_mode": "additive", + "provider_url_destination_allowed_hosts": [urlparse(sinks.operator).netloc, urlparse(sinks.arize).netloc], + }, + "router_settings": {**object_value(shared.get("router_settings") or {}), "num_retries": 0}, + "general_settings": {**object_value(shared.get("general_settings") or {}), "user_api_key_cache_ttl": 2}, + } + path: Final = directory / "team-sampling.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@contextmanager +def _two_pods(provider: Wire, sinks: SpanSinks, directory: Path) -> Generator[Pods]: + environment: Final = { + "LITELLM_OTEL_V2": "1", + "OTEL_BSP_SCHEDULE_DELAY": "300", + "MAPPER_NAMES": "genai,openinference", + "LANGFUSE_HOST": sinks.operator, + "LANGFUSE_PUBLIC_KEY": "pk-lf-operator", + "LANGFUSE_SECRET_KEY": "sk-lf-operator", + "ARIZE_SPACE_KEY": "space-operator", + "ARIZE_API_KEY": "arize-operator", + "ARIZE_HTTP_ENDPOINT": sinks.arize + "/v1/traces", + } + removed: Final = ("LITELLM_OTEL_TENANT_DESTINATION_MODE", "OTEL_TRACES_SAMPLER", "OTEL_TRACES_SAMPLER_ARG") + config: Final = _write_config(directory, sinks, provider) + for pod in ("first", "second"): + (directory / pod).mkdir() + with gateway_from_environment() as shared, scratch_database("team_sampling") as database_url: + overrides: Final = {**environment, "DATABASE_URL": database_url} + with ( + owned_proxy_process( + shared, directory / "first", overrides, config=config, remove_environment=removed + ) as first, + owned_proxy_process( + shared, + directory / "second", + overrides, + config=config, + remove_environment=removed, + workers=2, + healthcheck=WorkerHealthcheck.LENIENT, + ) as second, + replace(first.gateway, database_url=database_url).scenario() as scenario, + ): + eventually(second.worker_pids, lambda pids: len(pids) == 2, seconds=120) + yield Pods(first, second, scenario, sinks) + + +@pytest.fixture(scope="module") +def provider() -> Iterator[Wire]: + with wire_server(_upstream) as wire: + yield wire + + +@pytest.fixture(scope="module") +def pods(provider: Wire, audit_sinks: SpanSinks, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Pods]: + with _two_pods(provider, audit_sinks, tmp_path_factory.mktemp("team-sampling")) as started: + yield started diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index c7a057cca5c..782442aad22 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -4,6 +4,7 @@ import asyncio import contextvars import threading import time +import uuid from base64 import b64encode from collections.abc import Mapping from dataclasses import replace @@ -14,10 +15,12 @@ from typing import Final import pytest from opentelemetry import trace as trace_api +from opentelemetry.context import Context from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import ReadableSpan, TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.sdk.trace.id_generator import RandomIdGenerator from opentelemetry.trace import Status, StatusCode import litellm @@ -1159,12 +1162,36 @@ class TestFanOut: ARIZE_TEAM_PARAMS = {"arize_space_id": "space-team", "arize_api_key": "key-team"} +def caller_context(trace_id: int) -> Context: + return trace_api.set_span_in_context( + trace_api.NonRecordingSpan( + trace_api.SpanContext(trace_id=trace_id, span_id=1, is_remote=True, trace_flags=trace_api.TraceFlags(0x01)) + ) + ) + + def arize_destination(**rates: str) -> OtelDestination: destination = destination_for("arize", {**ARIZE_TEAM_PARAMS, **rates}) assert destination is not None, "the fixture must resolve for the test to mean anything" return destination +#: Trace ids whose SHA-256 reads 0.40 and 0.80 of the way through its first eight bytes, +#: so a rate of 0.5 keeps the first and drops the second. +KEPT_AT_HALF: Final = 4 +DROPPED_AT_HALF: Final = 5 + + +class _FixedTraceId(RandomIdGenerator): + """Every trace the provider starts gets ``trace_id``.""" + + def __init__(self, trace_id: int) -> None: + self._trace_id = trace_id + + def generate_trace_id(self) -> int: + return self._trace_id + + class TestTenantSampling: """A team's ``arize_success_sampling_rate`` and ``arize_error_sampling_rate`` decide how much of its traffic the fan-out delivers to its space, the way they already decide what @@ -1173,14 +1200,17 @@ class TestTenantSampling: NAMES = {"POST /v1/chat/completions", "auth /v1/chat/completions", "chat gpt-4"} @staticmethod - def _fan_out(dest_exporter, draw=None, global_exporter=None): - provider = TracerProvider() + def _fan_out( + dest_exporter: InMemorySpanExporter, + trace_id: int | None = None, + global_exporter: InMemorySpanExporter | None = None, + ) -> TracerProvider: + provider: Final = TracerProvider() if trace_id is None else TracerProvider(id_generator=_FixedTraceId(trace_id)) if global_exporter is not None: provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(global_exporter), "arize")) - kwargs = {"processor_factory": lambda _d: SimpleSpanProcessor(dest_exporter)} - if draw is not None: - kwargs["sampling_draw"] = draw - provider.add_span_processor(TenantFanOutSpanProcessor(**kwargs)) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) return provider @staticmethod @@ -1224,7 +1254,7 @@ class TestTenantSampling: def test_an_unset_rate_exports_everything(self): dest_exporter = InMemorySpanExporter() - provider = self._fan_out(dest_exporter, draw=lambda: 0.99) + provider: Final = self._fan_out(dest_exporter, trace_id=DROPPED_AT_HALF) self._run(provider, (arize_destination(),)) @@ -1300,10 +1330,10 @@ class TestTenantSampling: def test_a_span_that_ends_after_the_root_follows_the_requests_verdict(self): """A post-call database write that starts after the whole request tree has ended - goes where the rest of the tree went, without a second draw.""" + goes where the rest of the tree went, without a second verdict.""" kept_exporter, dropped_exporter = InMemorySpanExporter(), InMemorySpanExporter() - kept = self._fan_out(kept_exporter, draw=lambda: 0.1) - dropped = self._fan_out(dropped_exporter, draw=lambda: 0.9) + kept: Final = self._fan_out(kept_exporter, trace_id=KEPT_AT_HALF) + dropped: Final = self._fan_out(dropped_exporter, trace_id=DROPPED_AT_HALF) destinations = (arize_destination(arize_success_sampling_rate="0.5"),) def late_tree(provider): @@ -1327,14 +1357,263 @@ class TestTenantSampling: } assert dropped_exporter.get_finished_spans() == () + def test_a_span_that_ends_after_a_failed_root_follows_the_failed_requests_verdict(self): + """A post-call database write after a failed request goes where that request went, + though on its own it would answer to the success rate.""" + dest_exporter: Final = InMemorySpanExporter() + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + + def run(): + set_request_destinations(destinations) + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions") as root: + root.set_status(Status(StatusCode.ERROR, "upstream 500")) + root_context: Final = trace_api.set_span_in_context(root) + with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=root_context): + pass + + in_fresh_context(run) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + "POST /v1/chat/completions", + "postgres INSERT LiteLLM_SpendLogs", + } + + def test_a_failed_call_after_a_dropped_call_of_its_trace_still_answers_to_the_error_rate(self): + """A process that dropped a successful call of the trace keeps a later failed call of + it at an unset error rate, as a process that only served the failed call does.""" + served_both, served_failed = InMemorySpanExporter(), InMemorySpanExporter() + destinations: Final = (arize_destination(arize_success_sampling_rate="0.5"),) + both: Final = self._fan_out(served_both, trace_id=DROPPED_AT_HALF) + + self._run(both, destinations) + self._run(both, destinations, failed=True) + self._run(self._fan_out(served_failed, trace_id=DROPPED_AT_HALF), destinations, failed=True) + + assert [sorted(s.name for s in exporter.get_finished_spans()) for exporter in (served_both, served_failed)] == [ + sorted(self.NAMES) + ] * 2 + + def test_a_successful_call_after_a_kept_failed_call_of_its_trace_answers_to_the_success_rate(self): + """A process that kept a failed call of the trace drops a later successful call of it + at a zero success rate, as a process that only served the successful call does.""" + served_both, served_success = InMemorySpanExporter(), InMemorySpanExporter() + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + both: Final = self._fan_out(served_both, trace_id=KEPT_AT_HALF) + + self._run(both, destinations, failed=True) + self._run(both, destinations) + self._run(self._fan_out(served_success, trace_id=KEPT_AT_HALF), destinations) + + assert len(served_both.get_finished_spans()) == len(self.NAMES), "only the failed call" + assert all(s.name in self.NAMES for s in served_both.get_finished_spans()) + assert served_success.get_finished_spans() == () + + def test_a_late_child_that_fails_after_its_request_was_dropped_is_dropped_with_it(self): + """A post-call write that fails after its successful request was dropped goes where + the request went, rather than reaching the space on its own at the error rate.""" + dest_exporter: Final = InMemorySpanExporter() + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + + def run(): + set_request_destinations(destinations) + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions") as root: + root_context: Final = trace_api.set_span_in_context(root) + with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=root_context) as write: + write.set_status(Status(StatusCode.ERROR, "connection reset")) + + in_fresh_context(run) + + assert dest_exporter.get_finished_spans() == () + + def test_a_failure_deep_in_a_request_marks_the_whole_request(self): + """A database span that fails under the auth span puts the whole request, root + included, on the error rate.""" + dest_exporter: Final = InMemorySpanExporter() + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + + def run(): + set_request_destinations(destinations) + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("auth /v1/chat/completions"): + with tracer.start_as_current_span("postgres SELECT LiteLLM_VerificationToken") as lookup: + lookup.set_status(Status(StatusCode.ERROR, "connection reset")) + with tracer.start_as_current_span("chat gpt-4"): + pass + + in_fresh_context(run) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + *self.NAMES, + "postgres SELECT LiteLLM_VerificationToken", + } + + def test_a_span_two_levels_down_that_outlives_many_other_spans_still_closes_its_request(self): + """A classifier call under the route span that stays open while the worker serves + more requests than it tracks at once still counts against its own request, so the + request is decided once it ends, on its own outcome.""" + dest_exporter: Final = InMemorySpanExporter() + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + + def other_request(): + set_request_destinations(()) + with get_tracer(provider, "litellm").start_as_current_span("POST /v1/chat/completions"): + pass + + def run(): + set_request_destinations(destinations) + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions") as root: + with tracer.start_as_current_span("route gpt-4"): + with tracer.start_as_current_span("chat classifier"): + for _ in range(otel_providers._MAX_OPEN_REQUESTS + 1): + contextvars.Context().run(other_request) + with tracer.start_as_current_span("chat gpt-4"): + pass + root.set_status(Status(StatusCode.ERROR, "upstream 500")) + + in_fresh_context(run) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + "POST /v1/chat/completions", + "route gpt-4", + "chat classifier", + "chat gpt-4", + } + + def test_a_failure_two_levels_down_reaches_its_request_however_many_spans_are_open(self): + """A classifier call that fails while another request holds more spans open than + the worker tracks requests marks its own request, which is then kept at the error + rate as a whole.""" + dest_exporter: Final = InMemorySpanExporter() + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + others: Final = contextvars.Context() + + def open_other_request(): + set_request_destinations(()) + tracer: Final = get_tracer(provider, "litellm") + root: Final = tracer.start_span("POST /v1/chat/completions") + parent: Final = trace_api.set_span_in_context(root) + return ( + root, + *(tracer.start_span("chat flood", context=parent) for _ in range(otel_providers._MAX_OPEN_REQUESTS)), + ) + + def run(): + set_request_destinations(destinations) + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("route gpt-4"): + with tracer.start_as_current_span("chat classifier") as classifier: + held_open: Final = others.run(open_other_request) + classifier.set_status(Status(StatusCode.ERROR, "classifier timed out")) + return held_open + + held_open: Final = in_fresh_context(run) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + "POST /v1/chat/completions", + "route gpt-4", + "chat classifier", + } + for span in held_open: + span.end() + + def test_the_fan_out_forgets_a_span_once_it_ends_and_a_request_once_it_is_pushed_out(self): + """The request lookup holds only spans still open, and only those of the requests + the fan-out still counts, so neither ended spans nor a flood of open requests grow + it past the open request bound.""" + fan_out: Final = TenantFanOutSpanProcessor( + processor_factory=lambda _d: SimpleSpanProcessor(InMemorySpanExporter()) + ) + provider: Final = TracerProvider() + provider.add_span_processor(fan_out) + + def run(): + set_request_destinations((arize_destination(arize_success_sampling_rate="1.0"),)) + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("chat gpt-4"): + pass + assert (fan_out._roots, dict(fan_out._open)) == ({}, {}) + root: Final = tracer.start_span("POST /v1/chat/completions", context=trace_api.Context()) + tracer.start_span("route gpt-4", context=trace_api.set_span_in_context(root)) + return tuple( + tracer.start_span(f"POST {index}", context=trace_api.Context()) + for index in range(otel_providers._MAX_OPEN_REQUESTS) + ) + + held_open: Final = in_fresh_context(run) + + assert len(fan_out._roots) == otel_providers._MAX_OPEN_REQUESTS + for span in held_open: + span.end() + assert (fan_out._roots, dict(fan_out._open)) == ({}, {}) + + def test_a_span_starting_under_an_ended_span_below_the_root_follows_its_request(self): + """A post-call write started under the route span after its failed request ended + is kept with the request at the error rate, rather than decided on its own at the + success rate.""" + dest_exporter: Final = InMemorySpanExporter() + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + + def run(): + set_request_destinations(destinations) + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions") as root: + with tracer.start_as_current_span("route gpt-4") as route: + route_context: Final = trace_api.set_span_in_context(route) + root.set_status(Status(StatusCode.ERROR, "upstream 500")) + with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=route_context): + pass + + in_fresh_context(run) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + "POST /v1/chat/completions", + "route gpt-4", + "postgres INSERT LiteLLM_SpendLogs", + } + + def test_concurrent_calls_of_one_trace_each_answer_to_their_own_outcome(self): + """Two calls of the caller's trace in flight at once on one process, one failing, + are decided as two processes serving one each would decide them.""" + dest_exporter: Final = InMemorySpanExporter() + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + remote: Final = trace_api.SpanContext( + trace_id=KEPT_AT_HALF, span_id=1, is_remote=True, trace_flags=trace_api.TraceFlags(0x01) + ) + + def run(): + set_request_destinations(destinations) + tracer: Final = get_tracer(provider, "litellm") + caller: Final = trace_api.set_span_in_context(trace_api.NonRecordingSpan(remote)) + with tracer.start_as_current_span("POST failed", context=caller) as failed: + with tracer.start_as_current_span("POST succeeded", context=caller): + pass + failed.set_status(Status(StatusCode.ERROR, "upstream 500")) + + in_fresh_context(run) + + assert [s.name for s in dest_exporter.get_finished_spans()] == ["POST failed"] + def test_a_late_span_whose_verdict_was_forgotten_is_decided_on_its_own_not_stranded(self): """Once enough other requests have been decided to evict a request's verdict, a span of it that starts late (a post-call database write) opens and closes its own count, - so it is decided on a draw of its own as soon as it ends instead of waiting in a tree - nothing would ever close.""" + so it is decided on its own as soon as it ends instead of waiting in a tree nothing + would ever close.""" dest_exporter = InMemorySpanExporter() - provider = self._fan_out(dest_exporter, draw=lambda: 0.1) - destinations = (arize_destination(arize_success_sampling_rate="0.5"),) + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="1.0"),) def run(): set_request_destinations(destinations) @@ -1353,18 +1632,17 @@ class TestTenantSampling: assert names[0] == "POST /v1/chat/completions" assert names[-1] == "postgres INSERT LiteLLM_SpendLogs", "exported as it ended, not held for a root" - def test_a_callers_traceparent_neither_steers_the_draw_nor_hides_the_root(self): - """The draw is random, not read from the trace id a caller may have chosen, and the - server span under a remote parent still closes the tree.""" + def test_a_callers_traceparent_decides_by_its_trace_id_and_still_closes_the_tree(self): + """A request continuing a caller's trace is decided on that trace id, and the server + span under a remote parent still closes the tree.""" dest_exporter = InMemorySpanExporter() - draws = iter([0.9, 0.1]) - provider = self._fan_out(dest_exporter, draw=lambda: next(draws)) + provider: Final = self._fan_out(dest_exporter) destinations = (arize_destination(arize_success_sampling_rate="0.5"),) def run(): set_request_destinations(destinations) tracer = get_tracer(provider, "litellm") - for trace_id in (1, 2): + for trace_id in (DROPPED_AT_HALF, KEPT_AT_HALF): remote = trace_api.SpanContext( trace_id=trace_id, span_id=1, is_remote=True, trace_flags=trace_api.TraceFlags(0x01) ) @@ -1377,12 +1655,12 @@ class TestTenantSampling: kept = dest_exporter.get_finished_spans() assert [s.name for s in kept] == ["chat gpt-4", "POST /v1/chat/completions"] - assert all(s.context.trace_id == 2 for s in kept), "the second request, whose draw of 0.1 passed" + assert all(s.context.trace_id == KEPT_AT_HALF for s in kept) def test_a_root_that_never_ends_holds_the_tree_only_up_to_the_bound(self): dest_exporter = InMemorySpanExporter() - provider = self._fan_out(dest_exporter, draw=lambda: 0.0) - destinations = (arize_destination(arize_success_sampling_rate="0.5"),) + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="1.0"),) def run(): set_request_destinations(destinations) @@ -1397,8 +1675,8 @@ class TestTenantSampling: def test_a_flood_of_waiting_trees_decides_the_oldest_on_what_it_has(self): dest_exporter = InMemorySpanExporter() - provider = self._fan_out(dest_exporter, draw=lambda: 0.0) - destinations = (arize_destination(arize_success_sampling_rate="0.5"),) + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="1.0"),) def run(): set_request_destinations(destinations) @@ -1413,21 +1691,21 @@ class TestTenantSampling: assert [s.name for s in dest_exporter.get_finished_spans()] == ["chat 0"] - def test_a_flood_of_open_traces_forgets_the_oldest_and_decides_it_as_its_spans_end(self): - """The fan-out counts open spans for a bounded number of traces, a bound far above - what one process keeps in flight, since every trace of the provider is counted, not - only the sampled ones. A trace pushed out of that count is decided on what it has as + def test_a_flood_of_open_requests_forgets_the_oldest_and_decides_it_as_its_spans_end(self): + """The fan-out counts open spans for a bounded number of requests, a bound far above + what one process keeps in flight, since every request of the provider is counted, not + only the sampled ones. A request pushed out of that count is decided on what it has as its next span ends, root still open, rather than held until a bound gets to it.""" dest_exporter = InMemorySpanExporter() - provider = self._fan_out(dest_exporter, draw=lambda: 0.0) - destinations = (arize_destination(arize_success_sampling_rate="0.5"),) + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="1.0"),) def run(): set_request_destinations(destinations) tracer = get_tracer(provider, "litellm") root = tracer.start_span("POST /v1/chat/completions", context=trace_api.Context()) tracer.start_span("auth /v1/chat/completions", context=trace_api.set_span_in_context(root)).end() - for index in range(otel_providers._MAX_OPEN_TRACES): + for index in range(otel_providers._MAX_OPEN_REQUESTS): tracer.start_span(f"POST {index}", context=trace_api.Context()) assert dest_exporter.get_finished_spans() == (), "still held before the count is forgotten" tracer.start_span("chat gpt-4", context=trace_api.set_span_in_context(root)).end() @@ -1443,14 +1721,12 @@ class TestTenantSampling: def test_shutdown_decides_what_is_still_held(self): dest_exporter = InMemorySpanExporter() - fan_out = TenantFanOutSpanProcessor( - processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter), sampling_draw=lambda: 0.0 - ) + fan_out: Final = TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) provider = TracerProvider() provider.add_span_processor(fan_out) def run(): - set_request_destinations((arize_destination(arize_success_sampling_rate="0.5"),)) + set_request_destinations((arize_destination(arize_success_sampling_rate="1.0"),)) tracer = get_tracer(provider, "litellm") with tracer.start_as_current_span("POST /v1/chat/completions"): with tracer.start_as_current_span("chat gpt-4"): @@ -1476,14 +1752,12 @@ class TestTenantSampling: straggler.end() super().on_end(span) - fan_out = TenantFanOutSpanProcessor( - processor_factory=lambda _d: _EndsAStragglerOnExport(dest_exporter), sampling_draw=lambda: 0.0 - ) + fan_out: Final = TenantFanOutSpanProcessor(processor_factory=lambda _d: _EndsAStragglerOnExport(dest_exporter)) provider = TracerProvider() provider.add_span_processor(fan_out) def run(): - set_request_destinations((arize_destination(arize_success_sampling_rate="0.5"),)) + set_request_destinations((arize_destination(arize_success_sampling_rate="1.0"),)) tracer = get_tracer(provider, "litellm") root = tracer.start_span("POST /v1/chat/completions", context=trace_api.Context()) tracer.start_span("chat gpt-4", context=trace_api.set_span_in_context(root)).end() @@ -1495,20 +1769,20 @@ class TestTenantSampling: assert {s.name for s in dest_exporter.get_finished_spans()} == {"chat gpt-4", "embed ada"} - def test_a_zero_rate_drops_even_a_draw_of_zero(self): + def test_a_zero_rate_drops_a_trace_a_half_rate_keeps(self): dest_exporter = InMemorySpanExporter() - provider = self._fan_out(dest_exporter, draw=lambda: 0.0) + provider: Final = self._fan_out(dest_exporter, trace_id=KEPT_AT_HALF) self._run(provider, (arize_destination(arize_success_sampling_rate="0.0"),)) assert dest_exporter.get_finished_spans() == () - def test_the_draw_is_compared_to_the_rate_the_way_the_legacy_callback_compares_it(self): + def test_the_trace_id_is_read_against_the_rate(self): dropped_exporter, kept_exporter = InMemorySpanExporter(), InMemorySpanExporter() - destinations = (arize_destination(arize_success_sampling_rate="0.2"),) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.5"),) - self._run(self._fan_out(dropped_exporter, draw=lambda: 0.3), destinations) - self._run(self._fan_out(kept_exporter, draw=lambda: 0.2), destinations) + self._run(self._fan_out(dropped_exporter, trace_id=DROPPED_AT_HALF), destinations) + self._run(self._fan_out(kept_exporter, trace_id=KEPT_AT_HALF), destinations) assert dropped_exporter.get_finished_spans() == () assert {s.name for s in kept_exporter.get_finished_spans()} == self.NAMES @@ -1527,6 +1801,39 @@ class TestTenantSampling: assert all(names == self.NAMES for names in by_trace.values()) assert 0 < len(by_trace) < 64 + @pytest.mark.parametrize( + ("rates", "failed"), + [({"arize_success_sampling_rate": "0.5"}, False), ({"arize_error_sampling_rate": "0.5"}, True)], + ids=["success-rate", "error-rate"], + ) + def test_two_processes_serving_one_trace_reach_the_same_verdict(self, rates: Mapping[str, str], failed: bool): + """Two pods behind a load balancer each serve one call of a trace the caller + continued with its ``traceparent``: the team's space gets both calls or neither, + and a rate of 0.5 over UUID-shaped trace ids keeps some traces and drops others.""" + exporters: Final = (InMemorySpanExporter(), InMemorySpanExporter()) + pods: Final = tuple(self._fan_out(exporter) for exporter in exporters) + destinations: Final = (arize_destination(**rates),) + trace_ids: Final = tuple(uuid.uuid5(uuid.NAMESPACE_URL, f"pod/{index}").int for index in range(32)) + + def run(provider: TracerProvider): + set_request_destinations(destinations) + tracer: Final = get_tracer(provider, "litellm") + for trace_id in trace_ids: + with tracer.start_as_current_span("POST /v1/chat/completions", context=caller_context(trace_id)): + with tracer.start_as_current_span("chat gpt-4") as span: + if failed: + span.set_status(Status(StatusCode.ERROR, "upstream 500")) + + for provider in pods: + in_fresh_context(run, provider) + + first, second = ( + {s.context.trace_id for s in exporter.get_finished_spans() if s.context is not None} + for exporter in exporters + ) + assert first == second + assert 0 < len(first) < len(trace_ids) + def test_in_additive_mode_the_operators_own_copy_is_not_sampled(self, monkeypatch): """The rates are the team's setting for the team's space; the operator's backbone keeps every span.""" @@ -1547,7 +1854,7 @@ class TestTenantSampling: assert {s.name for s in dest_exporter.get_finished_spans()} == self.NAMES - def test_two_views_of_one_exporter_each_draw_at_their_own_rate(self): + def test_two_views_of_one_exporter_each_answer_to_their_own_rate(self): """Two destinations to the same space with different rates share an exporter, not a verdict: the 1.0 view gets the tree once and the 0.0 view adds nothing.""" dest_exporter = InMemorySpanExporter() @@ -1582,7 +1889,7 @@ class TestTenantSampling: @pytest.mark.parametrize("bad", ["abc", "-0.1", "1.5", "nan"]) def test_an_unusable_rate_exports_rather_than_dropping(self, bad): dest_exporter = InMemorySpanExporter() - provider = self._fan_out(dest_exporter, draw=lambda: 0.99) + provider: Final = self._fan_out(dest_exporter, trace_id=DROPPED_AT_HALF) self._run(provider, (arize_destination(arize_success_sampling_rate=bad),)) From 1ecc782986162c8359d38eb6b22a213fd7c8a0e0 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 26 Oct 2026 14:49:50 -0400 Subject: [PATCH 055/155] fix(vertex_ai): reuse rerank credential cache (#48430) --- litellm/utils.py | 7 +- .../test_vertex_ai_rerank_transformation.py | 86 +++++++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index e2820f9c682..4315ae426cf 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9571,7 +9571,7 @@ class ProviderConfigManager: return get_nvidia_nim_rerank_config(model) elif litellm.LlmProviders.VERTEX_AI == provider: - return litellm.VertexAIRerankConfig() + return ProviderConfigManager._get_vertex_ai_rerank_config_cached() elif litellm.LlmProviders.FIREWORKS_AI == provider: return litellm.FireworksAIRerankConfig() elif litellm.LlmProviders.VOYAGE == provider: @@ -9590,6 +9590,11 @@ class ProviderConfigManager: return get_dashscope_family_rerank_config(provider.value) return litellm.CohereRerankConfig() + @staticmethod + @lru_cache(maxsize=1) + def _get_vertex_ai_rerank_config_cached() -> BaseRerankConfig: + return litellm.VertexAIRerankConfig() + @staticmethod def get_provider_decisions_config(model: str, provider: LlmProviders) -> BaseDecisionsConfig | None: if provider == LlmProviders.PERPLEXITY: diff --git a/tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_transformation.py b/tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_transformation.py index 630b2e1eb34..8abd012f9d8 100644 --- a/tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_transformation.py +++ b/tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_transformation.py @@ -5,13 +5,19 @@ Based on the test patterns from other rerank providers and the current Vertex AI import json import os +from datetime import datetime, timedelta +from typing import Final from unittest.mock import MagicMock, patch import httpx import pytest +import responses +import respx +import litellm from litellm.llms.vertex_ai.rerank.transformation import VertexAIRerankConfig from litellm.types.rerank import RerankResponse +from litellm.utils import ProviderConfigManager class TestVertexAIRerankTransform: @@ -646,3 +652,83 @@ class TestVertexAIRerankTransform: second_call = mock_ensure_access_token.call_args_list[1] assert first_call.kwargs["credentials"] == "path/to/credentials.json" assert second_call.kwargs["credentials"] == "path/to/credentials.json" + + +@pytest.fixture +def clear_vertex_rerank_cache(request: pytest.FixtureRequest) -> None: + ProviderConfigManager._get_vertex_ai_rerank_config_cached.cache_clear() + request.addfinalizer( + ProviderConfigManager._get_vertex_ai_rerank_config_cached.cache_clear + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("clear_vertex_rerank_cache") +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +async def test_rerank_reuses_scoped_credentials_and_refreshes_expired_tokens( + is_async: bool, +) -> None: + now: Final = datetime(2025, 1, 1) + credentials: Final = tuple( + json.dumps( + { + "type": "authorized_user", + "client_id": identity, + "client_secret": "synthetic-client-secret", + "refresh_token": f"synthetic-refresh-{identity}", + } + ) + for identity in ("first-principal", "second-principal") + ) + with ( + patch("litellm.disable_aiohttp_transport", True), + patch("google.auth._helpers.utcnow", return_value=now) as clock, + responses.RequestsMock(assert_all_requests_are_fired=False) as token_http, + respx.mock as rank_http, + ): + for token in ("initial", "other-credential", "other-project", "renewed"): + token_http.post( + "https://oauth2.googleapis.com/token", + json={"access_token": token, "expires_in": 3600, "token_type": "Bearer"}, + ) + rank_http.post( + url__regex=r"https://discoveryengine\.googleapis\.com/v1/projects/[^/]+/" + r"locations/global/rankingConfigs/default_ranking_config:rank" + ).respond(200, json={"records": [{"id": "0", "score": 0.9}]}) + + async def rank( + credential: str, project: str, expected_token: str, refresh_count: int + ) -> None: + params: Final = { + "model": "vertex_ai/semantic-ranker-default@latest", + "query": f"Query for {expected_token}", + "documents": ["A relevant document"], + "vertex_credentials": credential, + "vertex_project": project, + "caching": False, + "num_retries": 0, + } + result: Final = ( + await litellm.arerank(**params) + if is_async + else litellm.rerank(**params) + ) + assert isinstance(result, RerankResponse) + assert result.results[0]["index"] == 0 + assert len(token_http.calls) == refresh_count + request: Final = rank_http.calls.last.request + assert request.headers["Authorization"] == f"Bearer {expected_token}" + assert request.headers["X-Goog-User-Project"] == project + assert str(request.url) == ( + f"https://discoveryengine.googleapis.com/v1/projects/{project}/" + "locations/global/rankingConfigs/default_ranking_config:rank" + ) + assert json.loads(request.content)["query"] == params["query"] + + await rank(credentials[0], "first-project", "initial", 1) + await rank(credentials[0], "first-project", "initial", 1) + await rank(credentials[1], "first-project", "other-credential", 2) + await rank(credentials[0], "second-project", "other-project", 3) + await rank(credentials[0], "first-project", "initial", 3) + clock.return_value = now + timedelta(hours=2) + await rank(credentials[0], "first-project", "renewed", 4) From 85871f791c12ca381b53dad8cbb7c8a47076ef4c Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 26 Oct 2026 14:50:31 -0400 Subject: [PATCH 056/155] fix(ui): render string inputs in general settings (#48431) --- .../general_settings.integration.test.tsx | 36 +++++++++++++++++++ .../_components/general_settings.tsx | 9 +++++ 2 files changed, 45 insertions(+) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx index 35434830d98..d7dfb8151fc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx @@ -80,6 +80,42 @@ describe("GeneralSettings General tab", () => { vi.mocked(deleteConfigFieldSetting).mockClear(); }); + it.each([ + ["maximum_spend_logs_retention_period", null, "7d"], + ["maximum_health_check_retention_period", "30d", "14d"], + ["maximum_daily_tag_spend_retention_period", undefined, "90d"], + ["maximum_spend_logs_cleanup_run_budget", "5m", "10m"], + ["maximum_spend_logs_cleanup_batch_timeout", "", "45s"], + ["mcp_client_id_header", "x-client-id", "X-MCP-Client.ID"], + ])("displays, updates and resets the String setting %s", async (fieldName, initialValue, updatedValue) => { + vi.mocked(getGeneralSettingsCall).mockResolvedValue([ + ...SETTINGS_FIXTURE, + { + field_name: fieldName, + field_type: "String", + field_value: initialValue, + field_description: "string setting", + stored_in_db: initialValue != null, + field_default_value: null, + }, + ]); + const user = userEvent.setup(); + renderWithProviders(); + await user.click(screen.getByRole("tab", { name: "General" })); + const input = await screen.findByRole("textbox", { name: fieldName }); + expect(input).toHaveValue(initialValue ?? ""); + fireEvent.change(input, { target: { value: updatedValue } }); + const row = screen.getByRole("row", { name: new RegExp(fieldName) }); + await user.click(within(row).getByRole("button", { name: "Update" })); + expect(vi.mocked(updateConfigFieldSetting).mock.calls).toEqual([["token", fieldName, updatedValue]]); + expect(within(row).getByText("In DB")).toBeInTheDocument(); + const resetIcon = row.querySelectorAll("td")[3].querySelector("svg"); + expect(resetIcon).not.toBeNull(); + await user.click(resetIcon as unknown as Element); + expect(vi.mocked(deleteConfigFieldSetting).mock.calls).toEqual([["token", fieldName]]); + expect(input).toHaveValue(""); + }); + it("updates max_ui_session_budget with its own value, not the value at its filtered index", async () => { const user = userEvent.setup(); renderWithProviders(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx index 5d28adb7ef3..67712d78a13 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx @@ -68,6 +68,15 @@ const SettingValueEditor: React.FC<{ /> ); } + if (setting.field_type === "String") { + return ( + onChange(setting.field_name, event.target.value)} + /> + ); + } if (setting.field_type === "Integer") { return ( Date: Mon, 26 Oct 2026 12:07:58 -0700 Subject: [PATCH 057/155] fix(guardrails): exclude litellm_metadata from pass-through guardrail scan text (#48276) * fix(guardrails): exclude litellm_metadata from pass-through guardrail scan text The pass-through guardrail translation json-dumps the request body as the scan text when no request_fields are configured. The unified guardrail hook injects litellm_metadata into that body first, so the dump carried the caller's key metadata. Exclude it alongside metadata and litellm_logging_obj; vendors still receive it through request_data. * test(guardrails): type the pass-through scan-text tests per AGENTS.md --- .../guardrail_translation/handler.py | 2 +- .../test_passthrough_guardrail_scan_text.py | 153 ++++++++++++++++++ .../guardrail_translation/test_handler.py | 16 ++ .../test_unified_guardrail.py | 33 ++++ 4 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 tests/integration/observability/test_passthrough_guardrail_scan_text.py diff --git a/litellm/llms/pass_through/guardrail_translation/handler.py b/litellm/llms/pass_through/guardrail_translation/handler.py index 3ce8d7ddad5..dea8bcb73a4 100644 --- a/litellm/llms/pass_through/guardrail_translation/handler.py +++ b/litellm/llms/pass_through/guardrail_translation/handler.py @@ -90,7 +90,7 @@ class PassThroughEndpointHandler(BaseTranslation): payload_to_check: Final = { k: v for k, v in without_server_streaming_classification(data).items() - if not k.startswith("_") and k not in ("metadata", "litellm_logging_obj") + if not k.startswith("_") and k not in ("metadata", "litellm_metadata", "litellm_logging_obj") } verbose_proxy_logger.debug("PassThroughEndpointHandler: Using full payload for guardrail") return safe_dumps(payload_to_check) diff --git a/tests/integration/observability/test_passthrough_guardrail_scan_text.py b/tests/integration/observability/test_passthrough_guardrail_scan_text.py new file mode 100644 index 00000000000..7437f696d4c --- /dev/null +++ b/tests/integration/observability/test_passthrough_guardrail_scan_text.py @@ -0,0 +1,153 @@ +import json +import uuid +from collections.abc import Iterator, Mapping +from contextlib import ExitStack +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import pytest +import yaml +from integration._support.anthropic_sse import message_json +from integration._support.client import Gateway +from integration._support.openai_wire import chat_reply +from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +GENERIC: Final = "passthrough-scan-generic" +AIRS: Final = "passthrough-scan-airs" +OPENAI_ROUTE: Final = "/openai/chat/completions" +ANTHROPIC_ROUTE: Final = "/anthropic/v1/messages" +ROUTE_BODIES: Final[Mapping[str, dict[str, JsonValue]]] = MappingProxyType( + { + OPENAI_ROUTE: {"model": "gpt-5.6", "messages": [{"role": "user", "content": "Say hi in one word"}]}, + ANTHROPIC_ROUTE: { + "model": "claude-sonnet-5-5", + "max_tokens": 16, + "messages": [{"role": "user", "content": "Say hi in one word"}], + }, + } +) +_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml" +_JSON: Final = TypeAdapter(dict[str, JsonValue]) + + +def _generic_verdict(_request: Request) -> Reply: + return Reply(body=json.dumps({"action": "NONE"}).encode()) + + +def _airs_verdict(_request: Request) -> Reply: + return Reply(body=json.dumps({"action": "allow", "category": "benign", "scan_id": "s", "report_id": "r"}).encode()) + + +def _openai_reply(_request: Request) -> Reply: + return chat_reply("chatcmpl-passthrough-scan", "gpt-5.6", "Hi", stream=False) + + +def _anthropic_reply(_request: Request) -> Reply: + return Reply(body=message_json("msg_passthrough_scan", "claude-sonnet-5-5", "Hi")) + + +def _parsed_texts(texts: JsonValue, vendor_body: dict[str, JsonValue]) -> list[JsonValue]: + assert isinstance(texts, list), vendor_body + return [json.loads(text) if isinstance(text, str) else text for text in texts] + + +def _scanned_payloads(guardrail: str, vendor_body: dict[str, JsonValue]) -> list[JsonValue]: + if guardrail == GENERIC: + return _parsed_texts(vendor_body["texts"], vendor_body) + contents: Final = vendor_body["contents"] + assert isinstance(contents, list), vendor_body + return _parsed_texts([content.get("prompt") for content in contents if isinstance(content, dict)], vendor_body) + + +@dataclass(frozen=True, slots=True) +class PassThroughRig: + owned: OwnedProxy + vendors: Mapping[str, Wire] + upstreams: Mapping[str, Wire] + + +@pytest.fixture(scope="module") +def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[PassThroughRig]: + directory: Final = tmp_path_factory.mktemp("passthrough-guardrail-scan") + with ExitStack() as stack: + vendors: Final = MappingProxyType( + { + GENERIC: stack.enter_context(wire_server(_generic_verdict)), + AIRS: stack.enter_context(wire_server(_airs_verdict)), + } + ) + upstreams: Final = MappingProxyType( + { + OPENAI_ROUTE: stack.enter_context(wire_server(_openai_reply)), + ANTHROPIC_ROUTE: stack.enter_context(wire_server(_anthropic_reply)), + } + ) + config: Final = _JSON.validate_python(yaml.safe_load(_CONFIG.read_text())) + path: Final = directory / "passthrough-guardrail-scan.yaml" + path.write_text( + yaml.safe_dump( + { + **config, + "guardrails": [ + { + "guardrail_name": GENERIC, + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": False, + "api_base": vendors[GENERIC].url, + "api_key": "synthetic-guardrail-key", + }, + }, + { + "guardrail_name": AIRS, + "litellm_params": { + "guardrail": "panw_prisma_airs", + "mode": "pre_call", + "default_on": False, + "api_base": vendors[AIRS].url, + "api_key": "synthetic-airs-key", + "profile_name": "integration", + }, + }, + ], + } + ) + ) + environment: Final = { + "OPENAI_API_BASE": upstreams[OPENAI_ROUTE].url, + "OPENAI_API_KEY": "synthetic-openai-key", + "ANTHROPIC_API_BASE": upstreams[ANTHROPIC_ROUTE].url, + "ANTHROPIC_API_KEY": "synthetic-anthropic-key", + } + owned: Final = stack.enter_context(owned_proxy_process(session_gateway, directory, environment, config=path)) + yield PassThroughRig(owned, vendors, upstreams) + + +@pytest.mark.parametrize("guardrail", [GENERIC, AIRS]) +@pytest.mark.parametrize("route", [OPENAI_ROUTE, ANTHROPIC_ROUTE]) +def test_pass_through_guardrail_scans_the_client_body_without_the_callers_key_metadata( + rig: PassThroughRig, route: str, guardrail: str +) -> None: + gateway: Final = rig.owned.gateway + with gateway.scenario() as scenario: + team: Final = scenario.team() + user: Final = scenario.user( + user_email=f"passthrough-scan-{uuid.uuid4().hex}@example.com", user_role="internal_user", teams=[team] + ) + key: Final = scenario.key(user_id=user, team_id=team) + body: Final = {**ROUTE_BODIES[route], "guardrails": [guardrail]} + + response: Final = gateway.request("POST", route, body, key=key) + + assert response.status_code == 200, response.text + scanned: Final = [ + _scanned_payloads(guardrail, _JSON.validate_json(request.body)) + for request in rig.vendors[guardrail].drain() + ] + assert scanned == [[body]], scanned + assert len(rig.upstreams[route].drain()) == 1 diff --git a/tests/unit/llms/pass_through/guardrail_translation/test_handler.py b/tests/unit/llms/pass_through/guardrail_translation/test_handler.py index 0dba0d7b90f..7cbabb95931 100644 --- a/tests/unit/llms/pass_through/guardrail_translation/test_handler.py +++ b/tests/unit/llms/pass_through/guardrail_translation/test_handler.py @@ -19,6 +19,22 @@ def test_full_payload_guardrail_text_excludes_the_server_streaming_marker(): assert json.loads(text) == body, text +def test_full_payload_guardrail_text_excludes_the_injected_litellm_metadata(): + body: Final = {"model": "m", "messages": [{"role": "user", "content": "hi"}], "guardrails": ["g"]} + litellm_metadata: Final = { + "user_api_key_token": "hashed-key", + "user_api_key_user_email": "someone@example.com", + "user_api_key_user_role": "proxy_admin", + } + + text: Final = PassThroughEndpointHandler()._extract_text_for_guardrail( + {**body, "litellm_metadata": litellm_metadata}, + None, + ) + + assert json.loads(text) == body, text + + @pytest.mark.parametrize( "marker", [ diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 3efbe6475b2..a132ce8eada 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -3,6 +3,7 @@ import asyncio import contextlib import io +import json import logging from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator from types import SimpleNamespace @@ -35,6 +36,7 @@ from litellm.llms.openai.chat.guardrail_translation.handler import ( from litellm.llms.openai.responses.guardrail_translation.handler import ( OpenAIResponsesHandler, ) +from litellm.llms.pass_through.guardrail_translation.handler import PassThroughEndpointHandler from litellm.proxy._experimental.mcp_server.guardrail_translation.handler import ( MCPGuardrailTranslationHandler, ) @@ -3053,3 +3055,34 @@ async def test_pre_call_hook_skips_attachment_scan_when_guardrail_has_none(): ) assert len(guardrail.apply_calls) == 1 + + +@pytest.mark.parametrize("hook", ["async_pre_call_hook", "async_moderation_hook"]) +@pytest.mark.asyncio +async def test_pass_through_guardrail_text_is_the_client_body_without_the_injected_key_metadata( + monkeypatch: pytest.MonkeyPatch, hook: str +) -> None: + _patch_translation_mappings(monkeypatch, {CallTypes.pass_through: PassThroughEndpointHandler}) + guardrail: Final = RecordingGuardrail() + body: Final = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "Say hi in one word"}], + "guardrails": ["recording-guardrail"], + } + data: Final = {**body, "guardrail_to_apply": guardrail} + user_api_key_dict: Final = UserAPIKeyAuth( + token="hashed-key", user_email="someone@example.com", team_id="team-1", user_role="proxy_admin" + ) + + if hook == "async_pre_call_hook": + await UnifiedLLMGuardrails().async_pre_call_hook( + user_api_key_dict=user_api_key_dict, cache=DualCache(), data=data, call_type=CallTypes.pass_through.value + ) + else: + await UnifiedLLMGuardrails().async_moderation_hook( + data=data, user_api_key_dict=user_api_key_dict, call_type=CallTypes.pass_through.value + ) + + texts: Final = [call["inputs"]["texts"] for call in guardrail.apply_calls] + assert [[json.loads(text) for text in call_texts] for call_texts in texts] == [[body]], texts + assert data["litellm_metadata"]["user_api_key_user_email"] == "someone@example.com", data["litellm_metadata"] From dab39d7402cb1e772045c5a702abe74f69f66970 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 12:09:14 -0700 Subject: [PATCH 058/155] fix(azure): set azure_ai/Microsoft-Decision-1 max_input_tokens to 32768 (#48434) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 3 ++- model_prices_and_context_window.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0a85d8a00cd..af2b4655d6f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76439,9 +76439,10 @@ "azure_ai/Microsoft-Decision-1": { "input_cost_per_token": 4.2e-08, "litellm_provider": "azure_ai", + "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'" + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure" }, "gpt-rosalind-discovery": { "cache_read_input_token_cost": 5e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0a85d8a00cd..af2b4655d6f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76439,9 +76439,10 @@ "azure_ai/Microsoft-Decision-1": { "input_cost_per_token": 4.2e-08, "litellm_provider": "azure_ai", + "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'" + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure" }, "gpt-rosalind-discovery": { "cache_read_input_token_cost": 5e-07, From c69e82f538690c40f16e7aa6519c7e05579a55ab Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 12:18:19 -0700 Subject: [PATCH 059/155] chore(cost-map): add openai batch and ultrafast 12-hour cache write rates from the pricing page (#48433) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...model_prices_and_context_window_backup.json | 18 ++++++++++++++++++ model_prices_and_context_window.json | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index af2b4655d6f..6ec1bd29beb 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -36840,10 +36840,14 @@ "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_12hr": 2.5e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 2.5e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 0.0001, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": 0.0003, + "cache_creation_input_token_cost_above_12hr_batches": 1.25e-05, "cache_creation_input_token_cost_above_12hr_flex": 1.25e-05, "cache_creation_input_token_cost_above_12hr_priority": 5e-05, + "cache_creation_input_token_cost_above_12hr_ultrafast": 0.00015, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05, @@ -36931,8 +36935,10 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_12hr": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-06, "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -37014,8 +37020,10 @@ "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_12hr": 2.5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 2.5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-06, + "cache_creation_input_token_cost_above_12hr_batches": 1.25e-07, "cache_creation_input_token_cost_above_12hr_flex": 1.25e-07, "cache_creation_input_token_cost_above_12hr_priority": 5e-07, "cache_creation_input_token_cost_decisions": 1.25e-07, @@ -37186,8 +37194,10 @@ "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_12hr": 1e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 1e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05, + "cache_creation_input_token_cost_above_12hr_batches": 5e-06, "cache_creation_input_token_cost_above_12hr_flex": 5e-06, "cache_creation_input_token_cost_above_12hr_priority": 2e-05, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, @@ -37268,8 +37278,10 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_12hr": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-06, "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -37349,8 +37361,10 @@ "cache_creation_input_token_cost": 2.5e-07, "cache_creation_input_token_cost_above_12hr": 5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-06, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-07, "cache_creation_input_token_cost_above_12hr_flex": 2.5e-07, "cache_creation_input_token_cost_above_12hr_priority": 1e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, @@ -75046,10 +75060,14 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_12hr": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": 6e-05, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-06, "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, "cache_creation_input_token_cost_above_12hr_priority": 1e-05, + "cache_creation_input_token_cost_above_12hr_ultrafast": 3e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index af2b4655d6f..6ec1bd29beb 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -36840,10 +36840,14 @@ "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_12hr": 2.5e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 2.5e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 0.0001, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": 0.0003, + "cache_creation_input_token_cost_above_12hr_batches": 1.25e-05, "cache_creation_input_token_cost_above_12hr_flex": 1.25e-05, "cache_creation_input_token_cost_above_12hr_priority": 5e-05, + "cache_creation_input_token_cost_above_12hr_ultrafast": 0.00015, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05, @@ -36931,8 +36935,10 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_12hr": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-06, "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -37014,8 +37020,10 @@ "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_12hr": 2.5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 2.5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-06, + "cache_creation_input_token_cost_above_12hr_batches": 1.25e-07, "cache_creation_input_token_cost_above_12hr_flex": 1.25e-07, "cache_creation_input_token_cost_above_12hr_priority": 5e-07, "cache_creation_input_token_cost_decisions": 1.25e-07, @@ -37186,8 +37194,10 @@ "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_12hr": 1e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 1e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05, + "cache_creation_input_token_cost_above_12hr_batches": 5e-06, "cache_creation_input_token_cost_above_12hr_flex": 5e-06, "cache_creation_input_token_cost_above_12hr_priority": 2e-05, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, @@ -37268,8 +37278,10 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_12hr": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-06, "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -37349,8 +37361,10 @@ "cache_creation_input_token_cost": 2.5e-07, "cache_creation_input_token_cost_above_12hr": 5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-06, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-07, "cache_creation_input_token_cost_above_12hr_flex": 2.5e-07, "cache_creation_input_token_cost_above_12hr_priority": 1e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, @@ -75046,10 +75060,14 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_12hr": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": 6e-05, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-06, "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, "cache_creation_input_token_cost_above_12hr_priority": 1e-05, + "cache_creation_input_token_cost_above_12hr_ultrafast": 3e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, From 52c11bf38393b81732b687a41fc3ebfe2740c4a1 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:39:48 +0000 Subject: [PATCH 060/155] feat(bedrock): add global. and us. openai.gpt-6.1-terra rows (#48441) * feat(bedrock): add global. and us. openai.gpt-6.1-terra rows * chore: rerun ci after flaky event loop timing test --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 78 +++++++++++++++++++ model_prices_and_context_window.json | 78 +++++++++++++++++++ 2 files changed, 156 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6ec1bd29beb..3bb8b9f8029 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -78773,5 +78773,83 @@ "supports_response_schema": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true + }, + "global.openai.gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, + "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1e-07, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_272k_tokens": 2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-06, + "output_cost_per_token_above_272k_tokens": 7.5e-06, + "source": "https://developers.openai.com/api/docs/models/gpt-6.1-terra", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_bedrock_converse_tool_result_images": false, + "supports_bedrock_runtime_chat_completions_response_format": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "us.openai.gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-06, + "cache_read_input_token_cost": 5.5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-07, + "input_cost_per_token": 1.1e-06, + "input_cost_per_token_above_272k_tokens": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-06, + "output_cost_per_token_above_272k_tokens": 8.25e-06, + "source": "https://developers.openai.com/api/docs/models/gpt-6.1-terra", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_bedrock_converse_tool_result_images": false, + "supports_bedrock_runtime_chat_completions_response_format": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6ec1bd29beb..3bb8b9f8029 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -78773,5 +78773,83 @@ "supports_response_schema": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true + }, + "global.openai.gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, + "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1e-07, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_272k_tokens": 2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-06, + "output_cost_per_token_above_272k_tokens": 7.5e-06, + "source": "https://developers.openai.com/api/docs/models/gpt-6.1-terra", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_bedrock_converse_tool_result_images": false, + "supports_bedrock_runtime_chat_completions_response_format": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "us.openai.gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-06, + "cache_read_input_token_cost": 5.5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-07, + "input_cost_per_token": 1.1e-06, + "input_cost_per_token_above_272k_tokens": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-06, + "output_cost_per_token_above_272k_tokens": 8.25e-06, + "source": "https://developers.openai.com/api/docs/models/gpt-6.1-terra", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_bedrock_converse_tool_result_images": false, + "supports_bedrock_runtime_chat_completions_response_format": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true } } From f96200bcea3a08adcd08ad4eb5ad169d226cdda5 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 26 Oct 2026 15:56:13 -0400 Subject: [PATCH 061/155] fix(vertex_ai): bound shared credential cache retention (#48442) --- litellm/llms/vertex_ai/vertex_llm_base.py | 141 +++++++----------- .../llms/vertex_ai/test_vertex_llm_base.py | 125 +++++++++++++++- .../test_google_secret_manager.py | 3 +- 3 files changed, 182 insertions(+), 87 deletions(-) diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 6cff9729189..0cdd92bb071 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -7,12 +7,14 @@ Handles Authentication and generating request urls for Vertex AI and Google AI S import asyncio import json import threading +from collections import OrderedDict from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final, Literal, Protocol from urllib.parse import urlparse import litellm from litellm._logging import verbose_logger +from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -70,16 +72,20 @@ class _VertexCredentialsObject(Protocol): def refresh(self, request: object) -> None: ... +_VertexCredentialCacheKey = tuple[str | None, str | None] + + class VertexBase: def __init__(self) -> None: super().__init__() self.access_token: str | None = None self.refresh_token: str | None = None self._credentials: GoogleCredentialsObject | None = None - self._credentials_project_mapping: dict[ - tuple[VERTEX_CREDENTIALS_TYPES | None, str | None], + self._credentials_project_mapping: OrderedDict[ + _VertexCredentialCacheKey, tuple[_VertexCredentialsObject, str | None], - ] = {} + ] = OrderedDict() + self._credentials_cache_lock = threading.RLock() self.project_id: str | None = None self.async_handler: AsyncHTTPHandler | None = None # Per-credential-key asyncio.Lock for single-flight async refresh. @@ -91,10 +97,10 @@ class VertexBase: # each lock; the entry is pruned when the count reaches zero, so the # dict stays bounded even in long-running high-cardinality deployments # without depending on any private asyncio internals. - self._async_refresh_locks: dict[tuple, asyncio.Lock] = {} - self._async_refresh_lock_refcounts: dict[tuple, int] = {} + self._async_refresh_locks: dict[_VertexCredentialCacheKey, asyncio.Lock] = {} + self._async_refresh_lock_refcounts: dict[_VertexCredentialCacheKey, int] = {} # Tracks in-flight background refresh tasks to avoid duplicate refreshes. - self._background_refresh_tasks: dict[tuple, asyncio.Task] = {} + self._background_refresh_tasks: dict[_VertexCredentialCacheKey, asyncio.Task] = {} # Protects the sync get_access_token refresh path. # Use RLock so that the reauthentication retry path (which calls # back into get_access_token while still holding the lock) can @@ -378,7 +384,7 @@ class VertexBase: with self._sync_refresh_lock: credentials.refresh(Request()) - def _acquire_async_refresh_lock(self, credential_cache_key: tuple) -> asyncio.Lock: + def _acquire_async_refresh_lock(self, credential_cache_key: _VertexCredentialCacheKey) -> asyncio.Lock: """Increment the refcount and return the lock for ``credential_cache_key``. Every call must be paired with ``_release_async_refresh_lock`` once the @@ -391,7 +397,7 @@ class VertexBase: ) return lock - def _release_async_refresh_lock(self, credential_cache_key: tuple, lock: asyncio.Lock) -> None: + def _release_async_refresh_lock(self, credential_cache_key: _VertexCredentialCacheKey, lock: asyncio.Lock) -> None: """Decrement the refcount and drop the lock entry when it reaches zero. Must be called only after the caller has released ``lock`` (i.e. once @@ -409,7 +415,7 @@ class VertexBase: def _try_get_cached_token( self, - credential_cache_key: tuple, + credential_cache_key: _VertexCredentialCacheKey, project_id: str | None, ) -> tuple[str, str] | None: """ @@ -432,7 +438,7 @@ class VertexBase: def _try_get_usable_cached_token( self, - credential_cache_key: tuple, + credential_cache_key: _VertexCredentialCacheKey, project_id: str | None, ) -> tuple[str, str, "TokenState", _VertexCredentialsObject, str | None] | None: """ @@ -458,19 +464,37 @@ class VertexBase: return creds.token, resolved_project, token_state, creds, cached_project_id def _unpack_cached_credentials( - self, credential_cache_key: tuple + self, credential_cache_key: _VertexCredentialCacheKey ) -> tuple[_VertexCredentialsObject | None, str | None]: """ Return (credentials, project_id) from the cache, or (None, None) if not cached. Handles both tuple and legacy cache formats. """ - if credential_cache_key not in self._credentials_project_mapping: - return None, None - cached_entry: Final = self._credentials_project_mapping[credential_cache_key] + with self._credentials_cache_lock: + cached_entry: Final = self._credentials_project_mapping.get(credential_cache_key) + if cached_entry is None: + return None, None + self._credentials_project_mapping.move_to_end(credential_cache_key) if isinstance(cached_entry, tuple): return cached_entry return cached_entry, cached_entry.quota_project_id or getattr(cached_entry, "project_id", None) + def _cache_credentials( + self, + credential_cache_key: _VertexCredentialCacheKey, + credentials: _VertexCredentialsObject, + project_id: str | None, + *, + replace: bool = True, + ) -> None: + with self._credentials_cache_lock: + if not replace and credential_cache_key in self._credentials_project_mapping: + return + self._credentials_project_mapping[credential_cache_key] = (credentials, project_id) + self._credentials_project_mapping.move_to_end(credential_cache_key) + while len(self._credentials_project_mapping) > max(0, DEFAULT_MAX_LRU_CACHE_SIZE): + self._credentials_project_mapping.popitem(last=False) + def _get_token_state(self, credentials: _VertexCredentialsObject) -> "TokenState": """ Return the token state using google-auth's TokenState enum. @@ -494,7 +518,7 @@ class VertexBase: self, credentials: VERTEX_CREDENTIALS_TYPES | None, project_id: str | None, - credential_cache_key: tuple, + credential_cache_key: _VertexCredentialCacheKey, ) -> tuple[_VertexCredentialsObject, str | None]: """Load credentials via load_auth (in thread) and cache the result.""" try: @@ -507,17 +531,12 @@ class VertexBase: raise if _credentials is None: raise ValueError("Could not resolve credentials") - self._credentials_project_mapping[credential_cache_key] = ( - _credentials, - credential_project_id, - ) + self._cache_credentials(credential_cache_key, _credentials, credential_project_id) return _credentials, credential_project_id async def _background_refresh_credentials( self, credentials: _VertexCredentialsObject, - credential_cache_key: tuple, - credential_project_id: str | None, ) -> None: """ Refresh credentials in the background without blocking the calling request. @@ -528,25 +547,13 @@ class VertexBase: try: verbose_logger.debug("Background proactive credential refresh") await asyncify(self.refresh_auth)(credentials) - # Only update the cache if it still points at the credentials - # object we just refreshed. The per-key async lock is not held - # here, so a concurrent INVALID path may have already replaced - # this entry (e.g. via _handle_reauthentication_async, which - # creates a fresh credentials object). In that case our write - # would clobber the newer entry with a stale reference. - cached_creds, _ = self._unpack_cached_credentials(credential_cache_key) - if cached_creds is credentials: - self._credentials_project_mapping[credential_cache_key] = ( - credentials, - credential_project_id, - ) except Exception: verbose_logger.debug( "Background credential refresh failed, will retry on next request", exc_info=True, ) - async def _await_in_flight_background_refresh(self, credential_cache_key: tuple) -> None: + async def _await_in_flight_background_refresh(self, credential_cache_key: _VertexCredentialCacheKey) -> None: """Wait for an in-flight background refresh to finish, if any. google-auth's ``Credentials.refresh()`` is not safe to invoke @@ -568,8 +575,7 @@ class VertexBase: def _schedule_background_refresh( self, credentials: _VertexCredentialsObject, - credential_cache_key: tuple, - credential_project_id: str | None, + credential_cache_key: _VertexCredentialCacheKey, ) -> None: """Kick off a single background refresh for ``credential_cache_key``. @@ -581,9 +587,7 @@ class VertexBase: if existing is not None and not existing.done(): return self._background_refresh_tasks.pop(credential_cache_key, None) - task: Final = asyncio.create_task( - self._background_refresh_credentials(credentials, credential_cache_key, credential_project_id) - ) + task: Final = asyncio.create_task(self._background_refresh_credentials(credentials)) def _drop_background_refresh_task(_fut: asyncio.Future[None]) -> None: if self._background_refresh_tasks.get(credential_cache_key) is _fut: @@ -796,7 +800,7 @@ class VertexBase: self, credentials: VERTEX_CREDENTIALS_TYPES | None, project_id: str | None, - credential_cache_key: tuple, + credential_cache_key: _VertexCredentialCacheKey, error: Exception, ) -> tuple[str, str]: """ @@ -822,8 +826,8 @@ class VertexBase: ) # Clear the cached credentials - if credential_cache_key in self._credentials_project_mapping: - del self._credentials_project_mapping[credential_cache_key] + with self._credentials_cache_lock: + self._credentials_project_mapping.pop(credential_cache_key, None) # Retry once with _retry_reauth=True to prevent infinite recursion try: @@ -846,7 +850,7 @@ class VertexBase: self, credentials: VERTEX_CREDENTIALS_TYPES | None, project_id: str | None, - credential_cache_key: tuple, + credential_cache_key: _VertexCredentialCacheKey, error: Exception, ) -> tuple[str, str]: """ @@ -856,7 +860,8 @@ class VertexBase: "Handling async reauthentication for project_id: %s. Clearing cache and retrying once.", project_id ) - self._credentials_project_mapping.pop(credential_cache_key, None) + with self._credentials_cache_lock: + self._credentials_project_mapping.pop(credential_cache_key, None) try: ( @@ -875,10 +880,7 @@ class VertexBase: # references the OLD credentials object we just replaced, and # leaving it would force the next request to do a redundant # refresh/reauth before realizing the cached creds are stale. - self._credentials_project_mapping[resolved_cache_key] = ( - _credentials, - credential_project_id, - ) + self._cache_credentials(resolved_cache_key, _credentials, credential_project_id) if _credentials.token is None or not isinstance(_credentials.token, str): raise ValueError( @@ -924,21 +926,11 @@ class VertexBase: # Convert dict credentials to string for caching cache_credentials: Final = json.dumps(credentials) if isinstance(credentials, dict) else credentials credential_cache_key: Final = (cache_credentials, project_id) - _credentials: _VertexCredentialsObject | None = None - verbose_logger.debug("Checking cached credentials for project_id: %s", project_id) - if credential_cache_key in self._credentials_project_mapping: + _credentials, credential_project_id = self._unpack_cached_credentials(credential_cache_key) + if _credentials is not None: verbose_logger.debug("Cached credentials found for project_id: %s.", project_id) - # Retrieve both credentials and cached project_id - cached_entry: Final = self._credentials_project_mapping[credential_cache_key] - verbose_logger.debug("cached_entry: %s", cached_entry) - if isinstance(cached_entry, tuple): - _credentials, credential_project_id = cached_entry - else: - # Backward compatibility with old cache format - _credentials = cached_entry - credential_project_id = _credentials.quota_project_id or getattr(_credentials, "project_id", None) verbose_logger.debug( "Using cached credentials for project_id: %s", credential_project_id, @@ -963,10 +955,7 @@ class VertexBase: f"Could not resolve credentials - either dynamically or from environment, for project_id: {project_id}" ) # Cache the project_id and credentials from load_auth result (resolved project_id) - self._credentials_project_mapping[credential_cache_key] = ( - _credentials, - credential_project_id, - ) + self._cache_credentials(credential_cache_key, _credentials, credential_project_id) ## VALIDATE CREDENTIALS verbose_logger.debug("Validating credentials") @@ -974,11 +963,7 @@ class VertexBase: project_id = credential_project_id # Update cache with resolved project_id for future lookups resolved_cache_key: Final = (cache_credentials, project_id) - if resolved_cache_key not in self._credentials_project_mapping: - self._credentials_project_mapping[resolved_cache_key] = ( - _credentials, - credential_project_id, - ) + self._cache_credentials(resolved_cache_key, _credentials, credential_project_id, replace=False) # Check if credentials are None before accessing attributes if _credentials is None: @@ -991,10 +976,6 @@ class VertexBase: try: verbose_logger.debug("Credentials expired, refreshing") self.refresh_auth(_credentials) - self._credentials_project_mapping[credential_cache_key] = ( - _credentials, - credential_project_id, - ) except Exception as e: # if refresh fails, it's possible the user has re-authenticated via `gcloud auth application-default login` # in this case, we should try to reload the credentials by clearing the cache and retrying @@ -1043,9 +1024,9 @@ class VertexBase: # callers on the lock just to schedule that refresh. usable: Final = self._try_get_usable_cached_token(credential_cache_key, project_id) if usable is not None: - cached_token, resolved_project, token_state, creds, cached_project_id = usable + cached_token, resolved_project, token_state, creds, _cached_project_id = usable if token_state == TokenState.STALE: - self._schedule_background_refresh(creds, credential_cache_key, cached_project_id) + self._schedule_background_refresh(creds, credential_cache_key) return cached_token, resolved_project # === SLOW PATH (per-key lock) === @@ -1075,10 +1056,7 @@ class VertexBase: # reauth that only repopulated the unresolved key), which # would force the next request through an unnecessary # refresh/reauth cycle. - self._credentials_project_mapping[resolved_cache_key] = ( - _credentials, - credential_project_id, - ) + self._cache_credentials(resolved_cache_key, _credentials, credential_project_id) # Use google-auth's token_state to decide refresh strategy: # - STALE: token is usable but within REFRESH_THRESHOLD (3:45) of @@ -1098,7 +1076,6 @@ class VertexBase: self._schedule_background_refresh( _credentials, credential_cache_key, - credential_project_id, ) return current_token, project_id @@ -1117,10 +1094,6 @@ class VertexBase: try: verbose_logger.debug("Credentials expired, refreshing") await asyncify(self.refresh_auth)(_credentials) - self._credentials_project_mapping[credential_cache_key] = ( - _credentials, - credential_project_id, - ) except Exception as e: if "Reauthentication is needed" in str(e): verbose_logger.debug("Reauthentication needed, clearing cache and retrying") diff --git a/tests/unit/llms/vertex_ai/test_vertex_llm_base.py b/tests/unit/llms/vertex_ai/test_vertex_llm_base.py index dd2b86b6f81..fa332cef5d8 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_llm_base.py +++ b/tests/unit/llms/vertex_ai/test_vertex_llm_base.py @@ -6,13 +6,17 @@ import os import pathlib import re import traceback -from typing import Final +from datetime import datetime, timedelta +from threading import Event +from typing import Final, Literal from unittest.mock import MagicMock, call, patch import google.auth.exceptions import pytest +import requests +import responses -from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE, DEFAULT_MAX_RECURSE_DEPTH import litellm @@ -2589,3 +2593,120 @@ def test_get_access_token_reraises_malformed_service_account_credentials_without assert [record.getMessage().split(". Error: ")[0] for record in caplog.records] == [ "Failed to load vertex credentials. Check to see if credentials containing partial/invalid information" ] + + +def _authorized_user_cache_credentials(identity: str) -> str: + return json.dumps( + { + "type": "authorized_user", + "client_id": identity, + "client_secret": "synthetic-secret", + "refresh_token": identity, + "quota_project_id": identity, + } + ) + + +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("inferred_project", [False, True], ids=["explicit", "inferred"]) +@pytest.mark.asyncio +async def test_credential_cache_bounds_retention_and_reloads_least_recently_used( + is_async: bool, inferred_project: bool +) -> None: + vertex: Final = VertexBase() + limit: Final = max(0, DEFAULT_MAX_LRU_CACHE_SIZE) + with ( + patch("google.auth._helpers.utcnow", return_value=datetime(2025, 1, 1)), + responses.RequestsMock() as oauth, + ): + def token_response(request: requests.PreparedRequest) -> tuple[int, dict[str, str], str]: + return 200, {}, json.dumps({"access_token": f"token-{len(oauth.calls) + 1}", "expires_in": 3600}) + + oauth.add_callback(responses.POST, "https://oauth2.googleapis.com/token", callback=token_response) + + async def access(index: int, infer: bool = False) -> tuple[str, str]: + identity: Final = f"project-{index}" + credentials: Final = _authorized_user_cache_credentials(identity) + project: Final = None if infer else identity + result: Final = ( + await vertex.get_access_token_async(credentials, project) + if is_async + else vertex.get_access_token(credentials, project) + ) + assert result[1] == identity + assert len(vertex._credentials_project_mapping) <= limit + return result + + if limit == 0: + assert await access(0, inferred_project) == ("token-1", "project-0") + assert await access(0, inferred_project) == ("token-2", "project-0") + return + + for index in range(limit): + assert await access(index) == (f"token-{index + 1}", f"project-{index}") + assert await access(0) == ("token-1", "project-0") + assert len(oauth.calls) == limit + assert await access(limit, inferred_project) == (f"token-{limit + 1}", f"project-{limit}") + assert len(oauth.calls) == limit + 1 + if limit > (2 if inferred_project else 1): + assert await access(0) == ("token-1", "project-0") + assert len(oauth.calls) == limit + 1 + evicted: Final = 1 if limit > 1 else 0 + assert await access(evicted) == (f"token-{limit + 2}", f"project-{evicted}") + assert len(oauth.calls) == limit + 2 + + +@pytest.mark.parametrize("mode", ["sync", "async", "background"]) +@pytest.mark.skipif(DEFAULT_MAX_LRU_CACHE_SIZE < 2, reason="Eviction during refresh requires two retained entries") +@pytest.mark.asyncio +async def test_evicted_credentials_finish_refresh_without_being_readmitted( + mode: Literal["sync", "async", "background"], +) -> None: + vertex: Final = VertexBase() + limit: Final = DEFAULT_MAX_LRU_CACHE_SIZE + now: Final = datetime(2025, 1, 1) + refresh_started: Final = Event() + release_refresh: Final = Event() + donor: Final = _authorized_user_cache_credentials("donor") + target: Final = _authorized_user_cache_credentials("target") + with patch("google.auth._helpers.utcnow", return_value=now) as clock, responses.RequestsMock() as oauth: + def token_response(request: requests.PreparedRequest) -> tuple[int, dict[str, str], str]: + body: Final = request.body.decode() if isinstance(request.body, bytes) else request.body + assert isinstance(body, str) + is_target: Final = "refresh_token=target" in body + if is_target and len(oauth.calls) == limit: + refresh_started.set() + assert release_refresh.wait(5), "OAuth refresh was not released" + return 200, {}, json.dumps( + {"access_token": f"token-{len(oauth.calls) + 1}", "expires_in": 3600 if is_target else 86400} + ) + + oauth.add_callback(responses.POST, "https://oauth2.googleapis.com/token", callback=token_response) + assert vertex.get_access_token(donor, None) == ("token-1", "donor") + assert vertex.get_access_token(donor, None) == ("token-1", "donor") + for index in range(limit - 2): + assert vertex.get_access_token(donor, f"filler-{index}") == (f"token-{index + 2}", f"filler-{index}") + assert vertex.get_access_token(target, "target") == (f"token-{limit}", "target") + clock.return_value = now + timedelta(seconds=3550 if mode == "background" else 7200) + refresh: Final = asyncio.create_task( + asyncio.to_thread(vertex.get_access_token, target, "target") + if mode == "sync" + else vertex.get_access_token_async(target, "target") + ) + try: + assert await asyncio.to_thread(refresh_started.wait, 5), "OAuth refresh did not start" + for index in range(limit - 2): + assert vertex.get_access_token(donor, f"filler-{index}") == (f"token-{index + 2}", f"filler-{index}") + assert vertex.get_access_token(donor, None) == ("token-1", "donor") + assert len(vertex._credentials_project_mapping) == limit + assert (target, "target") not in vertex._credentials_project_mapping + finally: + release_refresh.set() + completed: Final = await refresh + await asyncio.gather(*tuple(vertex._background_refresh_tasks.values())) + expected_token: Final = f"token-{limit if mode == 'background' else limit + 1}" + assert completed == (expected_token, "target") + assert len(oauth.calls) == limit + 1 + assert (target, "target") not in vertex._credentials_project_mapping + assert vertex.get_access_token(target, "target") == (f"token-{limit + 2}", "target") + assert len(vertex._credentials_project_mapping) == limit diff --git a/tests/unit/secret_managers/test_google_secret_manager.py b/tests/unit/secret_managers/test_google_secret_manager.py index 2051be08f7c..7effde4fb30 100644 --- a/tests/unit/secret_managers/test_google_secret_manager.py +++ b/tests/unit/secret_managers/test_google_secret_manager.py @@ -1,6 +1,7 @@ import base64 import binascii import json +from collections import OrderedDict from dataclasses import dataclass from typing import Final @@ -120,7 +121,7 @@ def manager(monkeypatch: pytest.MonkeyPatch) -> GoogleSecretManager: monkeypatch.setattr(litellm, "secret_manager_client", None) monkeypatch.setattr(litellm, "_key_management_system", None) monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings()) - monkeypatch.setattr(litellm.vertex_chat_completion, "_credentials_project_mapping", {}) + monkeypatch.setattr(litellm.vertex_chat_completion, "_credentials_project_mapping", OrderedDict()) monkeypatch.setattr(google.auth, "default", _default_credentials) monkeypatch.setenv("GOOGLE_SECRET_MANAGER_PROJECT_ID", PROJECT_ID) monkeypatch.delenv("GOOGLE_SECRET_MANAGER_ALWAYS_READ_SECRET_MANAGER", raising=False) From 9cb102b72159d4809256d4f87b3a31b3551e3364 Mon Sep 17 00:00:00 2001 From: shrey-berri Date: Mon, 26 Oct 2026 13:41:15 -0700 Subject: [PATCH 062/155] chore(deps): remove expired boto3 botocore and grpcio overrides (#48447) --- pyproject.toml | 5 ----- uv.lock | 3 --- 2 files changed, 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b15eb042959..88bb0b13d91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -379,11 +379,6 @@ exclude-newer = "3 days" [tool.uv.exclude-newer-package] # TODO: ryan, remove after 2026-10-23, once certifi 2026.10.19 (drops the compromised eMudhra root) clears the 3-day window certifi = "2026-10-20T03:00:00Z" -# TODO(shrey): remove after 2026-10-25, once grpcio 1.84.1 (drops the compromised eMudhra roots from roots.pem) clears the 3-day window -grpcio = "2026-10-22T15:00:00Z" -# TODO(shrey): remove after 2026-10-24, once boto3/botocore 1.43.117 (drops the compromised eMudhra roots from cacert.pem) clear the 3-day window -boto3 = "2026-10-21T19:31:00Z" -botocore = "2026-10-21T19:31:00Z" # TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window brotli = "2026-10-22T14:01:00Z" # TODO: ryan, remove after 2026-10-24, once mlflow 3.17.2 (GHSA-wqmh-qccw-pqwf) clears the 3-day window diff --git a/uv.lock b/uv.lock index 60bcb367e0f..fc4155524d7 100644 --- a/uv.lock +++ b/uv.lock @@ -15,11 +15,8 @@ exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for exclude-newer-span = "P3D" [options.exclude-newer-package] -boto3 = "2026-10-21T19:31:00Z" -botocore = "2026-10-21T19:31:00Z" brotli = "2026-10-22T14:01:00Z" certifi = "2026-10-20T03:00:00Z" -grpcio = "2026-10-22T15:00:00Z" mlflow = "2026-10-21T03:00:00Z" mlflow-skinny = "2026-10-21T03:00:00Z" mlflow-tracing = "2026-10-21T03:00:00Z" From 768960610bc7e77e854bdb79e14a47268f1c04fb Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:41:58 +0000 Subject: [PATCH 063/155] feat(cost-map): add baseten elephant-1 and mammoth-1 decision models (#48453) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...model_prices_and_context_window_backup.json | 18 ++++++++++++++++++ model_prices_and_context_window.json | 18 ++++++++++++++++++ tests/unit/test_utils.py | 1 + 3 files changed, 37 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3bb8b9f8029..4ee3bcb831d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -78851,5 +78851,23 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true + }, + "baseten/decision-models/baseten-elephant-9b-latest": { + "input_cost_per_token": 0.0, + "litellm_provider": "baseten", + "max_input_tokens": 16384, + "mode": "decisions", + "output_cost_per_token": 0.0, + "source": "https://inference.baseten.co/v1/models", + "supports_vision": true + }, + "baseten/decision-models/baseten-mammoth-27b-latest": { + "input_cost_per_token": 0.0, + "litellm_provider": "baseten", + "max_input_tokens": 16384, + "mode": "decisions", + "output_cost_per_token": 0.0, + "source": "https://inference.baseten.co/v1/models", + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3bb8b9f8029..4ee3bcb831d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -78851,5 +78851,23 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true + }, + "baseten/decision-models/baseten-elephant-9b-latest": { + "input_cost_per_token": 0.0, + "litellm_provider": "baseten", + "max_input_tokens": 16384, + "mode": "decisions", + "output_cost_per_token": 0.0, + "source": "https://inference.baseten.co/v1/models", + "supports_vision": true + }, + "baseten/decision-models/baseten-mammoth-27b-latest": { + "input_cost_per_token": 0.0, + "litellm_provider": "baseten", + "max_input_tokens": 16384, + "mode": "decisions", + "output_cost_per_token": 0.0, + "source": "https://inference.baseten.co/v1/models", + "supports_vision": true } } diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 55b7bc78a11..b9baef4056a 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1086,6 +1086,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "chat", "completion", "container", + "decisions", "image_edit", "embedding", "evaluation", From 3beee3cfe97f3b3c04458ba5ff2612c8bd9358a7 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Mon, 26 Oct 2026 13:42:01 -0700 Subject: [PATCH 064/155] fix(elevenlabs): map wav and pcm TTS formats to 24kHz output formats (#48455) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../text_to_speech/transformation.py | 4 +-- .../test_transformation.py | 2 +- ...levenlabs_text_to_speech_transformation.py | 28 +++++++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/litellm/llms/elevenlabs/text_to_speech/transformation.py b/litellm/llms/elevenlabs/text_to_speech/transformation.py index f910ef8c057..23ccf41c9da 100644 --- a/litellm/llms/elevenlabs/text_to_speech/transformation.py +++ b/litellm/llms/elevenlabs/text_to_speech/transformation.py @@ -56,9 +56,9 @@ class ElevenLabsTextToSpeechConfig(BaseTextToSpeechConfig): # Response format mappings from OpenAI to ElevenLabs FORMAT_MAPPINGS = { "mp3": "mp3_44100_128", - "pcm": "pcm_44100", + "pcm": "pcm_24000", "opus": "opus_48000_128", - # ElevenLabs does not support WAV, AAC, or FLAC formats. + "wav": "wav_24000", } ELEVENLABS_QUERY_PARAMS_KEY = "__elevenlabs_query_params__" diff --git a/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py b/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py index 86252a1a95d..f55644eccae 100644 --- a/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py +++ b/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py @@ -314,4 +314,4 @@ class TestElevenLabsTextToSpeechTransformation: ) assert voice_id in url - assert "output_format=pcm_44100" in url + assert "output_format=pcm_24000" in url diff --git a/tests/unit/llms/elevenlabs/test_elevenlabs_text_to_speech_transformation.py b/tests/unit/llms/elevenlabs/test_elevenlabs_text_to_speech_transformation.py index d05371d7df9..88f0ae1d524 100644 --- a/tests/unit/llms/elevenlabs/test_elevenlabs_text_to_speech_transformation.py +++ b/tests/unit/llms/elevenlabs/test_elevenlabs_text_to_speech_transformation.py @@ -55,3 +55,31 @@ def test_speech_keeps_an_internal_prefixed_kwarg_out_of_the_elevenlabs_request(r sent: Final = json.loads(respx_mock.calls[0].request.content) assert "_litellm_undeclared_sentinel" not in sent, sent assert sent["text"] == "hi" + + +@pytest.mark.parametrize( + ("response_format", "expected_output_format"), + [("wav", "wav_24000"), ("pcm", "pcm_24000")], +) +def test_speech_maps_wav_and_pcm_to_elevenlabs_output_formats( + respx_mock: respx.MockRouter, + response_format: str, + expected_output_format: str, +) -> None: + api_base: Final = "http://localhost:12346" + mock_route: Final = respx_mock.post(url__regex=rf"{api_base}/v1/text-to-speech/.*").mock( + return_value=httpx.Response(status_code=200, content=b"audio", headers={"content-type": "audio/mpeg"}) + ) + + litellm.speech( + model="elevenlabs/eleven_multilingual_v2", + input="hi", + voice="21m00Tcm4TlvDq8ikWAM", + response_format=response_format, + api_base=api_base, + api_key="fake_elevenlabs_api_key", + ) + + assert mock_route.called + request_url: Final = httpx.URL(str(respx_mock.calls[0].request.url)) + assert request_url.params["output_format"] == expected_output_format From c843cee062e08113f5afa101b0b6879824dd9025 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:55:51 +0000 Subject: [PATCH 065/155] feat(pricing): add github_copilot/gpt-6.1-terra (#48450) * feat(pricing): add github_copilot/gpt-6.1-terra * fix(pricing): raise github_copilot gpt-6 max_input_tokens to 922000 --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 32 ++++++++++++++++--- model_prices_and_context_window.json | 32 ++++++++++++++++--- 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4ee3bcb831d..b7e0c881455 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -33698,7 +33698,7 @@ "input_cost_per_token": 1e-05, "input_cost_per_token_above_272k_tokens": 2e-05, "litellm_provider": "github_copilot", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -33722,7 +33722,7 @@ "input_cost_per_token": 1e-07, "input_cost_per_token_above_272k_tokens": 2e-07, "litellm_provider": "github_copilot", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -33746,7 +33746,7 @@ "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "litellm_provider": "github_copilot", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -33770,7 +33770,7 @@ "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "litellm_provider": "github_copilot", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -78869,5 +78869,29 @@ "output_cost_per_token": 0.0, "source": "https://inference.baseten.co/v1/models", "supports_vision": true + }, + "github_copilot/gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, + "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1e-07, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_272k_tokens": 2e-06, + "litellm_provider": "github_copilot", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 5e-06, + "output_cost_per_token_above_272k_tokens": 7.5e-06, + "source": "https://github.blog/changelog/2026-10-26-gpt-6-1-terra-in-github-copilot", + "supported_endpoints": [ + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4ee3bcb831d..b7e0c881455 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -33698,7 +33698,7 @@ "input_cost_per_token": 1e-05, "input_cost_per_token_above_272k_tokens": 2e-05, "litellm_provider": "github_copilot", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -33722,7 +33722,7 @@ "input_cost_per_token": 1e-07, "input_cost_per_token_above_272k_tokens": 2e-07, "litellm_provider": "github_copilot", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -33746,7 +33746,7 @@ "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "litellm_provider": "github_copilot", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -33770,7 +33770,7 @@ "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "litellm_provider": "github_copilot", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -78869,5 +78869,29 @@ "output_cost_per_token": 0.0, "source": "https://inference.baseten.co/v1/models", "supports_vision": true + }, + "github_copilot/gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, + "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1e-07, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_272k_tokens": 2e-06, + "litellm_provider": "github_copilot", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 5e-06, + "output_cost_per_token_above_272k_tokens": 7.5e-06, + "source": "https://github.blog/changelog/2026-10-26-gpt-6-1-terra-in-github-copilot", + "supported_endpoints": [ + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true } } From b7a2226f438499509730c9fe649beb026b7cb4fe Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 26 Oct 2026 16:58:51 -0400 Subject: [PATCH 066/155] test(proxy): make tokenizer offload check deterministic (#48458) --- ...test_proxy_server_endpoints_and_startup.py | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 14196d4a1ac..a859ce96c70 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -15207,21 +15207,23 @@ async def test_token_counter_keeps_the_event_loop_free_during_a_huggingface_coun assert_loop_stayed_free(took, lags) -async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeypatch): - from litellm.rust_bridge._native import Tokenizer +async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeypatch: pytest.MonkeyPatch) -> None: + from queue import SimpleQueue + from threading import Thread, current_thread + + from tokenizers import Tokenizer from litellm import Router - from tests.unit.litellm_core_utils.event_loop_lag import assert_loop_stayed_free, timed_with_loop_lags - claude_tokenizer: Final = litellm.utils.select_tokenizer("claude-fable-5")["tokenizer"] + claude_tokenizer: Final = Tokenizer.from_str(litellm.utils.claude_json_str) + loop_thread: Final = current_thread() + loader_threads: Final[SimpleQueue[Thread]] = SimpleQueue() - class SlowHubTokenizer: - @staticmethod - def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> Tokenizer: - time.sleep(0.3) - return claude_tokenizer + def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> Tokenizer: + loader_threads.put(current_thread()) + return claude_tokenizer - monkeypatch.setattr("litellm.rust_bridge.tokenizer.from_pretrained", SlowHubTokenizer.from_pretrained) + monkeypatch.setattr("tokenizers.Tokenizer", types.SimpleNamespace(from_pretrained=from_pretrained)) monkeypatch.setattr( "litellm.proxy.proxy_server.llm_router", Router( @@ -15237,15 +15239,16 @@ async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeyp ), ) - response, took, lags = await timed_with_loop_lags( - lambda: proxy_server_module.token_counter( + litellm.utils._select_custom_tokenizer_helper.cache_clear() + try: + response: Final = await proxy_server_module.token_counter( TokenCountRequest(model="self-hosted", prompt="count me off the loop") ) - ) - - assert response.tokenizer_type == "huggingface_tokenizer" - assert response.total_tokens > 0 - assert_loop_stayed_free(took, lags) + assert loader_threads.get_nowait() is not loop_thread + assert response.tokenizer_type == "huggingface_tokenizer" + assert response.total_tokens > 0 + finally: + litellm.utils._select_custom_tokenizer_helper.cache_clear() async def test_token_counter_loads_a_custom_tokenizer_once_per_identifier_revision_and_token(monkeypatch): From 6d9344535093b6ad5a1b246af449c20d1db46004 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 26 Oct 2026 17:09:19 -0400 Subject: [PATCH 067/155] fix(proxy): preserve limiter state across SSE keepalive tasks (#48452) --- litellm/proxy/common_request_processing.py | 3 + .../hooks/test_parallel_request_limiter_v3.py | 136 +++++++++++++++++- 2 files changed, 138 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index cec176ac559..61cf4d696a1 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -119,6 +119,7 @@ from litellm.proxy.common_utils.sse_keepalive import ( ) from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression +from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash from litellm.proxy.native_compaction import with_proxy_compaction_executor from litellm.proxy.route_llm_request import ( route_request, @@ -1416,6 +1417,8 @@ async def open_sse_before_first_byte( if interval is None: return await produce_response + # Share admission state with stream cleanup before the producer copies this context. + get_or_create_request_stash() produce_task: Final = asyncio.ensure_future(produce_response) await asyncio.wait((produce_task,), timeout=interval) if produce_task.done(): diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 1ec505362dd..345f37f1db4 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -7,7 +7,7 @@ import contextvars import logging import sys import time -from collections.abc import Iterator, Sequence +from collections.abc import AsyncGenerator, Iterator, Sequence from contextlib import contextmanager from datetime import datetime, timedelta, timezone from typing import Any, Dict, Final, List, Optional @@ -4644,6 +4644,140 @@ async def test_success_event_releases_parallel_slot_v3(monkeypatch): ) +_SSE_KEEPALIVE_CALLBACK_TIME: Final = datetime(2026, 1, 1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("after_ping", [False, True], ids=["fast", "pinged"]) +@pytest.mark.parametrize("fails", [False, True], ids=["success", "failure"]) +@pytest.mark.parametrize("preadmitted", [False, True], ids=["producer-admission", "existing-admission"]) +async def test_sse_keepalive_releases_parallel_capacity( + after_ping: bool, fails: bool, preadmitted: bool +) -> None: + from fastapi.responses import StreamingResponse + + from litellm.proxy.common_request_processing import open_sse_before_first_byte + from litellm.proxy.common_utils.sse_keepalive import SSE_COMMENT_PING_BYTES + + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache)) + api_key: Final = hash_token("sk-keepalive-release") + auth: Final = UserAPIKeyAuth(api_key=api_key, max_parallel_requests=1) + counter: Final = f"{{api_key:{api_key}}}:max_parallel_requests" + + async def request(call_id: str) -> None: + data: Final = {"model": "test-model", "litellm_call_id": call_id} + upstream_answers: Final = asyncio.Event() + if not after_ping: + upstream_answers.set() + + async def admit() -> None: + await handler.async_pre_call_hook( + user_api_key_dict=auth, cache=cache, data=data, call_type="acompletion" + ) + + async def failure(exc: Exception) -> None: + await handler.async_post_call_failure_hook( + request_data=data, original_exception=exc, user_api_key_dict=auth + ) + + async def stream() -> AsyncGenerator[bytes, None]: + yield b"data: [DONE]\n\n" + await handler.async_log_success_event( + kwargs={ + "litellm_call_id": call_id, + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + }, + response_obj=ModelResponse(usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2)), + start_time=_SSE_KEEPALIVE_CALLBACK_TIME, + end_time=_SSE_KEEPALIVE_CALLBACK_TIME, + ) + + async def produce() -> StreamingResponse: + if not preadmitted: + await admit() + await upstream_answers.wait() + if fails: + raise HTTPException(status_code=502, detail="upstream failed") + return StreamingResponse(stream()) + + if preadmitted: + await admit() + response_call: Final = open_sse_before_first_byte( + produce(), 0.001 if after_ping else 60.0, on_late_failure=failure + ) + if fails and not after_ping: + with pytest.raises(HTTPException) as rejected: + await response_call + assert rejected.value.status_code == 502 + await failure(rejected.value) + return + response: Final = await response_call + first: Final = await anext(response.body_iterator) + if after_ping: + upstream_answers.set() + rest: Final = tuple([chunk async for chunk in response.body_iterator]) + assert first == (SSE_COMMENT_PING_BYTES if after_ping else b"data: [DONE]\n\n") + if fails: + error_frame: Final = rest[-2] + assert isinstance(error_frame, bytes) and b"upstream failed" in error_frame + assert rest[-1] == b"data: [DONE]\n\n" + elif after_ping: + assert rest[-1] == b"data: [DONE]\n\n" + else: + assert rest == () + + for index in range(3): + await contextvars.Context().run(asyncio.create_task, request(f"keepalive-{index}")) + assert handler._gauge_in_flight_from_cache_value(await cache.async_get_cache(counter)) == 0 + + +@pytest.mark.asyncio +async def test_sse_keepalive_rejects_overlap_and_reuses_completed_capacity() -> None: + from fastapi.responses import StreamingResponse + + from litellm.proxy.common_request_processing import open_sse_before_first_byte + + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache)) + api_key: Final = hash_token("sk-keepalive-overlap") + auth: Final = UserAPIKeyAuth(api_key=api_key, max_parallel_requests=1) + admitted: Final = asyncio.Event() + finish: Final = asyncio.Event() + + async def request(call_id: str) -> None: + async def stream() -> AsyncGenerator[bytes, None]: + await finish.wait() + yield b"data: [DONE]\n\n" + await handler.async_post_call_success_hook( + data={"litellm_call_id": call_id}, user_api_key_dict=auth, response=ModelResponse() + ) + + async def produce() -> StreamingResponse: + await handler.async_pre_call_hook( + user_api_key_dict=auth, + cache=cache, + data={"model": "test-model", "litellm_call_id": call_id}, + call_type="acompletion", + ) + admitted.set() + return StreamingResponse(stream()) + + response: Final = await open_sse_before_first_byte(produce(), 60.0) + assert tuple([chunk async for chunk in response.body_iterator]) == (b"data: [DONE]\n\n",) + + first: Final = contextvars.Context().run(asyncio.create_task, request("first")) + await admitted.wait() + try: + with pytest.raises(HTTPException) as rejected: + await contextvars.Context().run(asyncio.create_task, request("overlapping")) + assert rejected.value.status_code == 429 + finally: + finish.set() + await first + await contextvars.Context().run(asyncio.create_task, request("after-completion")) + + @pytest.mark.asyncio async def test_read_only_gauge_check_counts_without_acquiring_v3(): """ From ccd783d907d8b8a88add48c35732371e8701febf Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 26 Oct 2026 17:16:15 -0400 Subject: [PATCH 068/155] fix(proxy): reject negative RPM and TPM request limits (#48444) --- litellm/proxy/_types.py | 12 ++- .../management_endpoints/key_requests.py | 4 + .../management_endpoints/user_requests.py | 4 + tests/unit/proxy/test__types.py | 83 ++++++++++++++++++- 4 files changed, 98 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 2b01e02bb06..db433f07f3b 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1766,8 +1766,10 @@ class BudgetNewRequest(LiteLLMPydanticObjectBase): max_parallel_requests: int | None = Field( default=None, description="Max concurrent requests allowed for this budget id." ) - tpm_limit: int | None = Field(default=None, description="Max tokens per minute, allowed for this budget id.") - rpm_limit: int | None = Field(default=None, description="Max requests per minute, allowed for this budget id.") + tpm_limit: int | None = Field(default=None, ge=0, description="Max tokens per minute, allowed for this budget id.") + rpm_limit: int | None = Field( + default=None, ge=0, description="Max requests per minute, allowed for this budget id." + ) tpd_limit: int | None = Field( default=None, description="Max tokens per day, charged by batch submissions, allowed for this budget id." ) @@ -1878,6 +1880,8 @@ RouterSettingsDict = Annotated[ class NewTeamRequest(TeamBase): + tpm_limit: int | None = Field(default=None, ge=0) + rpm_limit: int | None = Field(default=None, ge=0) router_settings: RouterSettingsDict | None = None model_aliases: dict | None = None model_max_budget: GenericBudgetConfigType | None = Field( @@ -1957,8 +1961,8 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase): team_alias: str | None = None organization_id: str | None = None metadata: dict | None = None - tpm_limit: int | None = None - rpm_limit: int | None = None + tpm_limit: int | None = Field(default=None, ge=0) + rpm_limit: int | None = Field(default=None, ge=0) tpd_limit: int | None = None max_budget: float | None = None soft_budget: float | None = None diff --git a/litellm/types/proxy/management_endpoints/key_requests.py b/litellm/types/proxy/management_endpoints/key_requests.py index baa02b68d98..172357d8ac4 100644 --- a/litellm/types/proxy/management_endpoints/key_requests.py +++ b/litellm/types/proxy/management_endpoints/key_requests.py @@ -55,6 +55,8 @@ class LiteLLMKeyType(str, enum.Enum): class GenerateKeyRequest(KeyRequestBase): + tpm_limit: int | None = Field(default=None, ge=0) + rpm_limit: int | None = Field(default=None, ge=0) soft_budget: float | None = None send_invite_email: bool | None = None key_type: LiteLLMKeyType | None = Field( @@ -82,6 +84,8 @@ def _is_mapping(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: class UpdateKeyRequest(KeyRequestBase): + tpm_limit: int | None = Field(default=None, ge=0) + rpm_limit: int | None = Field(default=None, ge=0) # Note: the defaults of all Params here MUST BE NONE # else they will get overwritten duration: str | None = None diff --git a/litellm/types/proxy/management_endpoints/user_requests.py b/litellm/types/proxy/management_endpoints/user_requests.py index 49dda5995bc..e53dc4152c3 100644 --- a/litellm/types/proxy/management_endpoints/user_requests.py +++ b/litellm/types/proxy/management_endpoints/user_requests.py @@ -16,6 +16,8 @@ class NewUserRequestTeam(LiteLLMPydanticObjectBase): class NewUserRequest(GenerateRequestBase): + tpm_limit: int | None = Field(default=None, ge=0) + rpm_limit: int | None = Field(default=None, ge=0) max_budget: float | None = None user_email: str | None = None user_alias: str | None = None @@ -47,6 +49,8 @@ class NewUserRequest(GenerateRequestBase): class UpdateUserRequestNoUserIDorEmail(GenerateRequestBase): # shared with BulkUpdateUserRequest + tpm_limit: int | None = Field(default=None, ge=0) + rpm_limit: int | None = Field(default=None, ge=0) # repr=False keeps the plaintext out of management-endpoint alerts, which str() the request model password: str | None = Field(default=None, repr=False) spend: float | None = None diff --git a/tests/unit/proxy/test__types.py b/tests/unit/proxy/test__types.py index 68bfe4b34e5..76ca49362b4 100644 --- a/tests/unit/proxy/test__types.py +++ b/tests/unit/proxy/test__types.py @@ -2,23 +2,35 @@ import json from typing import Final import pytest -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError from litellm.proxy._types import ( ROLES_WITHIN_ORG, + BudgetNewRequest, ChangePasswordRequest, GenerateKeyRequest, + GenerateKeyResponse, KeyRequest, LiteLLM_AuditLogs, + LiteLLM_BudgetTable, + LiteLLM_EndUserTable, LiteLLM_TeamMembership, + LiteLLM_TeamTable, + LiteLLM_UserTable, LitellmUserRoles, + NewCustomerRequest, NewMCPServerRequest, + NewTeamRequest, NewUserRequest, + NewUserResponse, OrganizationMemberUpdateRequest, + RegenerateKeyRequest, ResetSpendRequest, UpdateKeyRequest, UpdateMCPServerRequest, + UpdateTeamRequest, UpdateUserRequest, + UpdateUserRequestNoUserIDorEmail, UserAPIKeyAuth, ) @@ -213,6 +225,75 @@ def test_a_max_budget_that_was_actually_sent_is_kept(sent): assert GenerateKeyRequest(max_budget=sent).max_budget == sent +RATE_LIMIT_REQUESTS: Final = ( + (GenerateKeyRequest, {}), + (RegenerateKeyRequest, {}), + (UpdateKeyRequest, {"key": "sk-test"}), + (NewUserRequest, {}), + (UpdateUserRequestNoUserIDorEmail, {}), + (UpdateUserRequest, {"user_id": "user-test"}), + (NewTeamRequest, {}), + (UpdateTeamRequest, {"team_id": "team-test"}), + (BudgetNewRequest, {}), + (NewCustomerRequest, {"user_id": "customer-test"}), +) + + +@pytest.mark.parametrize(("request_model", "identifiers"), RATE_LIMIT_REQUESTS) +@pytest.mark.parametrize("field", ("rpm_limit", "tpm_limit")) +@pytest.mark.parametrize("value", (-1, -100)) +def test_management_requests_reject_negative_rate_limits( + request_model: type[BaseModel], identifiers: dict[str, str], field: str, value: int +) -> None: + with pytest.raises(ValidationError) as error: + request_model.model_validate({**identifiers, field: value}) + assert error.value.errors()[0]["loc"] == (field,) + + +@pytest.mark.parametrize(("request_model", "identifiers"), RATE_LIMIT_REQUESTS) +@pytest.mark.parametrize("value", (None, 0, 100)) +def test_management_requests_preserve_explicit_rate_limits( + request_model: type[BaseModel], identifiers: dict[str, str], value: int | None +) -> None: + limits: Final = {"rpm_limit": value, "tpm_limit": value} + request: Final = request_model.model_validate({**identifiers, **limits}) + assert request.model_dump(include={"rpm_limit", "tpm_limit"}, exclude_unset=True) == limits + + +@pytest.mark.parametrize(("request_model", "identifiers"), RATE_LIMIT_REQUESTS) +def test_management_requests_leave_omitted_rate_limits_unset( + request_model: type[BaseModel], identifiers: dict[str, str] +) -> None: + request: Final = request_model.model_validate({**identifiers}) + assert request.model_dump(include={"rpm_limit", "tpm_limit"}) == {"rpm_limit": None, "tpm_limit": None} + assert request.model_dump(include={"rpm_limit", "tpm_limit"}, exclude_unset=True) == {} + + +@pytest.mark.parametrize(("response_model", "identifiers"), ( + (GenerateKeyResponse, {"key": "sk-test"}), + (NewUserResponse, {"key": "sk-test"}), + (LiteLLM_TeamTable, {"team_id": "team-test"}), + (LiteLLM_UserTable, {"user_id": "user-test"}), + (LiteLLM_BudgetTable, {"budget_id": "budget-test"}), +)) +def test_existing_negative_rate_limits_remain_readable( + response_model: type[BaseModel], identifiers: dict[str, str] +) -> None: + limits: Final = {"rpm_limit": -1, "tpm_limit": -100} + response: Final = response_model.model_validate({**identifiers, **limits}) + assert response.model_dump(include={"rpm_limit", "tpm_limit"}) == limits + + +def test_existing_customer_negative_rate_limits_remain_readable() -> None: + customer: Final = LiteLLM_EndUserTable.model_validate({ + "user_id": "customer-test", "blocked": False, + "litellm_budget_table": {"rpm_limit": -1, "tpm_limit": -100}, + }) + assert customer.litellm_budget_table is not None + assert customer.litellm_budget_table.rpm_limit == -1 + assert customer.litellm_budget_table.tpm_limit == -100 + + USER_IDENTIFIER_CASES = ( ({"user_id": "user-1"}, True), ({"user_email": "user@example.com"}, True), From dbd55d32023157fa1e13db994645878f8f870fd8 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Mon, 26 Oct 2026 14:55:08 -0700 Subject: [PATCH 069/155] fix(ui): keep Aria accessible in the dashboard header (#48454) * fix(ui): keep Aria accessible in the dashboard header * test(ui): wait for every pagination fixture log * test(ui): update assistant setup for Aria branding --- tests/e2e/ui/globalSetup.ts | 4 +- tests/e2e/ui/helpers/navigation.ts | 23 ++++-- .../e2e/ui/tests/logs/logsPagination.spec.ts | 61 ++++++++++++++- ui/litellm-dashboard/AGENTS.md | 2 +- .../src/components/DashboardHeader.test.tsx | 3 +- .../src/components/DashboardHeader.tsx | 75 ++++++++++--------- .../liteadmin/LiteAdmin.integration.test.tsx | 67 ++++++++--------- .../src/components/liteadmin/LiteAdmin.tsx | 37 +++------ 8 files changed, 161 insertions(+), 111 deletions(-) diff --git a/tests/e2e/ui/globalSetup.ts b/tests/e2e/ui/globalSetup.ts index d97789c227e..e18d1be390c 100644 --- a/tests/e2e/ui/globalSetup.ts +++ b/tests/e2e/ui/globalSetup.ts @@ -2,7 +2,7 @@ import { chromium, expect, request } from "@playwright/test"; import { users, Role, STORAGE_PATHS } from "./fixtures/users"; import { ARTIFACT_DIR, UI_BASE_URL } from "./constants"; import { expectUnrestrictedDashboard, setInvitedUserPassword } from "./helpers/userOnboarding"; -import { hideLiteAdmin } from "./helpers/navigation"; +import { setAriaHidden } from "./helpers/navigation"; import { masterKey as getMasterKey } from "./helpers/traffic"; import * as fs from "fs"; import * as path from "path"; @@ -78,7 +78,7 @@ async function globalSetup() { await dismiss.click(); } if (role === Role.ProxyAdmin) { - await hideLiteAdmin(page); + await setAriaHidden(page, true); } // The login flow stores a post-login return URL in the litellm_return_url // cookie. If the snapshot captures it before the app consumes it, every diff --git a/tests/e2e/ui/helpers/navigation.ts b/tests/e2e/ui/helpers/navigation.ts index 78288fab743..be46fcf0395 100644 --- a/tests/e2e/ui/helpers/navigation.ts +++ b/tests/e2e/ui/helpers/navigation.ts @@ -62,18 +62,29 @@ export async function dismissFeedbackPopup(page: PlaywrightPage): Promise } } -export async function hideLiteAdmin(page: PlaywrightPage): Promise { +export async function setAriaHidden(page: PlaywrightPage, hidden: boolean): Promise { await page.getByRole("button", { name: /Account menu/i }).click(); const panel = page.getByTestId("sidebar-account-menu-panel"); await expect(panel).toBeVisible({ timeout: 5_000 }); - const toggle = panel.getByRole("switch", { name: "Toggle hide LiteAdmin" }); - if ((await toggle.getAttribute("aria-checked")) !== "true") { - await toggle.click(); + const premium = await panel.getByText("Premium", { exact: true }).isVisible(); + const toggle = panel.getByRole("switch", { name: "Toggle hide Aria Agent", exact: true }); + if (premium) { + await expect(toggle).toBeVisible(); + if ((await toggle.getAttribute("aria-checked")) !== String(hidden)) { + await toggle.click(); + } + await expect(toggle).toHaveAttribute("aria-checked", String(hidden)); + } else { + await expect(toggle).toHaveCount(0); } - await expect(toggle).toHaveAttribute("aria-checked", "true"); await page.keyboard.press("Escape"); await expect(panel).toBeHidden(); - await expect(page.getByRole("button", { name: "LiteAdmin", exact: true })).toBeHidden(); + const aria = page.getByRole("banner").getByRole("button", { name: "Aria Agent", exact: true }); + if (premium && hidden) { + await expect(aria).toBeHidden(); + } else { + await expect(aria).toBeVisible(); + } } /** diff --git a/tests/e2e/ui/tests/logs/logsPagination.spec.ts b/tests/e2e/ui/tests/logs/logsPagination.spec.ts index 416e1c3171f..838a829a3e2 100644 --- a/tests/e2e/ui/tests/logs/logsPagination.spec.ts +++ b/tests/e2e/ui/tests/logs/logsPagination.spec.ts @@ -1,8 +1,14 @@ import { test, expect, type APIRequestContext, type Locator, type Page as PlaywrightPage } from "@playwright/test"; import { ADMIN_STORAGE_PATH } from "../../constants"; -import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; +import { navigateToPage, dismissFeedbackPopup, setAriaHidden } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; -import { CHAT_MODEL_A, createVirtualKey, sendChatCompletion, waitForSpendLog } from "../../helpers/traffic"; +import { + CHAT_MODEL_A, + createVirtualKey, + deleteVirtualKey, + sendChatCompletion, + waitForSpendLog, +} from "../../helpers/traffic"; /** * Session-grouped pagination (#38060): a page of N rows must render exactly N session rows, a @@ -45,6 +51,57 @@ async function setRowsPerPage(page: PlaywrightPage, size: "25" | "50" | "100"): await page.getByRole("option", { name: size, exact: true }).click(); } +test.describe("Logs pagination with Aria in the header", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + for (const viewport of [ + { width: 1440, height: 900 }, + { width: 390, height: 844 }, + ]) { + test(`keeps pagination reachable and Aria available at ${viewport.width}px`, async ({ page, request }) => { + await page.setViewportSize(viewport); + const alias = `e2e-aria-header-${uniqueSuffix()}`; + const key = await createVirtualKey(request, { key_alias: alias }); + try { + const ids = await Promise.all( + Array.from({ length: 26 }, (_, index) => + sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt: `${alias}-${index}`, + apiKey: key.key, + }), + ), + ); + await Promise.all(ids.map((id) => waitForSpendLog(request, id))); + await openLogs(page); + await setAriaHidden(page, false); + await applyKeyAliasFilter(page, await openFilterDrawer(page), alias); + await setRowsPerPage(page, "25"); + await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 1-25 of 26"); + await visibleTestId(page, "pagination-next").click(); + await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 26-26 of 26"); + + const aria = page.getByRole("banner").getByRole("button", { name: "Aria Agent", exact: true }); + await aria.click(); + await expect(page.getByRole("dialog", { name: "Aria Agent", exact: true })).toBeVisible(); + await page.keyboard.press("Escape"); + await expect(aria).toBeFocused(); + await visibleTestId(page, "pagination-first").click(); + await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 1-25 of 26"); + for (const destination of [Page.Usage, Page.LlmPlayground, Page.ApiKeys]) { + await navigateToPage(page, destination); + await aria.click(); + await expect(page.getByRole("dialog", { name: "Aria Agent", exact: true })).toBeVisible(); + await page.keyboard.press("Escape"); + await expect(aria).toBeFocused(); + } + } finally { + await deleteVirtualKey(request, key.token); + } + }); + } +}); + test.describe("Logs page session-grouped pagination", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); diff --git a/ui/litellm-dashboard/AGENTS.md b/ui/litellm-dashboard/AGENTS.md index bbf2e49aad7..b6fe5219022 100644 --- a/ui/litellm-dashboard/AGENTS.md +++ b/ui/litellm-dashboard/AGENTS.md @@ -30,7 +30,7 @@ Type tests are `*.test-d.ts` files run by the `types` vitest project (`npm run t -## This is NOT the Next.js you know +# This is NOT the Next.js you know This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` (resolved from this file's directory; in monorepos the `next` package may not be visible from the repo root) before writing any code. Heed deprecation notices. diff --git a/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx b/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx index 770315af49f..202a111ea97 100644 --- a/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx +++ b/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx @@ -124,7 +124,8 @@ describe("DashboardHeader breadcrumb", () => { const tools = await screen.findByRole("dialog", { name: "Gateway tools" }); expect(within(tools).getByRole("button", { name: "Switch workspace, current: AI Gateway" })).toBeInTheDocument(); expect(within(tools).getByRole("link", { name: "Docs" })).toBeInTheDocument(); - expect(within(tools).getByRole("button", { name: "Aria Agent" })).toBeInTheDocument(); + expect(within(screen.getByRole("banner")).getByRole("button", { name: "Aria Agent" })).toBeInTheDocument(); + expect(screen.getAllByRole("button", { name: "Aria Agent" })).toHaveLength(1); }); it("closes mobile tools when switching to desktop and keeps them closed when returning", async () => { diff --git a/ui/litellm-dashboard/src/components/DashboardHeader.tsx b/ui/litellm-dashboard/src/components/DashboardHeader.tsx index b31fc794f79..55f163c2495 100644 --- a/ui/litellm-dashboard/src/components/DashboardHeader.tsx +++ b/ui/litellm-dashboard/src/components/DashboardHeader.tsx @@ -73,43 +73,46 @@ export function DashboardHeader({ -
- {showWorkerSwitch && ( - <> - - - - )} +
- - - {!hideCommunityLinks && } - - - {deploymentHealth} - -
-
- - {deploymentHealth} - - - }> - - - - Gateway tools - - {showWorkerSwitch && } -
- - - - - {!hideCommunityLinks && } -
- - +
+ {showWorkerSwitch && ( + <> + + + + )} + + + {!hideCommunityLinks && } + + + {deploymentHealth} + +
+
+ + {deploymentHealth} + + + } + > + + + + Gateway tools + + {showWorkerSwitch && } +
+ + + + {!hideCommunityLinks && } +
+ + +
); diff --git a/ui/litellm-dashboard/src/components/liteadmin/LiteAdmin.integration.test.tsx b/ui/litellm-dashboard/src/components/liteadmin/LiteAdmin.integration.test.tsx index f32c99bedf3..ed66f7ce2ef 100644 --- a/ui/litellm-dashboard/src/components/liteadmin/LiteAdmin.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/liteadmin/LiteAdmin.integration.test.tsx @@ -240,7 +240,7 @@ describe("Aria Agent in the gateway", () => { session(role); const requests = gateway([answer("Here is your weekly spend.")]); renderWidget(); - await userEvent.setup().click(await screen.findByRole("button", { name: "Launch Aria Agent" })); + await userEvent.setup().click(await screen.findByRole("button", { name: "Aria Agent" })); const question = "How much have I spent this week?"; const suggestion = await screen.findByRole("button", { name: question }); await waitFor(() => expect(suggestion).toBeEnabled()); @@ -268,7 +268,7 @@ describe("Aria Agent in the gateway", () => { it("should disable suggested questions when no chat model is available", async () => { const requests = gateway([], { models: [] }); renderWidget(); - await userEvent.setup().click(await screen.findByRole("button", { name: "Launch Aria Agent" })); + await userEvent.setup().click(await screen.findByRole("button", { name: "Aria Agent" })); const picker = await screen.findByRole("combobox", { name: "Aria Agent model" }); expect(picker).toBeEnabled(); expect(screen.queryByText("No chat models are available for your account.")).not.toBeInTheDocument(); @@ -285,7 +285,7 @@ describe("Aria Agent in the gateway", () => { expect(requests.filter((request) => request.url.endsWith("/chat/completions"))).toHaveLength(0); }); - it("should restore launcher focus and preserve the conversation when the popup closes with Escape", async () => { + it("should restore header trigger focus and preserve the conversation when the popup closes with Escape", async () => { const requests = gateway([answer("Your current usage is $12.34."), answer("Last month was $10.")]); const user = userEvent.setup(); const backgroundAction = vi.fn(); @@ -293,7 +293,7 @@ describe("Aria Agent in the gateway", () => { return ; } renderWidget(PageAction); - await user.click(await screen.findByRole("button", { name: "Launch Aria Agent" })); + await user.click(await screen.findByRole("button", { name: "Aria Agent" })); const popup = await screen.findByRole("dialog", { name: "Aria Agent" }); expect(popup).toBeVisible(); await waitFor(() => expect(screen.getByPlaceholderText("Ask Aria…")).toBeEnabled()); @@ -306,8 +306,8 @@ describe("Aria Agent in the gateway", () => { await user.click(screen.getByPlaceholderText("Ask Aria…")); await user.keyboard("{Escape}"); await waitFor(() => expect(screen.queryByRole("dialog", { name: "Aria Agent" })).not.toBeInTheDocument()); - const launcher = screen.getByRole("button", { name: "Launch Aria Agent" }); - await waitFor(() => expect(launcher).toHaveFocus()); + const trigger = screen.getByRole("button", { name: "Aria Agent" }); + await waitFor(() => expect(trigger).toHaveFocus()); await user.keyboard("{Enter}"); expect(await screen.findByRole("dialog", { name: "Aria Agent" })).toBeVisible(); expect(screen.getByText("Your current usage is $12.34.")).toBeInTheDocument(); @@ -324,26 +324,25 @@ describe("Aria Agent in the gateway", () => { ]); await user.click(screen.getByRole("button", { name: "Close Aria Agent" })); await waitFor(() => expect(screen.queryByRole("dialog", { name: "Aria Agent" })).not.toBeInTheDocument()); - await waitFor(() => expect(screen.getByRole("button", { name: "Launch Aria Agent" })).toHaveFocus()); + await waitFor(() => expect(screen.getByRole("button", { name: "Aria Agent" })).toHaveFocus()); }); - it("should show the same product name on Home, Usage, the launcher and the popup", async () => { + it("should show the same product name on Home, Usage, the header trigger and the popup", async () => { gateway([]); const view = renderWidget(undefined, true); const home = await screen.findByRole("region", { name: "Ask Aria Agent" }); expect(within(home).getByRole("heading", { name: "Aria Agent" })).toHaveTextContent("Aria Agent"); expect(screen.getByRole("button", { name: "Aria Agent" })).toHaveTextContent("Aria Agent"); - expect(screen.getByRole("button", { name: "Launch Aria Agent" })).toHaveTextContent("Aria Agent"); view.refresh(false, true); const usage = await screen.findByRole("region", { name: "Ask Aria about usage" }); expect(within(usage).getByRole("heading", { name: "Aria Agent" })).toHaveTextContent("Aria Agent"); expect(screen.getByRole("button", { name: "Open Aria Agent" })).toHaveTextContent("Aria Agent"); - act(() => fireEvent.click(screen.getByRole("button", { name: "Launch Aria Agent" }))); + act(() => fireEvent.click(screen.getByRole("button", { name: "Aria Agent" }))); const popup = await screen.findByRole("dialog", { name: "Aria Agent" }); expect(within(popup).getByRole("heading", { name: "Aria Agent" })).toHaveTextContent("Aria Agent"); }); - it("should keep the same scoped conversation when leaving Usage and reopening the launcher", async () => { + it("should keep the same scoped conversation when leaving Usage and reopening the header trigger", async () => { session("internal_user"); const requests = gateway( [answer("Your usage is within budget."), answer("The same usage conversation continues.")], @@ -355,7 +354,7 @@ describe("Aria Agent in the gateway", () => { }, ); const view = renderWidget(undefined, false, true); - expect(screen.queryByRole("button", { name: "Launch Aria Agent" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Aria Agent" })).not.toBeInTheDocument(); const usage = await screen.findByRole("region", { name: "Ask Aria about usage" }); await selectModel("other-chat-model"); act(() => { @@ -366,14 +365,13 @@ describe("Aria Agent in the gateway", () => { view.refresh(false, false); expect(screen.queryByRole("region", { name: "Ask Aria about usage" })).not.toBeInTheDocument(); - expect(screen.queryByRole("button", { name: "Aria Agent" })).not.toBeInTheDocument(); - const launcher = screen.getByRole("button", { name: "Launch Aria Agent" }); - expect(launcher).toHaveTextContent("Aria Agent"); - act(() => launcher.focus()); + const trigger = screen.getByRole("button", { name: "Aria Agent" }); + expect(trigger).toHaveTextContent("Aria Agent"); + act(() => trigger.focus()); await userEvent.setup().keyboard("{Enter}"); expect(await screen.findByPlaceholderText("Ask Aria…")).toHaveValue("Check my usage"); expect(screen.getByRole("combobox", { name: "Aria Agent model" })).toHaveValue("other-chat-model"); - expect(screen.queryByRole("button", { name: "Launch Aria Agent" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Aria Agent" })).toHaveAttribute("aria-expanded", "true"); act(() => fireEvent.click(screen.getByRole("button", { name: "Send message" }))); await screen.findByText("Your usage is within budget."); act(() => { @@ -385,7 +383,7 @@ describe("Aria Agent in the gateway", () => { const restoredUsage = await screen.findByRole("region", { name: "Ask Aria about usage" }); expect(within(restoredUsage).getByRole("textbox", { name: "Message Aria Agent" })).toHaveValue("And last month?"); view.refresh(false, false); - act(() => fireEvent.click(screen.getByRole("button", { name: "Launch Aria Agent" }))); + act(() => fireEvent.click(screen.getByRole("button", { name: "Aria Agent" }))); expect(screen.getAllByRole("dialog", { name: "Aria Agent" })).toHaveLength(1); expect(screen.getByText("Your usage is within budget.")).toBeInTheDocument(); expect(screen.getByPlaceholderText("Ask Aria…")).toHaveValue("And last month?"); @@ -405,13 +403,13 @@ describe("Aria Agent in the gateway", () => { ]); }); - it("should keep the launcher unavailable without a signed-in session", async () => { + it("should keep the header trigger unavailable without a signed-in session", async () => { document.cookie = "token=; Max-Age=0; Path=/"; const requests = gateway([]); const { client } = renderWidget(); await screen.findByText("Session ready"); await waitFor(() => expect(client.isFetching()).toBe(0)); - expect(screen.queryByRole("button", { name: "Launch Aria Agent" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Aria Agent" })).not.toBeInTheDocument(); act(() => fireEvent.keyDown(document, { key: "j", ctrlKey: true })); expect(screen.queryByRole("dialog", { name: "Aria Agent" })).not.toBeInTheDocument(); expect(requests.every((request) => request.url.endsWith("/litellm-ui-config"))).toBe(true); @@ -494,7 +492,7 @@ describe("Aria Agent in the gateway", () => { expect(await screen.findByText("Your usage totals $12.34.")).toBeInTheDocument(); expect(screen.getAllByRole("dialog", { name: "Aria Agent" })).toHaveLength(1); expect(usage).not.toBeVisible(); - if (!writable) expect(screen.queryByRole("button", { name: "Aria Agent", hidden: true })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Aria Agent" })).toHaveAttribute("aria-expanded", "true"); const read = requests.find((request) => request.url.includes("/user/daily/activity/aggregated")); expect(read).toBeDefined(); expect(new URL(read!.url).searchParams.get("user_id")).toBe(scope === "user" ? "first-admin" : null); @@ -553,7 +551,7 @@ describe("Aria Agent in the gateway", () => { await waitFor(() => expect(usageInput).toBeEnabled()); expect(usageInput).toHaveValue(""); expect(screen.queryByRole("button", { name: "Confirm change" })).not.toBeInTheDocument(); - expect(screen.queryByRole("button", { name: "Aria Agent" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Aria Agent" })).toHaveAttribute("aria-expanded", "false"); expect(screen.queryByRole("dialog", { name: "Aria Agent" })).not.toBeInTheDocument(); act(() => send("Show my usage")); expect(await screen.findByText("Only your own usage is available.")).toBeInTheDocument(); @@ -812,7 +810,6 @@ describe("Aria Agent in the gateway", () => { await user.click(toggle); expect(toggle).toBeChecked(); expect(screen.queryByRole("button", { name: "Aria Agent" })).not.toBeInTheDocument(); - expect(screen.queryByRole("button", { name: "Launch Aria Agent" })).not.toBeInTheDocument(); expect(screen.queryByRole("region", { name: "Ask Aria Agent", hidden: true })).not.toBeInTheDocument(); view.unmount(); @@ -820,13 +817,12 @@ describe("Aria Agent in the gateway", () => { await screen.findByText("Session ready"); await waitFor(() => expect(restored.client.isFetching()).toBe(0)); expect(screen.queryByRole("button", { name: "Aria Agent" })).not.toBeInTheDocument(); - expect(screen.queryByRole("button", { name: "Launch Aria Agent" })).not.toBeInTheDocument(); await user.click(screen.getByRole("button", { name: /account menu/i })); const savedToggle = await screen.findByRole("switch", { name: "Toggle hide Aria Agent" }); expect(savedToggle).toBeChecked(); await user.click(savedToggle); expect(await screen.findByRole("button", { name: "Aria Agent" })).toBeInTheDocument(); - expect(screen.getByRole("button", { name: "Launch Aria Agent" })).toBeVisible(); + expect(screen.getByRole("button", { name: "Aria Agent" })).toBeVisible(); expect(await screen.findByRole("region", { name: "Ask Aria Agent" })).toBeVisible(); for (const premiumUser of [false, null, undefined, "true"]) { @@ -838,27 +834,26 @@ describe("Aria Agent in the gateway", () => { }); it.each([false, null, undefined, "true"])( - "should offer the scoped launcher without admin Home or header access when premium_user is %s", + "should offer the scoped header trigger without admin Home access when premium_user is %s", async (premiumUser) => { session("proxy_admin", "first-admin", { premium_user: premiumUser }); const requests = gateway([]); const { client } = renderWidget(undefined, true); await screen.findByText("Session ready"); await waitFor(() => expect(client.isFetching()).toBe(0)); - expect(screen.queryByRole("button", { name: "Aria Agent" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Aria Agent" })).toBeVisible(); expect(screen.queryByRole("region", { name: "Ask Aria Agent", hidden: true })).not.toBeInTheDocument(); - expect(screen.getByRole("button", { name: "Launch Aria Agent" })).toBeVisible(); expect(requests.every((request) => request.url.endsWith("/litellm-ui-config"))).toBe(true); act(() => fireEvent.keyDown(document, { key: "j", metaKey: true })); expect(await screen.findByRole("dialog", { name: "Aria Agent" })).toBeInTheDocument(); - expect(screen.queryByRole("button", { name: "Launch Aria Agent" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Aria Agent" })).toHaveAttribute("aria-expanded", "true"); act(() => fireEvent.keyDown(document, { key: "j", ctrlKey: true })); expect(screen.queryByRole("dialog", { name: "Aria Agent" })).not.toBeInTheDocument(); - expect(screen.getByRole("button", { name: "Launch Aria Agent" })).toBeVisible(); + expect(screen.getByRole("button", { name: "Aria Agent" })).toBeVisible(); }, ); - it("should discard admin state and preserve scoped launcher access when enterprise access is lost", async () => { + it("should discard admin state and preserve scoped header trigger access when enterprise access is lost", async () => { const requests = gateway([answer("Your usage remains available.")]); const view = renderWidget(); await openWidget(); @@ -866,9 +861,9 @@ describe("Aria Agent in the gateway", () => { await waitFor(() => expect(view.client.isFetching()).toBe(0)); session("proxy_admin", "first-admin", { premium_user: false }); view.refresh(); - expect(screen.queryByRole("button", { name: "Aria Agent" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Aria Agent" })).toHaveAttribute("aria-expanded", "false"); expect(screen.queryByRole("dialog", { name: "Aria Agent" })).not.toBeInTheDocument(); - act(() => fireEvent.click(screen.getByRole("button", { name: "Launch Aria Agent" }))); + act(() => fireEvent.click(screen.getByRole("button", { name: "Aria Agent" }))); await waitFor(() => expect(screen.getByPlaceholderText("Ask Aria…")).toBeEnabled()); expect(screen.getByPlaceholderText("Ask Aria…")).toHaveValue(""); act(() => send("Show my usage")); @@ -997,17 +992,17 @@ describe("Aria Agent in the gateway", () => { }); it.each(["proxy_admin_viewer", "internal_user", "internal_user_viewer", "org_admin"])( - "should expose only scoped usage tools from the dashboard launcher to %s", + "should expose only scoped usage tools from the dashboard header trigger to %s", async (role) => { session(role); const requests = gateway([answer("Usage is available.")]); const { client } = renderWidget(undefined, true); await screen.findByText("Session ready"); await waitFor(() => expect(client.isFetching()).toBe(0)); - expect(screen.queryByRole("button", { name: "Aria Agent" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Aria Agent" })).toBeVisible(); expect(screen.queryByRole("region", { name: "Ask Aria Agent", hidden: true })).not.toBeInTheDocument(); expect(requests.every((request) => request.url.endsWith("/litellm-ui-config"))).toBe(true); - act(() => fireEvent.click(screen.getByRole("button", { name: "Launch Aria Agent" }))); + act(() => fireEvent.click(screen.getByRole("button", { name: "Aria Agent" }))); await waitFor(() => expect(screen.getByPlaceholderText("Ask Aria…")).toBeEnabled()); act(() => send("Show my usage")); await screen.findByText("Usage is available."); diff --git a/ui/litellm-dashboard/src/components/liteadmin/LiteAdmin.tsx b/ui/litellm-dashboard/src/components/liteadmin/LiteAdmin.tsx index 66c1ffbc9b7..02194aabe7b 100644 --- a/ui/litellm-dashboard/src/components/liteadmin/LiteAdmin.tsx +++ b/ui/litellm-dashboard/src/components/liteadmin/LiteAdmin.tsx @@ -45,7 +45,7 @@ type LiteAdminState = { toggle: () => void; show: () => void; access: LiteAdminSession["access"]; - launcherRef: RefObject; + triggerRef: RefObject; homeContainer: HTMLDivElement | null; setHomeContainer: (container: HTMLDivElement | null) => void; usageContainer: HTMLDivElement | null; @@ -78,12 +78,11 @@ function useLiteAdminSession() { }; } -/** Keeps one Aria conversation available from the dashboard and its floating launcher. */ export function LiteAdminFrame({ children }: { children: ReactNode }) { const configured = useLiteAdminSession(); const sessionKey = configured?.key ?? null; const [open, setOpen] = useState(false); - const launcherRef = useRef(null); + const triggerRef = useRef(null); const [started, setStarted] = useState(false); const [homeContainer, setHomeContainer] = useState(null); const [usageContainer, updateUsageContainer] = useState(null); @@ -128,7 +127,7 @@ export function LiteAdminFrame({ children }: { children: ReactNode }) { toggle, show, access: configured.session.access, - launcherRef, + triggerRef, homeContainer, setHomeContainer, usageContainer, @@ -138,7 +137,6 @@ export function LiteAdminFrame({ children }: { children: ReactNode }) { } > {children} - {configured && !open && } {configured && showPanel && ( setOpen(false)} /> )} @@ -146,23 +144,6 @@ export function LiteAdminFrame({ children }: { children: ReactNode }) { ); } -function AriaLauncher({ show, buttonRef }: { show: () => void; buttonRef: RefObject }) { - return ( - - ); -} - export function LiteAdminHome() { const state = useContext(LiteAdminContext); if (!state || state.access.mode !== "admin") return null; @@ -189,18 +170,20 @@ export function LiteAdminHome() { export default function LiteAdminTrigger() { const state = useContext(LiteAdminContext); - if (!state || state.access.mode !== "admin") return null; + if (!state) return null; + const { triggerRef, open, toggle } = state; return ( ); @@ -358,7 +341,7 @@ function Panel({ initialFocus={() => initialFocus?.current ?? ref.current?.querySelector("textarea:enabled") ?? ref.current } - finalFocus={state?.launcherRef} + finalFocus={state?.triggerRef} className="flex h-[min(48rem,calc(100dvh-2rem))] flex-col gap-0 overflow-hidden p-0 shadow-xl sm:max-w-4xl" > {children} From a6fdc7c5a3fa8952f69623d56a52ada73dc5c893 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Mon, 26 Oct 2026 15:01:40 -0700 Subject: [PATCH 070/155] fix(proxy): scope dashboard search grants to user (#48464) Co-authored-by: Jodok Batlogg Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/auth_checks.py | 35 +++++- .../auth/test_search_tool_deny_by_default.py | 114 +++++++++++++++++- 2 files changed, 139 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 79e385c376a..448643ef59b 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -6461,8 +6461,29 @@ def _search_tool_deny_by_default(general_settings: Mapping[str, object]) -> bool return True +def _is_dashboard_session(valid_token: UserAPIKeyAuth | None) -> bool: + return valid_token is not None and valid_token.team_id == UI_TEAM_ID + + +def _is_search_tool_strict_identity(valid_token: UserAPIKeyAuth) -> bool: + if not _is_dashboard_session(valid_token): + return _is_strict_grant_identity(valid_token) + return valid_token.user_role not in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ) + + +def _search_tool_strict_layers( + valid_token: UserAPIKeyAuth, team_object: LiteLLM_TeamTable | None +) -> frozenset[GrantLayer]: + if _is_dashboard_session(valid_token): + return frozenset(("user",)) + return _strict_grant_layers(True, valid_token, team_object) + + def is_search_tool_deny_by_default_applied(valid_token: UserAPIKeyAuth, general_settings: Mapping[str, object]) -> bool: - return _search_tool_deny_by_default(general_settings) and _is_strict_grant_identity(valid_token) + return _search_tool_deny_by_default(general_settings) and _is_search_tool_strict_identity(valid_token) def _search_tool_denied(object_type: GrantLayer, message: str) -> ProxyException: @@ -6506,12 +6527,12 @@ async def resolve_search_tool_grants( load_team_object: TeamObjectLoader, ) -> SearchToolGrants: """ - Without `general_settings.search_tool_deny_by_default`, or for the master key and dashboard sessions, + Without `general_settings.search_tool_deny_by_default`, or for the master key and admin dashboard sessions, the key and team lists only restrict when nonempty. With it, the identities `_strict_grant_layers` names must each list the tool, and a missing record, `null`, `[]`, or a team or user that fails to load grants nothing """ if not is_search_tool_deny_by_default_applied(valid_token, general_settings): - team_object: Final = await load_team_object() + team_object: Final = await load_team_object() if not _is_dashboard_session(valid_token) else None return SearchToolGrants( grants=MappingProxyType( {"key": valid_token.object_permission, "team": team_object.object_permission if team_object else None} @@ -6521,8 +6542,10 @@ async def resolve_search_tool_grants( from litellm.proxy.proxy_server import prisma_client, user_api_key_cache - strict_team_object: Final = await _strict_team_object(load_team_object) - strict_layers: Final = _strict_grant_layers(True, valid_token, strict_team_object) + strict_team_object: Final = ( + await _strict_team_object(load_team_object) if not _is_dashboard_session(valid_token) else None + ) + strict_layers: Final = _search_tool_strict_layers(valid_token, strict_team_object) if prisma_client is None: return SearchToolGrants(grants=MappingProxyType({}), strict_layers=strict_layers) user_object: Final = ( @@ -6607,7 +6630,7 @@ def check_unregistered_search_fallback( """The unregistered provider fallback names no search tool that a grant could list, so deny-by-default denies it""" if not is_search_tool_deny_by_default_applied(valid_token, general_settings): return True - layer: Final = min(_strict_grant_layers(True, valid_token, None), key=("key", "team", "user").index) + layer: Final = min(_search_tool_strict_layers(valid_token, None), key=("key", "team", "user").index) raise _search_tool_denied( layer, "No registered search tool is available and search_tool_deny_by_default is enabled", diff --git a/tests/unit/proxy/auth/test_search_tool_deny_by_default.py b/tests/unit/proxy/auth/test_search_tool_deny_by_default.py index cd285e0386f..da036795889 100644 --- a/tests/unit/proxy/auth/test_search_tool_deny_by_default.py +++ b/tests/unit/proxy/auth/test_search_tool_deny_by_default.py @@ -1,8 +1,9 @@ from collections.abc import Mapping -from typing import Final, TypedDict +from typing import Final, TypedDict, cast from unittest.mock import AsyncMock, MagicMock import pytest +from fastapi import HTTPException from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS from litellm.proxy import proxy_server @@ -78,6 +79,14 @@ def _team_loader(team_tools: list[str] | None | str) -> TeamObjectLoader: return load +def _missing_dashboard_team_loader() -> tuple[TeamObjectLoader, AsyncMock]: + missing_team: Final = HTTPException( + status_code=404, detail="Team doesn't exist in db. Team=litellm-dashboard" + ) + load_team: Final = AsyncMock(side_effect=missing_team) + return cast(TeamObjectLoader, load_team), load_team + + async def _no_team() -> None: return None @@ -194,9 +203,6 @@ async def test_search_tool_access_is_unchanged_without_search_tool_deny_by_defau id="proxy admin virtual key not exempt", ), pytest.param(CallerFields(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS), None, None, None, id="master key exempt"), - pytest.param( - CallerFields(team_id=UI_TEAM_ID, virtual_key=False), None, None, None, id="dashboard session exempt" - ), ], ) @pytest.mark.asyncio @@ -248,6 +254,79 @@ async def test_search_tool_deny_by_default_does_not_load_the_user_when_off( get_user_object.assert_not_awaited() +@pytest.mark.parametrize( + "general_settings", + [{}, {"search_tool_deny_by_default": False}], + ids=["omitted", "false"], +) +@pytest.mark.parametrize( + "user_role", + [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.INTERNAL_USER], + ids=["admin", "internal-user"], +) +@pytest.mark.parametrize( + "key_tools, expected", + [ + pytest.param(None, None, id="unrestricted-key"), + pytest.param(["search-a"], None, id="key-grants-search-a"), + pytest.param(["search-b"], _LEGACY, id="key-excludes-search-a"), + ], +) +@pytest.mark.asyncio +async def test_dashboard_search_tool_grants_without_deny_by_default( + general_settings: Mapping[str, object], + user_role: LitellmUserRoles, + key_tools: list[str] | None, + expected: ProxyErrorTypes | None, +) -> None: + caller: Final = _caller( + virtual_key=False, + key_tools=key_tools, + team_id=UI_TEAM_ID, + user_role=user_role, + ) + load_team, load_team_mock = _missing_dashboard_team_loader() + assert await _denied_by(general_settings, caller, load_team) == expected + load_team_mock.assert_not_awaited() + + +@pytest.mark.parametrize( + "user_role", + [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY], + ids=["admin", "admin-view-only"], +) +@pytest.mark.asyncio +async def test_admin_dashboard_search_tool_grants_with_deny_by_default(user_role: LitellmUserRoles) -> None: + caller: Final = _caller(virtual_key=False, team_id=UI_TEAM_ID, user_role=user_role) + load_team, load_team_mock = _missing_dashboard_team_loader() + assert await _denied_by(_DENY_ON, caller, load_team) is None + load_team_mock.assert_not_awaited() + + +@pytest.mark.parametrize( + "user_id, user_tools, expected", + [ + pytest.param("user-1", ["search-a"], None, id="user-grants-search-a"), + pytest.param("user-1", ["search-b"], _USER, id="user-grants-another-tool"), + pytest.param("user-1", None, _USER, id="user-grant-null"), + pytest.param("user-2", None, _USER, id="no-user-row"), + ], +) +@pytest.mark.asyncio +async def test_internal_dashboard_search_tool_requires_the_user_grant_with_deny_by_default( + cache: UserApiKeyCache, + user_id: str, + user_tools: list[str] | None, + expected: ProxyErrorTypes | None, +) -> None: + if user_id == "user-1": + _cache_user(cache, user_tools) + caller: Final = _caller(virtual_key=False, team_id=UI_TEAM_ID, user_id=user_id) + load_team, load_team_mock = _missing_dashboard_team_loader() + assert await _denied_by(_DENY_ON, caller, load_team) == expected + load_team_mock.assert_not_awaited() + + @pytest.mark.parametrize( "general_settings, caller, expected", [ @@ -257,6 +336,24 @@ async def test_search_tool_deny_by_default_does_not_load_the_user_when_off( pytest.param(_DENY_ON, _caller(virtual_key=False), _USER, id="keyless user"), pytest.param(_DENY_ON, _caller(user_role=LitellmUserRoles.PROXY_ADMIN), _KEY, id="proxy admin key"), pytest.param(_DENY_ON, _caller(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS), None, id="master key"), + pytest.param( + _DENY_ON, + _caller(virtual_key=False, team_id=UI_TEAM_ID, user_role=LitellmUserRoles.INTERNAL_USER), + _USER, + id="internal dashboard user", + ), + pytest.param( + _DENY_ON, + _caller(virtual_key=False, team_id=UI_TEAM_ID, user_role=LitellmUserRoles.PROXY_ADMIN), + None, + id="admin dashboard", + ), + pytest.param( + _DENY_ON, + _caller(virtual_key=False, team_id=UI_TEAM_ID, user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY), + None, + id="admin view-only dashboard", + ), ], ) def test_unregistered_search_fallback_follows_search_tool_deny_by_default( @@ -268,3 +365,12 @@ def test_unregistered_search_fallback_follows_search_tool_deny_by_default( with pytest.raises(ProxyException) as exc_info: check_unregistered_search_fallback(caller, general_settings) assert (exc_info.value.type, exc_info.value.code) == (expected, "403") + + +@pytest.mark.asyncio +async def test_search_preserves_missing_real_team_error_without_deny_by_default() -> None: + missing_team: Final = HTTPException(status_code=404, detail="Team doesn't exist in db. Team=team-1") + load_team: Final = AsyncMock(side_effect=missing_team) + with pytest.raises(HTTPException) as exc_info: + await can_caller_call_search_tool("search-a", _caller(team_id="team-1"), {}, load_team) + assert exc_info.value is missing_team From 9a38ff4cb59fd60a5ca2bed9b0410783149a3e68 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 26 Oct 2026 18:27:13 -0400 Subject: [PATCH 071/155] fix(gemini): preserve tool media for older models (#48428) --- .../prompt_templates/factory.py | 5 +- .../llms/vertex_ai/gemini/transformation.py | 4 + .../vertex_and_google_ai_studio_gemini.py | 4 + .../test_vertex_ai_gemini_transformation.py | 73 +++++++++++++++---- 4 files changed, 71 insertions(+), 15 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 0aed2956b0a..3bc8dc4e23e 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1490,6 +1490,7 @@ def convert_to_gemini_tool_call_result( message: ChatCompletionToolMessage | ChatCompletionFunctionMessage, last_message_with_tool_calls: dict | None, forward_function_call_id: bool = False, + supports_multimodal_function_responses: bool = True, ) -> VertexPartType | list[VertexPartType]: """ OpenAI message with a tool result looks like: @@ -1652,9 +1653,9 @@ def convert_to_gemini_tool_call_result( _part: Final[VertexPartType] = {"function_response": _function_response} - # For multimodal function responses, Gemini expects media parts nested - # inside functionResponse.parts instead of sibling content parts. if inline_data_list: + if not supports_multimodal_function_responses: + return [_part, *(VertexPartType(inline_data=inline_data) for inline_data in inline_data_list)] _function_response["parts"] = [{"inline_data": inline_data} for inline_data in inline_data_list] return [_part] diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 71bb05ee155..ed86934db63 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -726,6 +726,9 @@ def gemini_convert_messages_with_history( from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig forward_function_call_id: Final = VertexGeminiConfig.forward_gemini_function_call_id(model or "") + supports_multimodal_function_responses: Final = model is None or ( + VertexGeminiConfig.supports_multimodal_function_responses(model) + ) try: while msg_i < len(messages): @@ -1029,6 +1032,7 @@ def gemini_convert_messages_with_history( messages[msg_i], last_message_with_tool_calls, forward_function_call_id=forward_function_call_id, + supports_multimodal_function_responses=supports_multimodal_function_responses, ) msg_i += 1 # Handle both single part and list of parts (for Computer Use with images) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index d788ccb0f1d..9acbc9c459c 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -689,6 +689,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) -> bool: return cls._forward_gemini_function_call_id(model) + @classmethod + def supports_multimodal_function_responses(cls, model: str) -> bool: + return cls.is_gemini_3_or_newer(model) + def _supports_penalty_parameters(self, model: str) -> bool: # Gemini 3 models do not support penalty parameters if VertexGeminiConfig._is_gemini_3_or_newer(model): diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index b97724319c7..75449eacfd3 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -2,7 +2,7 @@ import base64 from collections.abc import Sequence from itertools import chain from pathlib import Path -from typing import Final +from typing import Final, Literal import httpx import pytest @@ -1650,10 +1650,28 @@ def test_convert_tool_response_with_base64_image(): assert inline_data["data"] == test_image_base64 -def test_gemini_history_nests_multimodal_tool_response_parts(): - """Full history conversion should not emit sibling inline_data tool result parts.""" - test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" - messages = [ +@pytest.mark.parametrize("provider", ["gemini", "vertex_ai", "vertex_ai_beta"]) +@pytest.mark.parametrize( + "model,nested", + [ + (None, True), + ("gemini-2.5-pro", False), + ("gemini/gemini-2.5-flash", False), + ("vertex_ai/gemini-2.5-flash-lite", False), + ("123456789", False), + ("gemini-3.8-flash", True), + ("gemini/gemini-3.8-flash", True), + ("vertex_ai/gemini-3.8-flash", True), + ("gemini-flash-latest", True), + ], +) +def test_gemini_history_places_tool_media_for_model( + model: str | None, + nested: bool, + provider: Literal["gemini", "vertex_ai", "vertex_ai_beta"], +) -> None: + test_image_base64: Final = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + messages: Final = [ {"role": "user", "content": "Get me an image"}, { "role": "assistant", @@ -1679,24 +1697,53 @@ def test_gemini_history_nests_multimodal_tool_response_parts(): "data": test_image_base64, }, }, + { + "type": "file", + "file": {"file_data": "data:application/pdf;base64,JVBERi0xLjQ="}, + }, ], }, + {"role": "user", "content": "Describe the image and file again"}, ] - contents = gemini_convert_messages_with_history(messages=messages) - - tool_response_parts = contents[-1]["parts"] - assert len(tool_response_parts) == 1 - assert "inline_data" not in tool_response_parts[0] - function_response = tool_response_parts[0]["function_response"] - assert function_response["parts"] == [ + contents: Final = ( + gemini_convert_messages_with_history(messages=messages) + if model is None + else transform_request_body( + messages=messages, + model=model, + custom_llm_provider=provider, + optional_params={}, + litellm_params={}, + cached_content=None, + )["contents"] + ) + media_parts: Final = [ { "inline_data": { "data": test_image_base64, "mime_type": "image/png", } - } + }, + {"inline_data": {"data": "JVBERi0xLjQ=", "mime_type": "application/pdf"}}, ] + function_response: Final = { + "name": "get_image", + "response": {"image_ref": "inline"}, + **({"id": "call_get_image"} if model is not None and nested else {}), + **({"parts": media_parts} if nested else {}), + } + assert contents[-2] == { + "role": "user", + "parts": [ + {"function_response": function_response}, + *([] if nested else media_parts), + ], + } + assert contents[-1] == { + "role": "user", + "parts": [{"text": "Describe the image and file again"}], + } def test_convert_tool_response_text_only(): From 57ac10731149fbc2a5612a811c1dc15f6780dac1 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:28:53 -0700 Subject: [PATCH 072/155] ci: remove the Together AI and weekly cost map sync workflows (#48473) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...to_update_price_and_context_window_file.py | 319 ---------- .../auto_update_price_and_context_window.yml | 36 -- .github/workflows/sync-together-ai-models.yml | 65 --- scripts/sync_together_ai_models.py | 547 ------------------ ...to_update_price_and_context_window_file.py | 222 ------- tests/unit/test_sync_together_ai_models.py | 378 ------------ 6 files changed, 1567 deletions(-) delete mode 100644 .github/scripts/auto_update_price_and_context_window_file.py delete mode 100644 .github/workflows/auto_update_price_and_context_window.yml delete mode 100644 .github/workflows/sync-together-ai-models.yml delete mode 100644 scripts/sync_together_ai_models.py delete mode 100644 tests/unit/test_auto_update_price_and_context_window_file.py delete mode 100644 tests/unit/test_sync_together_ai_models.py diff --git a/.github/scripts/auto_update_price_and_context_window_file.py b/.github/scripts/auto_update_price_and_context_window_file.py deleted file mode 100644 index 4c9a05bd4f0..00000000000 --- a/.github/scripts/auto_update_price_and_context_window_file.py +++ /dev/null @@ -1,319 +0,0 @@ -import asyncio -import aiohttp -import json -import math -from typing import Any - -# Asynchronously fetch data from a given URL -async def fetch_data(url): - try: - # Create an asynchronous session - async with aiohttp.ClientSession() as session: - # Send a GET request to the URL - async with session.get(url) as resp: - # Raise an error if the response status is not OK - resp.raise_for_status() - # Parse the response JSON - resp_json = await resp.json() - print("Fetch the data from URL.") - # Return the 'data' field from the JSON response - return resp_json['data'] - except Exception as e: - # Print an error message if fetching data fails - print("Error fetching data from URL:", e) - return None - - -FRIENDLI_API_URL = "https://api.friendli.ai/serverless/v1/models" -FRIENDLI_PROVIDER = "friendliai" - -INHERITABLE_BASE_KEYS = ( - "supports_pdf_input", - "supports_assistant_prefill", - "supports_adaptive_thinking", - "supports_output_config", -) - -REASONING_EFFORT_LEVEL_ORDER = ("none", "minimal", "low", "medium", "high", "xhigh", "max") - - -def _find_base_model_entry(base_model: str, local_data: dict) -> str | None: - if not base_model: - return None - bm_tail = base_model.split("/")[-1].lower() - if base_model in local_data: - return base_model - for key in local_data: - if key.startswith("sample_spec") or key == "fallback_generalizations": - continue - if key.split("/")[-1].lower() == bm_tail: - return key - return None - - -def _reasoning_effort_levels(reasoning_options: list) -> list: - offered = { - val - for opt in reasoning_options or [] - if opt.get("type") == "effort" - for val in opt.get("values", []) - } - return [level for level in REASONING_EFFORT_LEVEL_ORDER if level in offered] - - -def _valid_token_price(value: object) -> bool: - try: - price = float(value) # pyright: ignore[reportArgumentType] # non-numeric values are rejected via the except - except (TypeError, ValueError): - return False - return math.isfinite(price) and price >= 0 - - -def _has_valid_token_prices(pricing: dict | None) -> bool: - prices = pricing or {} - return _valid_token_price(prices.get("input")) and _valid_token_price(prices.get("output")) - - -def _pricing(pricing: dict) -> dict: - out: dict[str, Any] = {} - if not pricing: - return out - if "input" in pricing: - out["input_cost_per_token"] = float(pricing["input"]) - if "output" in pricing: - out["output_cost_per_token"] = float(pricing["output"]) - if "input_cache_read" in pricing and pricing["input_cache_read"] is not None: - out["cache_read_input_token_cost"] = float(pricing["input_cache_read"]) - return out - - -def _modality_flags(input_mods: list) -> dict: - mods = input_mods or [] - has_image = "image" in mods - return { - "supports_vision": has_image, - "supports_image_input": has_image, - "supports_video_input": "video" in mods, - } - - -def transform_friendli_data(data: list, local_data: dict) -> dict: - transformed: dict[str, dict] = {} - if not data: - return transformed - for model in data: - # An unpriced row must never wholesale-replace an already priced local entry: - # missing prices cost-calculate as zero, silently zeroing tracked spend - if not _has_valid_token_prices(model.get("pricing")): - continue - model_id = model["id"] - base_model = model.get("base_model") or "" - entry: dict[str, Any] = { - "litellm_provider": FRIENDLI_PROVIDER, - } - - base_key = _find_base_model_entry(base_model, local_data) - if base_key: - base_entry = local_data[base_key] - for k in INHERITABLE_BASE_KEYS: - if k in base_entry: - entry[k] = base_entry[k] - - ctx = model.get("context_length") - if ctx is not None: - entry["max_input_tokens"] = int(ctx) - max_out = model.get("max_completion_tokens") - if max_out is not None: - entry["max_output_tokens"] = int(max_out) - entry["max_tokens"] = int(max_out) - - pricing = _pricing(model.get("pricing", {})) - entry.update(pricing) - entry["supports_prompt_caching"] = "cache_read_input_token_cost" in pricing - - reasoning = model.get("reasoning") is True - entry["supports_reasoning"] = reasoning - if reasoning: - entry["reasoning_effort_levels"] = _reasoning_effort_levels( - model.get("reasoning_options", []) - ) - - func = model.get("functionality", {}) - entry["supports_function_calling"] = func.get("tool_call") is True - entry["supports_parallel_function_calling"] = func.get("parallel_tool_call") is True - is_struct = func.get("structured_output") is True - entry["supports_response_schema"] = is_struct - entry["supports_native_structured_output"] = is_struct - entry["supports_system_messages"] = func.get("system_messages") is True - entry["supports_tool_choice"] = func.get("tool_choice") is True - - entry.update(_modality_flags(model.get("input_modalities", []))) - - entry["mode"] = model.get("mode", "chat") - - desc = model.get("description") - if desc: - entry["comment"] = desc - - dep = model.get("deprecation_date") - if dep: - entry["deprecation_date"] = dep.split("T")[0] - - entry["source"] = FRIENDLI_API_URL - - transformed[f"{FRIENDLI_PROVIDER}/{model_id}"] = entry - return transformed - -# Synchronize local data with remote data -def sync_local_data_with_remote(local_data, remote_data, replace_keys=frozenset()): - # Update existing keys in local_data with values from remote_data - # (replace_keys entries are swapped wholesale so a field the remote catalog - # dropped, e.g. cache pricing, cannot survive as a stale value) - for key in (set(local_data) & set(remote_data)): - if key in replace_keys: - local_data[key] = remote_data[key] - else: - local_data[key].update(remote_data[key]) - - # Add new keys from remote_data to local_data - for key in (set(remote_data) - set(local_data)): - local_data[key] = remote_data[key] - -# Write data to the json file -def write_to_file(file_path, data): - try: - # Open the file in write mode - with open(file_path, "w") as file: - # Dump the data as JSON into the file - json.dump(data, file, indent=4) - print("Values updated successfully.") - except Exception as e: - # Print an error message if writing to file fails - print("Error updating JSON file:", e) - -# Update the existing models and add the missing models for OpenRouter -def transform_openrouter_data(data): - transformed = {} - if not data: - return transformed - for row in data: - # Add the fields 'max_tokens' and 'input_cost_per_token' - obj = { - "max_tokens": row["context_length"], - "input_cost_per_token": float(row["pricing"]["prompt"]), - } - - # Add 'max_output_tokens' as a field if it is not None - if "top_provider" in row and "max_completion_tokens" in row["top_provider"] and row["top_provider"]["max_completion_tokens"] is not None: - obj['max_output_tokens'] = int(row["top_provider"]["max_completion_tokens"]) - - # Add the field 'output_cost_per_token' - obj.update({ - "output_cost_per_token": float(row["pricing"]["completion"]), - }) - - # Add field 'input_cost_per_image' if it exists and is non-zero - if "pricing" in row and "image" in row["pricing"] and float(row["pricing"]["image"]) != 0.0: - obj['input_cost_per_image'] = float(row["pricing"]["image"]) - - # Add the fields 'litellm_provider' and 'mode' - obj.update({ - "litellm_provider": "openrouter", - "mode": "chat" - }) - - # Add the 'supports_vision' field if the modality is 'multimodal' - if row.get('architecture', {}).get('modality') == 'multimodal': - obj['supports_vision'] = True - - # Use a composite key to store the transformed object - transformed[f'openrouter/{row["id"]}'] = obj - - return transformed - -# Update the existing models and add the missing models for Vercel AI Gateway -def transform_vercel_ai_gateway_data(data): - transformed = {} - if not data: - return transformed - for row in data: - # Rows without token pricing or token limits (video/embedding models) previously KeyError'd the whole sync - if any(row.get(k) is None for k in ("context_window", "max_tokens")) or any( - row.get("pricing", {}).get(k) is None for k in ("input", "output") - ): - continue - obj = { - "max_tokens": row["context_window"], - "input_cost_per_token": float(row["pricing"]["input"]), - "output_cost_per_token": float(row["pricing"]["output"]), - 'max_output_tokens': row['max_tokens'], - 'max_input_tokens': row["context_window"], - } - - # Handle cache pricing if available - if "pricing" in row: - if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None: - obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}") - - if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None: - obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}") - - mode = "embedding" if "embedding" in row["id"].lower() else "chat" - - obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode}) - - transformed[f'vercel_ai_gateway/{row["id"]}'] = obj - - return transformed - - -# Load local data from a specified file -def load_local_data(file_path): - try: - # Open the file in read mode - with open(file_path, "r") as file: - # Load and return the JSON data - return json.load(file) - except FileNotFoundError: - # Print an error message if the file is not found - print("File not found:", file_path) - return None - except json.JSONDecodeError as e: - # Print an error message if JSON decoding fails - print("Error decoding JSON:", e) - return None - -def main(): - local_file_path = "model_prices_and_context_window.json" # Path to the local data file - openrouter_url = "https://openrouter.ai/api/v1/models" # URL to fetch OpenRouter data - vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models" # URL to fetch Vercel AI Gateway data - - # Load local data from file - local_data = load_local_data(local_file_path) - - # Fetch OpenRouter data - openrouter_data = asyncio.run(fetch_data(openrouter_url)) - # Transform the fetched OpenRouter data - openrouter_data = transform_openrouter_data(openrouter_data) - - # Fetch Vercel AI Gateway data - vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url)) - # Transform the fetched Vercel AI Gateway data - vercel_data = transform_vercel_ai_gateway_data(vercel_data) - - friendli_data = asyncio.run(fetch_data(FRIENDLI_API_URL)) - friendli_data = transform_friendli_data(friendli_data, local_data) - - # Combine both datasets - all_remote_data = {**openrouter_data, **vercel_data, **friendli_data} - - # If both local and openrouter data are available, synchronize and save - if local_data and all_remote_data: - sync_local_data_with_remote(local_data, all_remote_data, replace_keys=frozenset(friendli_data)) - write_to_file(local_file_path, local_data) - else: - print("Failed to fetch model data from either local file or URL.") - -# Entry point of the script -if __name__ == "__main__": - main() diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml deleted file mode 100644 index 838ba35f831..00000000000 --- a/.github/workflows/auto_update_price_and_context_window.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Updates model_prices_and_context_window.json and Create Pull Request - -on: - schedule: - - cron: "0 0 * * 0" # Run every Sundays at midnight - #- cron: "0 0 * * *" # Run daily at midnight - -permissions: - contents: write - pull-requests: write - -jobs: - auto_update_price_and_context_window: - if: github.repository == 'BerriAI/litellm' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - name: Set up uv - uses: ./.github/actions/setup-uv-with-retries - with: - version: "0.10.9" - - name: Update JSON Data - run: | - uv run --frozen --with 'aiohttp==3.13.3' python ".github/scripts/auto_update_price_and_context_window_file.py" - - name: Create Pull Request - run: | - git add model_prices_and_context_window.json - git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')" - gh pr create --title "Update model_prices_and_context_window.json file" \ - --body "Automated update for model_prices_and_context_window.json" \ - --head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \ - --base main - env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} diff --git a/.github/workflows/sync-together-ai-models.yml b/.github/workflows/sync-together-ai-models.yml deleted file mode 100644 index 5325510e45e..00000000000 --- a/.github/workflows/sync-together-ai-models.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Sync Together AI model registry - -on: - schedule: - - cron: "30 6 * * *" - workflow_dispatch: - -permissions: - contents: write - pull-requests: write - -jobs: - sync_together_ai_models: - if: github.repository == 'BerriAI/litellm' - runs-on: ubuntu-latest - env: - BASE_BRANCH: ${{ github.event.repository.default_branch }} - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - ref: ${{ env.BASE_BRANCH }} - persist-credentials: false - - name: Set up uv - uses: ./.github/actions/setup-uv-with-retries - with: - version: "0.10.9" - - name: Look for an already-open sync PR - id: existing - run: | - open_pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --limit 1000 --json headRefName \ - --jq '[.[].headRefName | select(startswith("litellm_together_registry_sync_"))] | first // empty')" - echo "open_pr=$open_pr" >> "$GITHUB_OUTPUT" - if [ -n "$open_pr" ]; then - echo "An open sync PR already exists on branch $open_pr; skipping this run." - fi - env: - GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }} - - name: Run the sync - if: steps.existing.outputs.open_pr == '' - run: | - uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md" - env: - TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} - - name: Create a pull request when the registry changed - if: steps.existing.outputs.open_pr == '' - run: | - if git diff --quiet; then - echo "Registry already in sync; no PR needed." - exit 0 - fi - branch="litellm_together_registry_sync_$(date +'%Y-%m-%d')" - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git checkout -b "$branch" - git add model_prices_and_context_window.json \ - litellm/model_prices_and_context_window_backup.json - git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')" - gh auth setup-git - git push origin "$branch" - gh pr create --title "feat(models): sync together_ai model registry" \ - --body-file "$RUNNER_TEMP/pr_body.md" \ - --head "$branch" \ - --base "$BASE_BRANCH" - env: - GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }} diff --git a/scripts/sync_together_ai_models.py b/scripts/sync_together_ai_models.py deleted file mode 100644 index 12b128890f1..00000000000 --- a/scripts/sync_together_ai_models.py +++ /dev/null @@ -1,547 +0,0 @@ -"""Sync the together_ai entries of model_prices_and_context_window.json with Together's live serverless catalog. - -Pulls ``GET https://api.together.ai/v1/models?serverless`` plus the deprecations doc, maps API fields onto -registry fields, merges the reviewed capability rules below for everything the API cannot express, and diffs -the result against the registry. Dry run (the default) prints the diff summary and the generated PR body; -``--write`` applies the changes to the root cost map and its ``litellm/`` backup copy. - -Policy highlights: -- Prices arrive per 1M tokens with float artifacts and are normalized to clean per-token values. -- A registry entry absent from the serverless catalog is marked with ``deprecation_date`` from the docs - deprecation table, never deleted; absences with no docs date are surfaced for a human call. -- Availability comes from the API: a model the docs list as removed but the API still serves stays live, - with the conflict surfaced as a warning. -- Manually curated values the API cannot express (``metadata.successor``, ``max_output_tokens`` on existing - entries, capability flags no rule covers) are never overwritten; conflicts are surfaced instead. -""" - -import argparse -import json -import os -import re -import sys -from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field -from pathlib import Path -from types import MappingProxyType -from typing import Final - -import httpx -from pydantic import BaseModel, TypeAdapter, ValidationError - -MODELS_URL: Final = "https://api.together.ai/v1/models?serverless" -DEPRECATIONS_URL: Final = "https://docs.together.ai/docs/deprecations.md" -PROVIDER: Final = "together_ai" -PREFIX: Final = "together_ai/" -SOURCE_URL: Final = "https://docs.together.ai/docs/serverless-models" -COST_MAP_RELPATHS: Final = ( - "model_prices_and_context_window.json", - "litellm/model_prices_and_context_window_backup.json", -) - -TYPE_TO_MODE: Final = MappingProxyType({"chat": "chat", "embedding": "embedding", "moderation": "chat"}) - - -class SyncError(RuntimeError): - pass - - -class CatalogPricing(BaseModel): - input: float - output: float - cached_input: float | None = None - - -class CatalogModel(BaseModel): - id: str - type: str - context_length: int | None = None - pricing: CatalogPricing - - -CATALOG_ADAPTER: Final = TypeAdapter(list[CatalogModel]) - -RegistryEntry = dict[str, object] -CostMap = dict[str, object] - - -@dataclass(frozen=True, slots=True) -class CapabilityRule: - model_id: str - fields: Mapping[str, bool | int] - provenance: str - - -def _rule(model_id: str, provenance: str, **fields: bool | int) -> CapabilityRule: - return CapabilityRule(model_id=model_id, fields=MappingProxyType(dict(fields)), provenance=provenance) - - -_TOOLS: Final = MappingProxyType( - { - "supports_function_calling": True, - "supports_parallel_function_calling": True, - "supports_response_schema": True, - "supports_tool_choice": True, - } -) - -CAPABILITY_RULES: Final = ( - _rule( - "MiniMaxAI/MiniMax-M3", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/minimax-m3", - **_TOOLS, - supports_reasoning=True, - supports_vision=True, - ), - _rule("Prism-ML/Ternary-Bonsai-27B", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule( - "Qwen/Qwen3.5-9B", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/qwen3-5-9b", - **_TOOLS, - supports_reasoning=True, - supports_vision=True, - ), - _rule( - "Qwen/Qwen3.6-Plus", - "reviewed for the LIT-5968 backfill; hybrid reasoning model without a documented tools contract", - supports_reasoning=True, - ), - _rule("Qwen/Qwen3.7-Max", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule("Qwen/Qwen3.7-Plus", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule("Qwen/Qwen3.8-2.4T-A95B", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule("arize-ai/qwen-2-1.5b-instruct", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule( - "deepseek-ai/DeepSeek-V4-Flash-0731", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/deepseek-v4-flash", - **_TOOLS, - ), - _rule( - "deepseek-ai/DeepSeek-V4-Pro", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/deepseek-v4-pro", - **_TOOLS, - supports_reasoning=True, - ), - _rule( - "deepseek-ai/DeepSeek-V4-Pro-0813", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/deepseek-v4-pro", - **_TOOLS, - ), - _rule("google/gemma-3n-E4B-it", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule( - "google/gemma-4-31B-it", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/gemma-4-31b-it", - **_TOOLS, - supports_vision=True, - ), - _rule( - "intfloat/multilingual-e5-large-instruct", - "embedding dims per https://huggingface.co/intfloat/multilingual-e5-large-instruct", - output_vector_size=1024, - ), - _rule( - "meta-llama/Llama-3.3-70B-Instruct-Turbo", - "reviewed for the LIT-5968 backfill against https://docs.together.ai/docs/function-calling", - **_TOOLS, - ), - _rule( - "meta-llama/Llama-Guard-4-12B", - "moderation classifier with a chat-shaped API; no tools per the LIT-5968 backfill review", - ), - _rule("meta-models/Muse-Glimmer-30B", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule( - "moonshotai/Kimi-K2.7-Code", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/kimi-k2-7-code", - **_TOOLS, - supports_vision=True, - ), - _rule( - "moonshotai/Kimi-K3", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/kimi-k3", - **_TOOLS, - supports_reasoning=True, - supports_vision=True, - ), - _rule( - "nvidia/nemotron-3-ultra-550b-a55b", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/nemotron-3-ultra", - **_TOOLS, - supports_reasoning=True, - ), - _rule( - "openai/gpt-oss-120b", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/gpt-oss-120b", - **_TOOLS, - supports_reasoning=True, - ), - _rule( - "openai/gpt-oss-20b", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/gpt-oss-20b", - **_TOOLS, - ), - _rule("pearl-ai/gemma-4-31b-it", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule( - "thinkingmachines/Inkling", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/inkling", - **_TOOLS, - ), - _rule("thinkingmachines/Inkling-Small", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule( - "zai-org/GLM-5.2", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/glm-5-2;" - " 128K output ceiling per https://docs.z.ai/guides/llm/glm-5.2", - **_TOOLS, - supports_reasoning=True, - max_output_tokens=128000, - max_tokens=128000, - ), - _rule( - "zai-org/GLM-5.3-Flash", - "reviewed for LIT-6489 against https://www.together.ai/models/glm-5-3-flash;" - " 128K output ceiling per https://docs.z.ai/guides/llm/glm-5.3", - **_TOOLS, - supports_reasoning=True, - supports_vision=True, - max_output_tokens=128000, - max_tokens=128000, - ), -) - -RULES_BY_ID: Final = MappingProxyType({rule.model_id: rule for rule in CAPABILITY_RULES}) - - -@dataclass(frozen=True, slots=True) -class DeprecationDoc: - removal_dates: Mapping[str, str] - redirects: Mapping[str, str] - - -_REDIRECT_ROW: Final = re.compile(r"^\|\s*`([^`]+)`\s*\|\s*`([^`]+)`\s*\|") -_REMOVAL_ROW: Final = re.compile(r"^\|\s*(\d{4}-\d{2}-\d{2})\s*\|\s*`([^`]+)`\s*\|") - - -def _section(markdown: str, heading: str) -> str: - level: Final = heading.split(" ", 1)[0] - start: Final = markdown.find(f"\n{heading}\n") - if start < 0: - return "" - body: Final = markdown[start + 1 + len(heading) :] - next_heading: Final = re.search(rf"^{re.escape(level)} ", body, flags=re.MULTILINE) - return body[: next_heading.start()] if next_heading else body - - -def parse_deprecations(markdown: str) -> DeprecationDoc: - redirect_rows: Final = tuple( - m.groups() - for m in (_REDIRECT_ROW.match(line) for line in _section(markdown, "## Active model redirects").splitlines()) - if m - ) - inference: Final = _section(_section(markdown, "## Deprecation history"), "### Inference") - removal_rows: Final = tuple(m.groups() for m in (_REMOVAL_ROW.match(line) for line in inference.splitlines()) if m) - if not redirect_rows or not removal_rows: - raise SyncError( - "deprecations doc parsed to zero redirect or removal rows; the table format at " - f"{DEPRECATIONS_URL} changed and the parser needs updating" - ) - removal_dates: Final = {model: date for date, model in reversed(removal_rows)} - return DeprecationDoc( - removal_dates=MappingProxyType(dict(reversed(removal_dates.items()))), - redirects=MappingProxyType({original: target for original, target in redirect_rows}), - ) - - -def per_token(price_per_million: float) -> float: - return float(f"{price_per_million / 1e6:.6g}") - - -def _resolve_name(name: str, universe: frozenset[str]) -> str | None: - if name in universe: - return name - suffix_matches: Final = tuple(candidate for candidate in universe if candidate.endswith(f"/{name}")) - return suffix_matches[0] if len(suffix_matches) == 1 else None - - -def resolve_successor(model_id: str, doc: DeprecationDoc, live_ids: frozenset[str]) -> str | None: - canonical: Final = live_ids | frozenset(doc.removal_dates) - redirects: Final = { - (_resolve_name(raw_source, canonical) or raw_source): (_resolve_name(raw_target, canonical) or raw_target) - for raw_source, raw_target in doc.redirects.items() - } - seen: Final = set() - current = model_id # rebind-ok: walks the redirect chain - while current in redirects and current not in seen: - seen.add(current) - current = redirects[current] # rebind-ok: walks the redirect chain - return current if current != model_id and current in live_ids else None - - -@dataclass(frozen=True, slots=True) -class SyncOutcome: - cost_map: CostMap - added: tuple[str, ...] = () - updated: tuple[str, ...] = () - deprecated: tuple[str, ...] = () - reappeared: tuple[str, ...] = () - warnings: tuple[str, ...] = () - skipped_types: Mapping[str, int] = field(default_factory=dict) - - @property - def has_changes(self) -> bool: - return bool(self.added or self.updated or self.deprecated or self.reappeared) - - -def _api_fields(model: CatalogModel) -> RegistryEntry: - cached: Final = model.pricing.cached_input - return { - "input_cost_per_token": per_token(model.pricing.input), - "output_cost_per_token": per_token(model.pricing.output), - **({"cache_read_input_token_cost": per_token(cached), "supports_prompt_caching": True} if cached else {}), - **({"max_input_tokens": model.context_length} if model.context_length is not None else {}), - } - - -def _new_entry(model: CatalogModel, mode: str) -> RegistryEntry: - rule: Final = RULES_BY_ID.get(model.id) - legacy_ceiling: Final = {} if model.context_length is None else {"max_tokens": model.context_length} - merged: Final = { - **_api_fields(model), - **legacy_ceiling, - "litellm_provider": PROVIDER, - "mode": mode, - "source": SOURCE_URL, - **(dict(rule.fields) if rule else {}), - } - return dict(sorted(merged.items())) - - -def _updated_entry(entry: RegistryEntry, model: CatalogModel) -> tuple[RegistryEntry, tuple[str, ...]]: - rule: Final = RULES_BY_ID.get(model.id) - desired: Final = {**_api_fields(model), **(dict(rule.fields) if rule else {})} - dropped: Final = () if model.pricing.cached_input else ("cache_read_input_token_cost", "supports_prompt_caching") - changes: Final = tuple( - f"{name}: {entry.get(name)!r} -> {value!r}" for name, value in desired.items() if entry.get(name) != value - ) + tuple( - f"{name}: {entry[name]!r} removed (no longer in the catalog pricing)" for name in dropped if name in entry - ) - merged: Final = {name: value for name, value in {**entry, **desired}.items() if name not in dropped} - return dict(sorted(merged.items())), changes - - -def _with_new_keys_in_block(original: CostMap, result: CostMap, new_keys: Sequence[str]) -> CostMap: - provider_keys: Final = tuple(key for key in original if key.startswith(PREFIX)) - if not new_keys or not provider_keys: - return result - block_end: Final = provider_keys[-1] - return { - key: value - for existing in original - for key, value in ( - (existing, result[existing]), - *((new, result[new]) for new in sorted(new_keys) if existing == block_end), - ) - } - - -def compute_sync(cost_map: CostMap, catalog: Sequence[CatalogModel], doc: DeprecationDoc) -> SyncOutcome: - live_ids: Final = frozenset(model.id for model in catalog) - token_models: Final = {model.id: model for model in catalog if model.type in TYPE_TO_MODE} - skipped: Final = { - model.type: sum(1 for m in catalog if m.type == model.type) - for model in catalog - if model.type not in TYPE_TO_MODE - } - registry_ids: Final = {key.removeprefix(PREFIX): key for key in cost_map if key.startswith(PREFIX)} - - added: Final[list[str]] = [] - updated: Final[list[str]] = [] - deprecated: Final[list[str]] = [] - reappeared: Final[list[str]] = [] - warnings: Final[list[str]] = [] - result: Final[CostMap] = dict(cost_map) - - for model_id, model in sorted(token_models.items()): - mode: Final = TYPE_TO_MODE[model.type] - key: Final = f"{PREFIX}{model_id}" - if model_id in doc.removal_dates: - warnings.append( - f"`{key}` is listed as removed on {doc.removal_dates[model_id]} in the docs but the serverless " - "catalog still serves it; availability kept from the API" - ) - entry = result.get(key) - if not isinstance(entry, dict): - result[key] = _new_entry(model, mode) - added.append(key) - if model.type == "chat" and model_id not in RULES_BY_ID: - warnings.append( - f"`{key}` added without a capability rule; review its tools/vision/reasoning support and add one" - ) - continue - if entry.get("mode") != mode: - warnings.append( - f"`{key}` has curated mode {entry.get('mode')!r} but the catalog maps to {mode!r}; left unchanged" - ) - new_entry, changes = _updated_entry(entry, model) - if "deprecation_date" in new_entry: - new_entry.pop("deprecation_date") - reappeared.append(key) - if changes: - updated.append(f"{key}: " + "; ".join(changes)) - if changes or key in reappeared: - result[key] = new_entry - - for model_id, key in sorted(registry_ids.items()): - if model_id in token_models: - continue - entry = result.get(key) - if not isinstance(entry, dict): - continue - removal_date: Final = doc.removal_dates.get(model_id) - successor: Final = resolve_successor(model_id, doc, live_ids) - metadata = entry.get("metadata") - curated_successor: Final = metadata.get("successor") if isinstance(metadata, dict) else None - new_entry = dict(entry) - if removal_date is not None and entry.get("deprecation_date") != removal_date: - if "deprecation_date" in entry: - warnings.append( - f"`{key}` has curated deprecation_date {entry.get('deprecation_date')!r} but the docs list " - f"{removal_date!r}; left unchanged" - ) - else: - new_entry["deprecation_date"] = removal_date - if removal_date is None and "deprecation_date" not in entry: - warnings.append( - f"`{key}` is absent from the serverless catalog with no removal date in the docs; " - "needs a human deprecation call" - ) - if successor is not None: - desired_successor: Final = f"{PREFIX}{successor}" - if curated_successor is None: - new_entry["metadata"] = dict( - sorted({**(metadata if isinstance(metadata, dict) else {}), "successor": desired_successor}.items()) - ) - elif curated_successor != desired_successor: - warnings.append( - f"`{key}` has curated successor {curated_successor!r} but the docs redirects resolve to " - f"{desired_successor!r}; left unchanged" - ) - if new_entry != entry: - result[key] = dict(sorted(new_entry.items())) - deprecated.append(f"{key}: " + ", ".join(sorted(set(new_entry) - set(entry)) or ["updated"])) - - return SyncOutcome( - cost_map=_with_new_keys_in_block(cost_map, result, tuple(added)), - added=tuple(added), - updated=tuple(updated), - deprecated=tuple(deprecated), - reappeared=tuple(reappeared), - warnings=tuple(warnings), - skipped_types=MappingProxyType(skipped), - ) - - -def _section_block(title: str, lines: Sequence[str], backtick: bool) -> str: - bullets: Final = "\n".join(f"- `{line}`" if backtick else f"- {line}" for line in lines) or "- none" - return f"### {title} ({len(lines)})\n{bullets}\n" - - -def render_pr_body(outcome: SyncOutcome) -> str: - skipped: Final = ", ".join(f"{kind} ({count})" for kind, count in sorted(outcome.skipped_types.items())) or "none" - return ( - "Automated daily sync of the together_ai entries in model_prices_and_context_window.json against " - f"`GET {MODELS_URL}` and {DEPRECATIONS_URL} by scripts/sync_together_ai_models.py.\n" - "\n" - f"{_section_block('Added', outcome.added, backtick=True)}" - "\n" - f"{_section_block('Updated', outcome.updated, backtick=True)}" - "\n" - f"{_section_block('Marked deprecated', outcome.deprecated, backtick=True)}" - "\n" - f"{_section_block('Returned to the catalog', outcome.reappeared, backtick=True)}" - "\n" - f"{_section_block('Warnings needing a human call', outcome.warnings, backtick=False)}" - "\n" - f"Catalog model types outside the sync's token-pricing scope, skipped: {skipped}\n" - ) - - -def render_summary(outcome: SyncOutcome) -> str: - return ( - f"added={len(outcome.added)} updated={len(outcome.updated)} deprecated={len(outcome.deprecated)} " - f"reappeared={len(outcome.reappeared)} warnings={len(outcome.warnings)}" - ) - - -def load_catalog(raw: bytes) -> list[CatalogModel]: - parsed: Final = json.loads(raw) - entries: Final = parsed.get("data") if isinstance(parsed, dict) else parsed - try: - catalog: Final = CATALOG_ADAPTER.validate_python(entries) - except ValidationError as error: - raise SyncError(f"the catalog response no longer matches the expected shape: {error}") from error - if not any(model.type in TYPE_TO_MODE for model in catalog): - raise SyncError( - "the catalog response contains no token-priced models; refusing to mark the whole registry deprecated" - ) - return catalog - - -def _fetch(url: str, headers: Mapping[str, str]) -> bytes: - response: Final = httpx.get(url, headers=dict(headers), timeout=30, follow_redirects=True) - if response.status_code != 200: - raise SyncError(f"GET {url} returned {response.status_code}") - return response.content - - -def _serialize(cost_map: CostMap) -> str: - return json.dumps(cost_map, indent=4, ensure_ascii=False) + "\n" - - -def main(argv: Sequence[str]) -> int: - parser: Final = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--write", action="store_true", help="apply the sync to the cost map files (default: dry run)") - parser.add_argument("--models-json", type=Path, help="recorded catalog response to use instead of the live API") - parser.add_argument( - "--deprecations-md", type=Path, help="recorded deprecations doc to use instead of the live docs" - ) - parser.add_argument("--pr-body-file", type=Path, help="write the generated PR body to this path") - parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parent.parent) - args: Final = parser.parse_args(argv) - - if args.models_json is not None: - catalog_raw: Final = args.models_json.read_bytes() - else: - api_key: Final = os.environ.get("TOGETHER_API_KEY") - if not api_key: - raise SyncError("TOGETHER_API_KEY is not set and --models-json was not given") - catalog_raw = _fetch(MODELS_URL, {"Authorization": f"Bearer {api_key}"}) # rebind-ok: branch-dependent source - catalog: Final = load_catalog(catalog_raw) - markdown: Final = ( - args.deprecations_md.read_text() if args.deprecations_md is not None else _fetch(DEPRECATIONS_URL, {}).decode() - ) - doc: Final = parse_deprecations(markdown) - - cost_map_path: Final = args.repo_root / COST_MAP_RELPATHS[0] - cost_map: Final = json.loads(cost_map_path.read_text()) - outcome: Final = compute_sync(cost_map, catalog, doc) - body: Final = render_pr_body(outcome) - - if args.pr_body_file is not None: - args.pr_body_file.write_text(body) - if args.write and outcome.has_changes: - for relpath in COST_MAP_RELPATHS: - (args.repo_root / relpath).write_text(_serialize(outcome.cost_map)) - print(render_summary(outcome)) - print() - print(body) - if not args.write: - print("dry run: no files were touched") - elif not outcome.has_changes: - print("registry already in sync: no files were touched") - return 0 - - -if __name__ == "__main__": - try: - raise SystemExit(main(sys.argv[1:])) - except SyncError as error: - print(f"SYNC FAILED: {error}", file=sys.stderr) - raise SystemExit(1) from error diff --git a/tests/unit/test_auto_update_price_and_context_window_file.py b/tests/unit/test_auto_update_price_and_context_window_file.py deleted file mode 100644 index 435747e9a09..00000000000 --- a/tests/unit/test_auto_update_price_and_context_window_file.py +++ /dev/null @@ -1,222 +0,0 @@ -"""Unit tests for the Friendli transform in -`.github/scripts/auto_update_price_and_context_window_file.py`.""" - -from __future__ import annotations - -import importlib.util -import sys -from pathlib import Path - -import pytest - -SCRIPT_PATH = ( - Path(__file__).resolve().parents[2] - / ".github" - / "scripts" - / "auto_update_price_and_context_window_file.py" -) - - -@pytest.fixture(scope="module") -def sync_module(): - spec = importlib.util.spec_from_file_location( - "auto_update_price_and_context_window_file", SCRIPT_PATH - ) - assert spec and spec.loader, f"Could not load spec for {SCRIPT_PATH}" - module = importlib.util.module_from_spec(spec) - sys.modules["auto_update_price_and_context_window_file"] = module - spec.loader.exec_module(module) - return module - - -def _reasoning_model(**overrides: object) -> dict: - model = { - "id": "zai-org/GLM-Test", - "base_model": "zhipuai/glm-test", - "context_length": 1048576, - "max_completion_tokens": 131072, - "pricing": {"input": "0.00000015", "output": "0.0000005", "input_cache_read": "0.00000003"}, - "reasoning": True, - "reasoning_options": [{"type": "effort", "values": ["max", "high", "low"]}], - "functionality": { - "tool_call": True, - "parallel_tool_call": True, - "structured_output": True, - "system_messages": True, - "tool_choice": True, - }, - "input_modalities": ["text", "image", "video"], - "mode": "chat", - } - model.update(overrides) - return model - - -def test_transform_emits_declared_effort_levels_in_canonical_order(sync_module): - entry = sync_module.transform_friendli_data([_reasoning_model()], {})[ - "friendliai/zai-org/GLM-Test" - ] - assert entry["supports_reasoning"] is True - assert entry["reasoning_effort_levels"] == ["low", "high", "max"] - assert not any(k.endswith("_reasoning_effort") for k in entry) - - -def test_transform_reasoning_model_without_effort_options_declares_empty_levels(sync_module): - model = _reasoning_model(reasoning_options=[{"type": "budget_tokens", "values": []}]) - entry = sync_module.transform_friendli_data([model], {})["friendliai/zai-org/GLM-Test"] - assert entry["reasoning_effort_levels"] == [] - - -def test_transform_non_reasoning_model_declares_no_levels(sync_module): - model = _reasoning_model(reasoning=False, reasoning_options=[]) - entry = sync_module.transform_friendli_data([model], {})["friendliai/zai-org/GLM-Test"] - assert entry["supports_reasoning"] is False - assert "reasoning_effort_levels" not in entry - - -def test_transform_max_tokens_mirrors_output_cap_not_context(sync_module): - entry = sync_module.transform_friendli_data([_reasoning_model()], {})[ - "friendliai/zai-org/GLM-Test" - ] - assert entry["max_input_tokens"] == 1048576 - assert entry["max_output_tokens"] == 131072 - assert entry["max_tokens"] == entry["max_output_tokens"] - - -def test_transform_prompt_caching_follows_cache_pricing(sync_module): - cached = sync_module.transform_friendli_data([_reasoning_model()], {})[ - "friendliai/zai-org/GLM-Test" - ] - assert cached["supports_prompt_caching"] is True - assert cached["cache_read_input_token_cost"] == 3e-08 - - uncached_model = _reasoning_model(pricing={"input": "0.00000014", "output": "0.0000004"}) - uncached = sync_module.transform_friendli_data([uncached_model], {})[ - "friendliai/zai-org/GLM-Test" - ] - assert uncached["supports_prompt_caching"] is False - assert "cache_read_input_token_cost" not in uncached - - -def test_transform_modalities_set_vision_image_and_video_flags(sync_module): - entry = sync_module.transform_friendli_data([_reasoning_model()], {})[ - "friendliai/zai-org/GLM-Test" - ] - assert entry["supports_vision"] is True - assert entry["supports_image_input"] is True - assert entry["supports_video_input"] is True - - text_only = _reasoning_model(input_modalities=["text"]) - entry_text = sync_module.transform_friendli_data([text_only], {})[ - "friendliai/zai-org/GLM-Test" - ] - assert entry_text["supports_vision"] is False - assert entry_text["supports_image_input"] is False - assert entry_text["supports_video_input"] is False - - -def test_transform_skips_rows_without_valid_token_prices_so_priced_local_entries_survive(sync_module): - local = { - "friendliai/zai-org/GLM-Test": { - "litellm_provider": "friendliai", - "input_cost_per_token": 1.5e-07, - "output_cost_per_token": 5e-07, - } - } - unpriced_rows = [ - _reasoning_model(pricing={}), - _reasoning_model(pricing=None), - _reasoning_model(pricing={"input": "0.00000015"}), - _reasoning_model(pricing={"output": "0.0000005"}), - _reasoning_model(pricing={"input": "not-a-number", "output": "0.0000005"}), - _reasoning_model(pricing={"input": "-0.00000015", "output": "0.0000005"}), - _reasoning_model(pricing={"input": "inf", "output": "0.0000005"}), - _reasoning_model(pricing={"input": "nan", "output": "0.0000005"}), - ] - remote = sync_module.transform_friendli_data(unpriced_rows, local) - assert remote == {} - sync_module.sync_local_data_with_remote(local, remote, replace_keys=frozenset(remote)) - assert local["friendliai/zai-org/GLM-Test"]["input_cost_per_token"] == 1.5e-07 - assert local["friendliai/zai-org/GLM-Test"]["output_cost_per_token"] == 5e-07 - - -def test_transform_keeps_zero_priced_rows(sync_module): - free_model = _reasoning_model(pricing={"input": "0", "output": "0"}) - entry = sync_module.transform_friendli_data([free_model], {})["friendliai/zai-org/GLM-Test"] - assert entry["input_cost_per_token"] == 0.0 - assert entry["output_cost_per_token"] == 0.0 - - -def test_transforms_survive_failed_fetch(sync_module): - assert sync_module.transform_friendli_data(None, {}) == {} - assert sync_module.transform_friendli_data([], {}) == {} - assert sync_module.transform_openrouter_data(None) == {} - assert sync_module.transform_vercel_ai_gateway_data(None) == {} - - -def test_vercel_transform_skips_rows_without_token_pricing_or_limits(sync_module): - rows = [ - { - "id": "wan-video", - "pricing": {"video_duration_pricing": [{"resolution": "720p", "cost_per_second": "0.1"}]}, - }, - { - "id": "qwen3-embedding", - "context_window": 32768, - "max_tokens": 32768, - "pricing": {"input": "0.00000001"}, - }, - { - "id": "no-limits-chat", - "pricing": {"input": "0.000001", "output": "0.000002"}, - }, - { - "id": "good-chat", - "context_window": 128000, - "max_tokens": 8192, - "pricing": {"input": "0.000001", "output": "0.000002"}, - }, - ] - transformed = sync_module.transform_vercel_ai_gateway_data(rows) - assert list(transformed) == ["vercel_ai_gateway/good-chat"] - assert transformed["vercel_ai_gateway/good-chat"]["input_cost_per_token"] == 1e-06 - assert transformed["vercel_ai_gateway/good-chat"]["output_cost_per_token"] == 2e-06 - - -def test_sync_replaces_friendli_entries_so_dropped_cache_pricing_does_not_survive(sync_module): - local = { - "friendliai/zai-org/GLM-Test": { - "litellm_provider": "friendliai", - "cache_read_input_token_cost": 3e-08, - "supports_prompt_caching": True, - } - } - uncached_model = _reasoning_model(pricing={"input": "0.00000014", "output": "0.0000004"}) - remote = sync_module.transform_friendli_data([uncached_model], local) - sync_module.sync_local_data_with_remote(local, remote, replace_keys=frozenset(remote)) - synced = local["friendliai/zai-org/GLM-Test"] - assert "cache_read_input_token_cost" not in synced - assert synced["supports_prompt_caching"] is False - - -def test_sync_still_merges_entries_outside_replace_keys(sync_module): - local = {"openrouter/some-model": {"input_cost_per_token": 1e-06, "supports_vision": True}} - remote = {"openrouter/some-model": {"input_cost_per_token": 2e-06}} - sync_module.sync_local_data_with_remote(local, remote) - assert local["openrouter/some-model"] == {"input_cost_per_token": 2e-06, "supports_vision": True} - - -def test_transform_inherits_allowlisted_keys_from_base_model_entry(sync_module): - local = { - "zhipuai/glm-test": { - "supports_pdf_input": True, - "supports_assistant_prefill": True, - "input_cost_per_token": 9e-06, - } - } - entry = sync_module.transform_friendli_data([_reasoning_model()], local)[ - "friendliai/zai-org/GLM-Test" - ] - assert entry["supports_pdf_input"] is True - assert entry["supports_assistant_prefill"] is True - assert entry["input_cost_per_token"] == 1.5e-07 diff --git a/tests/unit/test_sync_together_ai_models.py b/tests/unit/test_sync_together_ai_models.py deleted file mode 100644 index b8a85bcfbdc..00000000000 --- a/tests/unit/test_sync_together_ai_models.py +++ /dev/null @@ -1,378 +0,0 @@ -import importlib.util -import json -from pathlib import Path -from types import MappingProxyType - -import pytest - -ROOT = Path(__file__).resolve().parents[2] -SCRIPT = ROOT / "scripts" / "sync_together_ai_models.py" -FIXTURES = Path(__file__).resolve().parent / "fixtures" / "together_ai_sync" - -_spec = importlib.util.spec_from_file_location("sync_together_ai_models", SCRIPT) -assert _spec is not None and _spec.loader is not None -sync = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(sync) - -RECORDED_CATALOG = sync.load_catalog(FIXTURES.joinpath("models_serverless.json").read_bytes()) -RECORDED_DOC = sync.parse_deprecations(FIXTURES.joinpath("deprecations.md").read_text()) - - -def _doc(removal_dates: dict[str, str], redirects: dict[str, str] | None = None) -> object: - return sync.DeprecationDoc( - removal_dates=MappingProxyType(removal_dates), - redirects=MappingProxyType(redirects or {}), - ) - - -def _chat_model(model_id: str, ctx: int = 4096, price: float = 1.0, cached: float | None = None) -> object: - return sync.CatalogModel( - id=model_id, - type="chat", - context_length=ctx, - pricing=sync.CatalogPricing(input=price, output=price, cached_input=cached), - ) - - -@pytest.mark.parametrize( - ("per_million", "expected"), - [ - (3, 3e-06), - (15, 1.5e-05), - (1.4, 1.4e-06), - (0.25999999999999995, 2.6e-07), - (0.060000000000000005, 6e-08), - (1.0399999999999998, 1.04e-06), - (0, 0.0), - ], -) -def test_per_token_normalizes_float_artifacts(per_million: float, expected: float) -> None: - assert sync.per_token(per_million) == expected - - -def test_parse_deprecations_recorded_fixture() -> None: - assert dict(RECORDED_DOC.redirects) == { - "mistralai/Mistral-7B-Instruct-v0.3": "mistralai/Ministral-3-14B-Instruct-2512", - "Kimi-K2": "Kimi-K2-0905", - "DeepSeek-V3": "DeepSeek-V3.1", - "DeepSeek-V3-0324": "DeepSeek-V3.1", - "DeepSeek-R1": "DeepSeek-R1-0528", - } - assert len(RECORDED_DOC.removal_dates) == 208 - assert RECORDED_DOC.removal_dates["google/gemma-3n-E4B-it"] == "2026-08-04" - - -def test_parse_deprecations_duplicate_rows_keep_most_recent_date() -> None: - assert RECORDED_DOC.removal_dates["Qwen/Qwen3-235B-A22B-Thinking-2507"] == "2026-04-16" - - -@pytest.mark.parametrize( - "markdown", - [ - "# Deprecations\n\nNothing here anymore.\n", - "\n## Active model redirects\n\n| A | B |\n| --- | --- |\n| `x` | `y` |\n\n## Something else\n", - "\n## Deprecation history\n\n### Inference\n\n| Date | Model | R |\n| --- | --- | --- |\n| 2026-01-01 | `m` | No |\n", - ], -) -def test_parse_deprecations_raises_when_a_table_parses_empty(markdown: str) -> None: - with pytest.raises(sync.SyncError): - sync.parse_deprecations(markdown) - - -def test_load_catalog_raises_on_shape_change() -> None: - with pytest.raises(sync.SyncError): - sync.load_catalog(b'[{"id": "x", "type": "chat"}]') - - -def test_load_catalog_raises_when_no_token_models_remain() -> None: - only_video = json.dumps([{"id": "v", "type": "video", "pricing": {"input": 0, "output": 0}}]).encode() - with pytest.raises(sync.SyncError): - sync.load_catalog(only_video) - - -def test_recorded_catalog_counts() -> None: - assert len(RECORDED_CATALOG) == 102 - assert sum(1 for model in RECORDED_CATALOG if model.type in sync.TYPE_TO_MODE) == 26 - assert sum(1 for model in RECORDED_CATALOG if model.pricing.cached_input) == 13 - - -def test_added_chat_model_matches_reviewed_registry_shape() -> None: - outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) - assert len(outcome.added) == 26 - assert not outcome.deprecated - assert outcome.cost_map["together_ai/moonshotai/Kimi-K3"] == { - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 1048576, - "max_tokens": 1048576, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "source": "https://docs.together.ai/docs/serverless-models", - "supports_function_calling": True, - "supports_parallel_function_calling": True, - "supports_prompt_caching": True, - "supports_reasoning": True, - "supports_response_schema": True, - "supports_tool_choice": True, - "supports_vision": True, - } - - -def test_added_embedding_model_has_no_output_token_cap() -> None: - outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) - assert outcome.cost_map["together_ai/intfloat/multilingual-e5-large-instruct"] == { - "input_cost_per_token": 2e-08, - "litellm_provider": "together_ai", - "max_input_tokens": 514, - "max_tokens": 514, - "mode": "embedding", - "output_cost_per_token": 2e-08, - "output_vector_size": 1024, - "source": "https://docs.together.ai/docs/serverless-models", - } - - -def test_moderation_type_maps_to_chat_mode() -> None: - outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) - guard = outcome.cost_map["together_ai/meta-llama/Llama-Guard-4-12B"] - assert guard["mode"] == "chat" - assert "max_output_tokens" not in guard - - -def test_output_ceiling_comes_from_the_rule_never_from_context_length() -> None: - glm = next(model for model in RECORDED_CATALOG if model.id == "zai-org/GLM-5.2") - fresh = sync.compute_sync({}, [_chat_model("acme/unreviewed", ctx=1048576), glm], _doc({"x": "2026-01-01"})) - unreviewed = fresh.cost_map["together_ai/acme/unreviewed"] - assert "max_output_tokens" not in unreviewed - assert (unreviewed["max_input_tokens"], unreviewed["max_tokens"]) == (1048576, 1048576) - reviewed = fresh.cost_map["together_ai/zai-org/GLM-5.2"] - assert (reviewed["max_input_tokens"], reviewed["max_output_tokens"], reviewed["max_tokens"]) == ( - 1048575, - 128000, - 128000, - ) - inflated = { - "together_ai/zai-org/GLM-5.2": { - "input_cost_per_token": 1.4e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 1048575, - "max_output_tokens": 1048575, - "max_tokens": 1048575, - "mode": "chat", - "output_cost_per_token": 4.4e-06, - } - } - corrected = sync.compute_sync(inflated, [glm], _doc({"x": "2026-01-01"})) - assert corrected.cost_map["together_ai/zai-org/GLM-5.2"]["max_output_tokens"] == 128000 - assert any("max_output_tokens: 1048575 -> 128000" in line for line in corrected.updated) - - -def test_docs_removed_but_live_model_stays_live_with_warning() -> None: - outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) - gemma = outcome.cost_map["together_ai/google/gemma-3n-E4B-it"] - assert "deprecation_date" not in gemma - assert any("gemma-3n-E4B-it" in warning and "2026-08-04" in warning for warning in outcome.warnings) - - -def test_price_change_updates_api_fields_and_keeps_curated_ones() -> None: - registry = { - "together_ai/acme/chat-1": { - "input_cost_per_token": 9e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 4096, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 9e-07, - "supports_audio_input": True, - } - } - outcome = sync.compute_sync(registry, [_chat_model("acme/chat-1", ctx=8192, price=2.0)], _doc({"x": "2026-01-01"})) - entry = outcome.cost_map["together_ai/acme/chat-1"] - assert entry["input_cost_per_token"] == 2e-06 - assert entry["max_input_tokens"] == 8192 - assert entry["max_output_tokens"] == 2048 - assert entry["supports_audio_input"] is True - assert len(outcome.updated) == 1 - assert "input_cost_per_token" in outcome.updated[0] - - -def test_cached_input_appearing_and_disappearing() -> None: - registry = { - "together_ai/acme/chat-1": { - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1e-06, - "supports_prompt_caching": True, - }, - "together_ai/acme/chat-2": { - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1e-06, - }, - } - catalog = [_chat_model("acme/chat-1"), _chat_model("acme/chat-2", cached=0.25999999999999995)] - outcome = sync.compute_sync(registry, catalog, _doc({"x": "2026-01-01"})) - assert "cache_read_input_token_cost" not in outcome.cost_map["together_ai/acme/chat-1"] - assert "supports_prompt_caching" not in outcome.cost_map["together_ai/acme/chat-1"] - assert outcome.cost_map["together_ai/acme/chat-2"]["cache_read_input_token_cost"] == 2.6e-07 - assert outcome.cost_map["together_ai/acme/chat-2"]["supports_prompt_caching"] is True - - -def test_capability_rule_backfills_existing_entry() -> None: - registry = { - "together_ai/moonshotai/Kimi-K3": { - "input_cost_per_token": 3e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 1048576, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - } - } - kimi = next(model for model in RECORDED_CATALOG if model.id == "moonshotai/Kimi-K3") - outcome = sync.compute_sync(registry, [kimi], _doc({"x": "2026-01-01"})) - assert outcome.cost_map["together_ai/moonshotai/Kimi-K3"]["supports_reasoning"] is True - assert any("supports_reasoning" in line for line in outcome.updated) - - -def test_disappeared_model_gets_docs_date_and_is_never_deleted() -> None: - registry = { - "together_ai/acme/gone": { - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1e-06, - } - } - outcome = sync.compute_sync(registry, [_chat_model("acme/alive")], _doc({"acme/gone": "2026-07-01"})) - assert outcome.cost_map["together_ai/acme/gone"]["deprecation_date"] == "2026-07-01" - assert outcome.deprecated == ("together_ai/acme/gone: deprecation_date",) - - -def test_disappeared_model_without_docs_date_warns_instead() -> None: - registry = { - "together_ai/acme/gone": { - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1e-06, - } - } - outcome = sync.compute_sync(registry, [_chat_model("acme/alive")], _doc({"other": "2026-07-01"})) - assert "deprecation_date" not in outcome.cost_map["together_ai/acme/gone"] - assert not outcome.deprecated - assert any("acme/gone" in warning and "human" in warning for warning in outcome.warnings) - - -def test_curated_deprecation_date_is_never_overwritten() -> None: - registry = { - "together_ai/acme/gone": { - "deprecation_date": "2026-06-15", - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1e-06, - } - } - outcome = sync.compute_sync(registry, [_chat_model("acme/alive")], _doc({"acme/gone": "2026-07-01"})) - assert outcome.cost_map["together_ai/acme/gone"]["deprecation_date"] == "2026-06-15" - assert any("2026-06-15" in warning and "2026-07-01" in warning for warning in outcome.warnings) - - -def test_redirect_chain_resolves_to_final_live_model() -> None: - doc = _doc({"acme/a": "2026-01-01"}, redirects={"acme/a": "acme/b", "acme/b": "acme/c"}) - live = frozenset({"acme/c"}) - assert sync.resolve_successor("acme/a", doc, live) == "acme/c" - - -def test_redirect_dead_end_yields_no_successor() -> None: - doc = _doc({"acme/a": "2026-01-01"}, redirects={"acme/a": "acme/b"}) - assert sync.resolve_successor("acme/a", doc, frozenset({"acme/other"})) is None - - -def test_redirect_short_names_resolve_by_unique_suffix() -> None: - doc = _doc({"moonshotai/Kimi-K2": "2026-01-01"}, redirects={"Kimi-K2": "Kimi-K2-0905"}) - live = frozenset({"moonshotai/Kimi-K2-0905"}) - assert sync.resolve_successor("moonshotai/Kimi-K2", doc, live) == "moonshotai/Kimi-K2-0905" - - -def test_successor_written_only_when_not_curated() -> None: - registry = { - "together_ai/acme/a": { - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1e-06, - }, - "together_ai/acme/b": { - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "metadata": {"successor": "together_ai/acme/curated"}, - "mode": "chat", - "output_cost_per_token": 1e-06, - }, - } - doc = _doc({"acme/a": "2026-01-01", "acme/b": "2026-01-01"}, redirects={"acme/a": "acme/c", "acme/b": "acme/c"}) - outcome = sync.compute_sync(registry, [_chat_model("acme/c")], doc) - assert outcome.cost_map["together_ai/acme/a"]["metadata"] == {"successor": "together_ai/acme/c"} - assert outcome.cost_map["together_ai/acme/b"]["metadata"] == {"successor": "together_ai/acme/curated"} - assert any("acme/curated" in warning for warning in outcome.warnings) - - -def test_reappearance_clears_deprecation_date() -> None: - registry = { - "together_ai/acme/back": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1e-06, - } - } - outcome = sync.compute_sync(registry, [_chat_model("acme/back")], _doc({"x": "2026-01-01"})) - assert "deprecation_date" not in outcome.cost_map["together_ai/acme/back"] - assert outcome.reappeared == ("together_ai/acme/back",) - - -def test_new_chat_model_without_rule_is_flagged() -> None: - outcome = sync.compute_sync({}, [_chat_model("acme/unreviewed")], _doc({"x": "2026-01-01"})) - assert any("acme/unreviewed" in warning and "capability rule" in warning for warning in outcome.warnings) - - -def test_new_keys_land_at_the_end_of_the_provider_block() -> None: - registry = { - "aaa": {"mode": "chat"}, - "together_ai/acme/old": { - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1e-06, - }, - "zzz": {"mode": "chat"}, - } - outcome = sync.compute_sync(registry, [_chat_model("acme/old"), _chat_model("acme/new")], _doc({"x": "2026-01-01"})) - assert list(outcome.cost_map) == ["aaa", "together_ai/acme/old", "together_ai/acme/new", "zzz"] - - -def test_sync_is_idempotent_over_the_repo_cost_map() -> None: - cost_map = json.loads((ROOT / "model_prices_and_context_window.json").read_text()) - first = sync.compute_sync(cost_map, RECORDED_CATALOG, RECORDED_DOC) - second = sync.compute_sync(first.cost_map, RECORDED_CATALOG, RECORDED_DOC) - assert not second.has_changes - assert second.cost_map == first.cost_map - - -def test_pr_body_lists_every_section_and_the_skipped_types() -> None: - outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) - body = sync.render_pr_body(outcome) - assert "### Added (26)" in body - assert "### Warnings needing a human call" in body - assert "image (29)" in body - assert "video (38)" in body From 8b1910c95ae7f9c47135cb41c1e7bbb2700f36e5 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 26 Oct 2026 18:30:05 -0400 Subject: [PATCH 073/155] fix(mcp): advertise valid OAuth resources for toolset URLs (#48474) --- .../mcp_server/auth/user_api_key_auth_mcp.py | 8 +- litellm/proxy/proxy_server.py | 3 +- tests/integration/mcp/test_mcp_oauth_flows.py | 92 ++++++++++++++++--- .../auth/test_user_api_key_auth_mcp.py | 41 +++++++++ 4 files changed, 129 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 824633bdf5a..f883031a529 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -27,6 +27,7 @@ from litellm.proxy._experimental.mcp_server.oauth_utils import ( get_passthrough_resource_metadata_url, get_passthrough_www_authenticate, get_request_base_url, + get_route_relative_request_path, well_known_root_suffix, ) from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( @@ -387,7 +388,7 @@ def _gateway_dcr_challenge( """The RFC 9728 challenge pointing the client at the protected-resource metadata matching the scope it requested: the per-server document (same URL spelling the request arrived on) when the single target advertises gateway-owned sign-in, - else the gateway's aggregate document. Either way the client discovers the gateway + else a gateway document containing the public resource. Either way the client discovers the gateway as its authorization server and starts the same sign-in flow. ``invalid_token`` adds the RFC 6750 error code for a request that DID @@ -397,10 +398,13 @@ def _gateway_dcr_challenge( client_ip: Final = IPAddressUtils.get_mcp_client_ip(request) target: Final = _gateway_dcr_challenge_target(route, mcp_servers, client_ip) server_scope: Final = gateway_dcr_challenge_scope(route, mcp_servers, client_ip) if target is None else None + public_path: Final = get_route_relative_request_path(request.scope).rstrip("/") + aggregate_resource: Final = public_path == "/mcp" or public_path.startswith("/mcp/") + metadata_suffix: Final = f"{well_known_root_suffix()}/mcp" if aggregate_resource else "" resource_metadata_url: Final = ( get_passthrough_resource_metadata_url(request.scope, target) if target is not None - else f"{get_request_base_url(request)}/.well-known/oauth-protected-resource{well_known_root_suffix()}/mcp" + else f"{get_request_base_url(request)}/.well-known/oauth-protected-resource{metadata_suffix}" ) error_attr: Final = 'error="invalid_token", ' if invalid_token else "" scope_attr: Final = f', scope="{server_scope}"' if server_scope is not None else "" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index df948b69a67..129cd7e1589 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -21795,8 +21795,7 @@ async def toolset_mcp_route(toolset_name: str, request: Request): detail=f"Toolset '{toolset_name}' not found", ) - scope: Final = dict(request.scope) - scope["path"] = "/mcp" + scope: Final = {**request.scope, "_original_path": request.scope.get("path", ""), "path": "/mcp"} token: Final = mcp_active_toolset_id.set(toolset.toolset_id) try: diff --git a/tests/integration/mcp/test_mcp_oauth_flows.py b/tests/integration/mcp/test_mcp_oauth_flows.py index 327b053d333..90a462472b2 100644 --- a/tests/integration/mcp/test_mcp_oauth_flows.py +++ b/tests/integration/mcp/test_mcp_oauth_flows.py @@ -792,17 +792,23 @@ def _ui_session_cookie(gateway: Gateway, user_id: str) -> dict[str, str]: def _gateway_session_sign_in( - gateway: Gateway, user_id: str, resource: str | None = None + gateway: Gateway, + user_id: str, + resource: str | None = None, + *, + registration_endpoint: str = "/register", + authorization_endpoint: str = "/authorize", + token_endpoint: str = "/token", ) -> tuple[dict[str, object], str]: registered: Final = gateway.client.post( - "/register", json={"redirect_uris": [CLIENT_REDIRECT], "client_name": "integration"} + registration_endpoint, json={"redirect_uris": [CLIENT_REDIRECT], "client_name": "integration"} ) assert registered.status_code in (200, 201), registered.text client_id: Final = str(_response_object(registered)["client_id"]) pkce: Final = _Pkce(secrets.token_urlsafe(48)) cookies: Final = _ui_session_cookie(gateway, user_id) started: Final = gateway.client.get( - "/authorize", + authorization_endpoint, params={ "client_id": client_id, "redirect_uri": CLIENT_REDIRECT, @@ -823,13 +829,14 @@ def _gateway_session_sign_in( callback: Final = parse_qs(urlsplit(completed.headers["location"]).query) assert "code" in callback, completed.headers["location"] issued: Final = gateway.client.post( - "/token", + token_endpoint, data={ "grant_type": "authorization_code", "code": callback["code"][0], "redirect_uri": CLIENT_REDIRECT, "client_id": client_id, "code_verifier": pkce.verifier, + **({} if resource is None else {"resource": resource}), }, ) assert issued.status_code == 200, issued.text @@ -1292,10 +1299,18 @@ def test_lite_logout_revokes_the_cli_refresh_token_sent_the_way_the_cli_sends_it assert completion.status_code == 200, completion.text -def _toolset_rpc(gateway: Gateway, bearer: str, name: str, method: str, params: dict[str, object]) -> Outcome: +def _toolset_rpc( + gateway: Gateway, + bearer: str, + name: str, + method: str, + params: dict[str, object], + *, + route: str = "/toolset/{name}/mcp", +) -> Outcome: def post(rpc_method: str, rpc_params: dict[str, object]) -> httpx.Response: return gateway.client.post( - f"/toolset/{name}/mcp", + route.format(name=name), json={"jsonrpc": "2.0", "id": 1, "method": rpc_method, "params": rpc_params}, headers={"Authorization": f"Bearer {bearer}", "Accept": "application/json, text/event-stream"}, ) @@ -1306,7 +1321,12 @@ def _toolset_rpc(gateway: Gateway, bearer: str, name: str, method: str, params: return _outcome_from_rpc(post(method, params)) -def test_gateway_session_bearer_of_a_team_member_is_served_the_team_toolset_on_its_route(gateway: Gateway) -> None: +@pytest.mark.parametrize("route", ["/toolset/{name}/mcp", "/{name}/mcp"]) +def test_gateway_session_bearer_of_a_team_member_is_served_the_team_toolset_on_its_route( + gateway: Gateway, route: str +) -> None: + from mcp.shared.auth_utils import check_resource_allowed + with mcp_peer() as peer, gateway.scenario() as scenario: alias: Final = "lit6029sess" + uuid.uuid4().hex[:6] server_id: Final = register_mcp(scenario, peer, alias) @@ -1315,19 +1335,69 @@ def test_gateway_session_bearer_of_a_team_member_is_served_the_team_toolset_on_i granted_id: Final = create_toolset(scenario, ((server_id, "add"),), toolset_name=granted_name) create_toolset(scenario, ((server_id, "multiply"),), toolset_name=withheld_name) member: Final = scenario.member(scenario.team(object_permission={"mcp_toolsets": [granted_id]})) - bearer: Final = _gateway_session_bearer(gateway, member) + path: Final = route.format(name=granted_name) + anonymous: Final = gateway.client.post( + path, + headers=ACCEPT, + json={"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": dict(INITIALIZE)}, + ) + assert anonymous.status_code == 401, anonymous.text + metadata_url: Final = anonymous.headers["www-authenticate"].split('resource_metadata="', 1)[1].split('"', 1)[0] + protected: Final = gateway.client.get(metadata_url) + assert protected.status_code == 200, protected.text + protected_body: Final = _response_object(protected) + resource: Final = str(protected_body["resource"]) + assert check_resource_allowed( + requested_resource=f"{_base(gateway)}{path}", configured_resource=resource + ), protected.text + issuers: Final = TypeAdapter(tuple[str, ...]).validate_python(protected_body["authorization_servers"]) + assert len(issuers) == 1, protected.text + issuer: Final = urlsplit(issuers[0]) + authorization: Final = gateway.client.get( + f"{issuer.scheme}://{issuer.netloc}/.well-known/oauth-authorization-server{issuer.path}" + ) + assert authorization.status_code == 200, authorization.text + authorization_body: Final = _response_object(authorization) + assert authorization_body["issuer"] == issuers[0], authorization.text + token_endpoint: Final = str(authorization_body["token_endpoint"]) + issued, client_id = _gateway_session_sign_in( + gateway, + member, + resource, + registration_endpoint=str(authorization_body["registration_endpoint"]), + authorization_endpoint=str(authorization_body["authorization_endpoint"]), + token_endpoint=token_endpoint, + ) + bearer: Final = _issued_token(issued) assert bearer.startswith("llm_session_"), bearer[:16] - listed: Final = _toolset_rpc(gateway, bearer, granted_name, "tools/list", {}) + listed: Final = _toolset_rpc(gateway, bearer, granted_name, "tools/list", {}, route=route) assert listed.tools == (f"{alias}-add",), listed.raw peer.drain() called: Final = _toolset_rpc( - gateway, bearer, granted_name, "tools/call", {"name": f"{alias}-add", "arguments": {"a": 4, "b": 5}} + gateway, bearer, granted_name, "tools/call", + {"name": f"{alias}-add", "arguments": {"a": 4, "b": 5}}, route=route ) assert called.ok and called.text == "9", called.raw assert len(tool_calls(peer.drain())) == 1 - denied: Final = _toolset_rpc(gateway, bearer, withheld_name, "tools/list", {}) + denied: Final = _toolset_rpc(gateway, bearer, withheld_name, "tools/list", {}, route=route) assert denied.status == 403, denied.raw assert tool_calls(peer.drain()) == () + rotated: Final = gateway.client.post( + token_endpoint, + data={ + "grant_type": "refresh_token", + "refresh_token": str(issued["refresh_token"]), + "client_id": client_id, + "resource": resource, + }, + ) + assert rotated.status_code == 200, rotated.text + rotated_body: Final = _response_object(rotated) + assert rotated_body["refresh_token"] != issued["refresh_token"], rotated.text + refreshed: Final = _toolset_rpc( + gateway, _issued_token(rotated_body), granted_name, "tools/list", {}, route=route + ) + assert refreshed.tools == listed.tools, refreshed.raw def test_resource_scoped_session_bearer_opens_a_team_toolset_inside_its_server_and_none_outside( diff --git a/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index e8689b2ba89..16130364364 100644 --- a/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -7693,6 +7693,47 @@ class TestMCPDcrBridgeDualCredential: assert "invalid_token" in str(exc_info.value.headers) +@pytest.mark.parametrize("public_path", ["/toolset/analysis/mcp", "/analysis/mcp", "/mcp", "/mcp/", "/mcp/proxy"]) +@pytest.mark.parametrize("root_path", ["", "/litellm"]) +@pytest.mark.parametrize("invalid_token", [False, True]) +def test_gateway_challenge_metadata_contains_the_public_resource( + public_path: str, root_path: str, invalid_token: bool, +) -> None: + from mcp.shared.auth_utils import check_resource_allowed + from starlette.requests import Request + from starlette.types import Scope + + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import _gateway_dcr_challenge + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + _build_aggregate_protected_resource_response, + oauth_protected_resource_root, + ) + + scope: Final[Scope] = { + "type": "http", "method": "POST", "scheme": "https", "path": "/mcp", + "root_path": root_path, "_original_path": f"{root_path}{public_path}", + "headers": [(b"host", b"gateway.example.com")], + } + request: Final = Request(scope) + challenge: Final = _gateway_dcr_challenge(request, "/mcp", None, invalid_token) + header: Final = (challenge.headers or {})["WWW-Authenticate"] + metadata_url: Final = header.split('resource_metadata="', 1)[1].split('"', 1)[0] + toolset: Final = public_path in ("/toolset/analysis/mcp", "/analysis/mcp") + assert metadata_url == ( + f"https://gateway.example.com{root_path}/.well-known/oauth-protected-resource" + + ("" if toolset else "/mcp") + ) + metadata: Final = ( + oauth_protected_resource_root(request) if toolset else _build_aggregate_protected_resource_response(request) + ) + assert check_resource_allowed( + requested_resource=f"https://gateway.example.com{root_path}{public_path}", + configured_resource=metadata["resource"], + ) + assert challenge.status_code == 401 + assert ('error="invalid_token"' in header) is invalid_token + + @pytest.mark.asyncio class TestAggregateGatewayDcrChallenge: """The mcp_gateway_dcr front door: a 401 on the aggregate /mcp scope must From 48a893aefc1f4cee13f41d2b6f5bde8b382aa6b9 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 26 Oct 2026 15:38:17 -0700 Subject: [PATCH 074/155] build(deps): drop expired exclude-newer pins for certifi, mlflow and pypdf (#48461) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- pyproject.toml | 8 -------- uv.lock | 5 ----- 2 files changed, 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 88bb0b13d91..fc860a8fdf4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -377,16 +377,8 @@ required-version = ">=0.10.9" exclude-newer = "3 days" [tool.uv.exclude-newer-package] -# TODO: ryan, remove after 2026-10-23, once certifi 2026.10.19 (drops the compromised eMudhra root) clears the 3-day window -certifi = "2026-10-20T03:00:00Z" # TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window brotli = "2026-10-22T14:01:00Z" -# TODO: ryan, remove after 2026-10-24, once mlflow 3.17.2 (GHSA-wqmh-qccw-pqwf) clears the 3-day window -mlflow = "2026-10-21T03:00:00Z" -mlflow-skinny = "2026-10-21T03:00:00Z" -mlflow-tracing = "2026-10-21T03:00:00Z" -# TODO: ryan, remove after 2026-10-25, once pypdf 6.21.2 (GHSA-6c7f-9265-g8fw) clears the 3-day window -pypdf = "2026-10-21T17:00:00Z" [tool.uv.sources] litellm-proxy-extras = { workspace = true } diff --git a/uv.lock b/uv.lock index fc4155524d7..f86614046e1 100644 --- a/uv.lock +++ b/uv.lock @@ -16,11 +16,6 @@ exclude-newer-span = "P3D" [options.exclude-newer-package] brotli = "2026-10-22T14:01:00Z" -certifi = "2026-10-20T03:00:00Z" -mlflow = "2026-10-21T03:00:00Z" -mlflow-skinny = "2026-10-21T03:00:00Z" -mlflow-tracing = "2026-10-21T03:00:00Z" -pypdf = "2026-10-21T17:00:00Z" [manifest] members = [ From 9a35ed93f9c9dbf7778c28fb3250a9df7cec4e70 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Mon, 26 Oct 2026 15:40:03 -0700 Subject: [PATCH 075/155] feat(ui): simplify auto router setup with deployed model defaults (#48451) * feat(ui): simplify auto router setup with deployed model defaults * fix(ui): recognize catalog Opus deployment formats * fix(ui): address auto router setup review gaps * fix(ui): preserve Gemini auto router suggestions * fix(ui): prefer current aliases and wait for team selection * fix(ui): default auto routers to classify each user turn * fix(ui): parse Grok snapshot dates separately from releases * refactor(ui): use non-mutating auto router sorting * feat(ui): allow custom models in simple auto router setup --- .../app/(dashboard)/hooks/models/useModels.ts | 1 + .../AutoRouters/AutoRoutersPanel.tsx | 5 +- .../add_model/AutoRouterPresetSelect.tsx | 125 ++++ .../add_model/AutoRouterSimpleSetup.tsx | 190 +++++++ .../add_auto_router_tab.integration.test.tsx | 458 ++++++++++----- .../add_model/add_auto_router_tab.tsx | 432 +++++++------- .../components/add_model/auto_setup.test.ts | 537 ++++++++++-------- .../src/components/add_model/auto_setup.ts | 347 +++++++---- 8 files changed, 1413 insertions(+), 682 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/add_model/AutoRouterPresetSelect.tsx create mode 100644 ui/litellm-dashboard/src/components/add_model/AutoRouterSimpleSetup.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts index 24f76065f34..744db87ad11 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts @@ -121,6 +121,7 @@ export interface AutoRouterDeployment extends AutoRouterCandidateDeployment { /** False for config.yaml-defined deployments, which the update and delete routes refuse. */ db_model?: boolean | null; base_model?: string | null; + blocked?: boolean | null; mode?: string | null; created_at?: string | null; updated_at?: string | null; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx index 6f7e04f92c6..2d1c24685f0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx @@ -106,10 +106,7 @@ export function AutoRoutersPanel({ > Add Auto Router - - Choose a classifier to route each request to a model. Called like any other model, so clients keep using a - single model name. - + Choose models for your auto router. { + switch (availability.kind) { + case "available": + return null; + case "loading": + return "Checking model availability..."; + case "unverifiable": + return "Cannot verify these models are available"; + case "missing_models": + return `Missing: ${availability.models.join(", ")}`; + } +}; + +// "loading"/"unverifiable" are transient system states, not a gap specific to this preset; only a +// caller-specific missing-model reason gets the alarming red treatment. +const isPresetHintAlarming = (availability: PresetAvailability): boolean => availability.kind === "missing_models"; + +type Props = { + selectedPreset?: string; + presets: AutoRouterPreset[]; + presetAvailability: (preset: AutoRouterPreset) => PresetAvailability; + onPresetChange: (key: string | undefined) => void; + pending: boolean; + unavailable: boolean; + onRetry: () => void; +}; + +export default function AutoRouterPresetSelect({ + selectedPreset, + presets, + presetAvailability, + onPresetChange, + pending, + unavailable, + onRetry, +}: Props) { + const sortedPresetOptions = React.useMemo( + () => + presets + .map((preset) => ({ preset, availability: presetAvailability(preset) })) + .sort((a, b) => Number(b.availability.kind === "available") - Number(a.availability.kind === "available")), + [presets, presetAvailability], + ); + + const templateItems = React.useMemo( + () => [ + ...sortedPresetOptions.map(({ preset }) => ({ value: preset.key, label: preset.label })), + { value: "custom", label: "Start from scratch" }, + ], + [sortedPresetOptions], + ); + + return ( +
+
+ +

Replaces classifier and model choices

+ {pending &&
Loading templates...
} + {unavailable && ( +
+ Could not load templates, so only Start from scratch is shown.{" "} + +
+ )} +
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterSimpleSetup.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterSimpleSetup.tsx new file mode 100644 index 00000000000..64c3d4de63f --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterSimpleSetup.tsx @@ -0,0 +1,190 @@ +import { Button } from "@/components/ui/button"; +import { Label } from "@/components/ui/label"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; +import { ROUTER_FAMILIES, ROUTER_GOALS, type FamilyPreference, type RouterGoal, type SolverModel } from "./auto_setup"; + +type Props = { + customized: boolean; + hasSuggestions: boolean; + onReset: () => void; + goal: RouterGoal; + family: FamilyPreference; + models: readonly SolverModel[]; + everyday?: SolverModel; + hard?: SolverModel; + waitingForTeam: boolean; + loading: boolean; + error: boolean; + notice: string | null; + onGoalChange: (goal: RouterGoal) => void; + onFamilyChange: (family: FamilyPreference) => void; + onModelChange: (role: "everyday" | "hard", model: string) => void; + onAdvanced: () => void; + onRetry: () => void; +}; + +const sharedFamily = (everyday?: SolverModel, hard?: SolverModel) => + everyday?.family && everyday.family === hard?.family ? ROUTER_FAMILIES[everyday.family] : null; + +const modelDescription = (model: SolverModel) => + [model.model !== model.name ? model.model : null, model.provider].filter(Boolean).join(" · "); + +export default function AutoRouterSimpleSetup({ + customized, + hasSuggestions, + onReset, + goal, + family, + models, + everyday, + hard, + waitingForTeam, + loading, + error, + notice, + onGoalChange, + onFamilyChange, + onModelChange, + onAdvanced, + onRetry, +}: Props) { + const inventoryReady = !waitingForTeam && !loading; + const showPreferences = !inventoryReady || error || hasSuggestions; + const sameFamily = sharedFamily(everyday, hard); + const familyItems = [ + { value: "auto", label: sameFamily ? `Auto (${sameFamily})` : "Auto" }, + ...Object.entries(ROUTER_FAMILIES).map(([value, label]) => ({ value, label })), + ]; + return ( +
+ {showPreferences && ( + <> +
+ +
+ {Object.entries(ROUTER_GOALS).map(([key, label]) => ( + + ))} +
+
+
+ + +
+ + )} + {waitingForTeam &&

Select a team to see available models.

} + {!waitingForTeam && loading && ( +

+ + Loading deployed models +

+ )} + {inventoryReady && error && ( +

+ Could not load deployed models.{" "} + +

+ )} + {inventoryReady && !error && ( + <> +
+
+ + {customized && hasSuggestions && ( + + )} +
+
+ {(["everyday", "hard"] as const).map((role) => { + const chosen = role === "everyday" ? everyday : hard; + const other = role === "everyday" ? hard : everyday; + const choices = models.filter((model) => model.name !== other?.name); + const items = choices.map((model) => ({ value: model.name, label: model.name })); + const label = role === "everyday" ? "Everyday" : "Hard tasks"; + return ( +
+
+ +

+ {role === "everyday" ? "Your cheaper model" : "Your stronger model"} +

+
+ + {chosen &&

{modelDescription(chosen)}

} +
+ ); + })} +
+
+ {(!everyday || !hard) && ( +

+ {models.length < 2 + ? "Two deployed models are needed. Add a model or use Advanced configuration." + : "Choose two models to get started."} +

+ )} + {notice && !customized && ( +

+ {notice} +

+ )} + + )} + +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx index ee9cbe3c480..52c4e9093a4 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx @@ -1,3 +1,4 @@ +import { autoRouterListKey } from "@/app/(dashboard)/hooks/models/useModels"; import { normalizeTierModels } from "./complexity_router_tiers"; import { openAutoRouterAdvanced, @@ -174,6 +175,12 @@ vi.mock("../common_components/team_dropdown", () => ({ ), })); +const renderAdvanced: typeof renderWithProviders = (...args) => { + const result = renderWithProviders(...args); + fireEvent.click(screen.getByRole("button", { name: "Advanced configuration", exact: true })); + return result; +}; + const Harness = () => ; describe("AddAutoRouterTab", () => { @@ -192,12 +199,185 @@ describe("AddAutoRouterTab", () => { vi.mocked(modelCostMap).mockResolvedValue({}); }); + it("creates the goal-selected latest OpenAI pair using its Bedrock deployment aliases", async () => { + mockFetchAvailableModels.mockResolvedValue( + ["small", "middle-old", "middle", "large"].map((model_group) => ({ model_group, mode: "chat" })), + ); + mockFetchAllModelDeployments.mockResolvedValue([ + { model_name: "small", litellm_params: { model: "bedrock/us.openai.gpt-6-luna" } }, + { model_name: "middle-old", litellm_params: { model: "openai/gpt-6-sol" } }, + { model_name: "middle", litellm_params: { model: "bedrock/us.openai.gpt-6.1-sol" } }, + { model_name: "large", litellm_params: { model: "bedrock/global.openai.gpt-6-astra" } }, + ]); + renderWithProviders(); + expect(await screen.findByRole("combobox", { name: "Everyday model" })).toHaveTextContent("small"); + expect(screen.queryByText("Define your tiers")).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Best quality", exact: true })); + expect(screen.getByRole("combobox", { name: "Everyday model" })).toHaveTextContent("middle"); + expect(screen.getAllByText("gpt-6.1-sol · Bedrock").some((element) => element.tagName === "P")).toBe(true); + await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled()); + fireEvent.click(screen.getByRole("button", { name: "Add Auto Router" })); + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls[0][0]).toMatchObject({ + auto_router_name: "auto-router", + complexity_router_config: { + classifier_type: "heuristic", + classification_mode: "user_turn", + tiers: { SIMPLE: ["middle"], MEDIUM: ["middle"], COMPLEX: ["large"], REASONING: ["large"] }, + }, + }); + }); + + it("creates a router from two custom models without a goal or family decision", async () => { + const customModels = ["fast-custom", "quality-custom"]; + mockFetchAvailableModels.mockResolvedValue(customModels.map((model_group) => ({ model_group, mode: "chat" }))); + mockFetchAllModelDeployments.mockResolvedValue( + customModels.map((model_name) => ({ + model_name, + litellm_params: { model: `openai/${model_name}` }, + })), + ); + renderWithProviders(); + expect(await screen.findByText("Choose two models to get started.")).toBeVisible(); + expect(screen.queryByRole("group", { name: "Routing goal" })).not.toBeInTheDocument(); + expect(screen.queryByRole("combobox", { name: "Model family" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled(); + await chooseSelectOption( + userEvent.setup(), + screen.getByRole("combobox", { name: "Everyday model" }), + /fast-custom/, + ); + await chooseSelectOption( + userEvent.setup(), + screen.getByRole("combobox", { name: "Hard tasks model" }), + /quality-custom/, + ); + expect(screen.queryByRole("button", { name: "Use suggested models" })).not.toBeInTheDocument(); + await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled()); + fireEvent.click(screen.getByRole("button", { name: "Add Auto Router" })); + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls[0][0].complexity_router_config).toMatchObject({ + classifier_type: "heuristic", + classification_mode: "user_turn", + tiers: { + SIMPLE: ["fast-custom"], + MEDIUM: ["fast-custom"], + COMPLEX: ["quality-custom"], + REASONING: ["quality-custom"], + }, + }); + }); + + it("saves concrete solver names expanded by the backend from an openai wildcard route", async () => { + mockFetchAvailableModels.mockResolvedValue( + ["openai/gpt-6-luna", "openai/gpt-6.1-sol", "openai/gpt-6-astra"].map((model_group) => ({ + model_group, + mode: "chat", + providers: ["openai"], + })), + ); + mockFetchAllModelDeployments.mockResolvedValue([{ model_name: "openai/*", litellm_params: { model: "openai/*" } }]); + renderWithProviders(); + await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled()); + fireEvent.click(screen.getByRole("button", { name: "Add Auto Router" })); + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls[0][0]).toMatchObject({ + complexity_router_config: { + classifier_type: "heuristic", + classification_mode: "user_turn", + tiers: { + SIMPLE: ["openai/gpt-6-luna"], + MEDIUM: ["openai/gpt-6-luna"], + COMPLEX: ["openai/gpt-6-astra"], + REASONING: ["openai/gpt-6-astra"], + }, + }, + }); + }); + + it.each(["groups", "deployments"])( + "keeps chosen models and saving available after a failed %s background refresh", + async (source) => { + mockFetchAvailableModels.mockResolvedValue( + ["claude-haiku-5-5", "claude-opus-5-5"].map((model_group) => ({ model_group, mode: "chat" })), + ); + renderWithProviders(); + await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled()); + const fetch = source === "groups" ? mockFetchAvailableModels : mockFetchAllModelDeployments; + fetch.mockRejectedValue(new Error("Background refresh unavailable")); + const queryKey = + source === "groups" ? ["availableModels", "autoRouter", "token"] : autoRouterListKey("", "Admin"); + await act(async () => { + await testQueryClient.refetchQueries({ queryKey }); + }); + await waitFor(() => expect(testQueryClient.getQueryState(queryKey)?.status).toBe("error")); + expect(screen.getByRole("combobox", { name: "Everyday model" })).toHaveTextContent("claude-haiku-5-5"); + expect(screen.queryByText("Could not load deployed models.")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled(); + fireEvent.click(screen.getByRole("button", { name: "Add Auto Router" })); + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce()); + }, + ); + + it("keeps manual solver choices on goal changes and carries them into Advanced", async () => { + mockFetchAvailableModels.mockResolvedValue( + ["claude-haiku-5-5", "custom-fast", "claude-opus-5-5"].map((model_group) => ({ + model_group, + mode: "chat", + })), + ); + renderWithProviders(); + const everyday = await screen.findByRole("combobox", { name: "Everyday model" }); + await userEvent.click(everyday); + await userEvent.click(await screen.findByRole("option", { name: /custom-fast/ })); + fireEvent.click(screen.getByRole("button", { name: "Cheapest", exact: true })); + expect(screen.getByRole("combobox", { name: "Everyday model" })).toHaveTextContent("custom-fast"); + expect(screen.getByRole("combobox", { name: "Hard tasks model" })).toHaveTextContent("claude-opus-5-5"); + fireEvent.click(screen.getByRole("button", { name: "Advanced configuration", exact: true })); + expectTierModel("Simple", "custom-fast"); + expectTierModel("Reasoning", "claude-opus-5-5"); + }); + + it("keeps standard setup usable when the Heuristic v2 allowance is exhausted", async () => { + mockFetchAvailableModels.mockResolvedValue( + ["claude-haiku-5-5", "claude-opus-5-5"].map((model_group) => ({ model_group, mode: "chat" })), + ); + vi.mocked(apiClient.post).mockResolvedValue({ + allowances: [{ key: "heuristic_v2", limit: 1, remaining: 0, available: true }], + error: null, + }); + renderWithProviders(); + await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled()); + expect(screen.queryByText(/allowance/i)).not.toBeInTheDocument(); + }); + + it("blocks creation when deployment identity cannot be verified, then recovers on retry", async () => { + mockFetchAvailableModels.mockResolvedValue( + ["claude-haiku-5-5", "claude-opus-5-5"].map((model_group) => ({ model_group, mode: "chat" })), + ); + mockFetchAllModelDeployments.mockRejectedValueOnce(new Error("Unavailable")); + renderWithProviders(); + expect(await screen.findByRole("alert")).toHaveTextContent("Could not load deployed models"); + expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled(); + fireEvent.click(screen.getByRole("button", { name: "Retry", exact: true })); + await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled()); + }); + + it("offers Advanced instead of inventing a second solver for a one-model deployment", async () => { + mockFetchAvailableModels.mockResolvedValue([{ model_group: "claude-opus-5-5", mode: "chat" }]); + renderWithProviders(); + expect(await screen.findByText(/Two deployed models are needed/)).toBeVisible(); + expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled(); + fireEvent.click(screen.getByRole("button", { name: "Advanced configuration", exact: true })); + expect(screen.getByText("Define your tiers")).toBeVisible(); + }); + it("shows Jev essentials before tiers and saves tuning from its own disclosure", async () => { mockFetchAvailableModels.mockResolvedValue([ ...ALL_FAMILY_MODELS, { model_group: "my-decision", mode: "evaluation" }, ]); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); await userEvent.click(screen.getByRole("radio", { name: "Decisions Model" })); expect(screen.queryByLabelText("Classifier Model")).not.toBeInTheDocument(); @@ -250,7 +430,7 @@ describe("AddAutoRouterTab", () => { { model_name: "ordinary-chat", litellm_params: { model: "mixed-provider/chat-model" } }, { model_name: "unavailable-decider", litellm_params: { model: underlyingModel } }, ]); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); await userEvent.click(screen.getByRole("radio", { name: "Decisions Model" })); await userEvent.click(screen.getByRole("combobox", { name: "Decision model" })); @@ -269,7 +449,7 @@ describe("AddAutoRouterTab", () => { it("uses the judge picker empty state when no decision deployments exist", async () => { mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); await userEvent.click(screen.getByRole("radio", { name: "Decisions Model" })); await userEvent.click(screen.getByRole("combobox", { name: "Decision model" })); @@ -280,7 +460,7 @@ describe("AddAutoRouterTab", () => { it("does not refill a tier cleared by the user when models refresh", async () => { mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); await userEvent.click(within(tierChips("Simple")).getByRole("button", { name: "" })); await act(async () => { @@ -294,7 +474,7 @@ describe("AddAutoRouterTab", () => { allowances: [{ key: "heuristic_v2", limit: 1, remaining, available: true }], error: null, }); - renderWithProviders(); + renderAdvanced(); await waitFor(() => expect(screen.getByRole("button", { name: "Heuristic" })).toHaveTextContent("Rule-based")); }); @@ -305,21 +485,21 @@ describe("AddAutoRouterTab", () => { ? "This change needs an available rule-based tuning allowance" : null, })); - renderWithProviders(); + renderAdvanced(); await waitFor(() => expect(screen.getByRole("button", { name: "Heuristic" })).toHaveTextContent("Rule-based")); expect(screen.queryByRole("alert")).not.toBeInTheDocument(); expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled(); }); it("keeps the Rule-based default when the form reopens with a cached allowance", async () => { - const first = renderWithProviders(); + const first = renderAdvanced(); await waitFor(() => expect(apiClient.post).toHaveBeenCalled()); first.unmount(); vi.mocked(apiClient.post).mockResolvedValueOnce({ allowances: [{ key: "heuristic_v2", limit: 1, remaining: 0, available: true }], error: null, }); - renderWithProviders(); + renderAdvanced(); await waitFor(() => expect(screen.getByRole("button", { name: "Heuristic" })).toHaveTextContent("Rule-based")); }); @@ -330,7 +510,7 @@ describe("AddAutoRouterTab", () => { allowances: [{ key: "heuristic_v2", limit: 1, remaining: 0, available: true }], error: null, }); - renderWithProviders(); + renderAdvanced(); fireEvent.click(screen.getByRole("button", { name: "Heuristic" })); const option = screen.getByRole("menuitemradio", { name: /^Heuristic v2/ }); await waitFor(() => expect(option).toHaveTextContent("0 of 1 available")); @@ -383,7 +563,7 @@ describe("AddAutoRouterTab", () => { ...ALL_FAMILY_MODELS, { model_group: "my-decision", mode: "evaluation" }, ]); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); await user.click(screen.getByRole("radio", { name: "Decisions Model" })); await userEvent.click(screen.getByRole("combobox", { name: "Decision model" })); @@ -431,7 +611,7 @@ describe("AddAutoRouterTab", () => { ...ALL_FAMILY_MODELS, { model_group: "my-decision", mode: "evaluation" }, ]); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); await user.click(screen.getByRole("radio", { name: "Decisions Model" })); await userEvent.click(screen.getByRole("combobox", { name: "Decision model" })); @@ -468,7 +648,7 @@ describe("AddAutoRouterTab", () => { complete = resolve; }), ); - renderWithProviders(); + renderAdvanced(); await userEvent.click(screen.getByRole("radio", { name: "LLM" })); complete?.({ allowances: [{ key: "heuristic_v2", limit: 1, remaining: 0, available: true }], error: null }); await waitFor(() => expect(screen.getByRole("radio", { name: "LLM" })).toBeChecked()); @@ -484,7 +664,7 @@ describe("AddAutoRouterTab", () => { complete = resolve; }), ); - renderWithProviders(); + renderAdvanced(); await userEvent.click(screen.getByRole("radio", { name: family })); await selectAutoRouterOption("How often to classify", "Every new user message"); await act(async () => { @@ -508,8 +688,8 @@ describe("AddAutoRouterTab", () => { { model_group: "capable", mode: "chat" }, { model_group: "judge", mode: "chat" }, ]); - renderWithProviders(); - await user.type(screen.getByLabelText("Auto Router Name"), "forecast-router"); + renderAdvanced(); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "forecast-router" } }); await selectAutoRouterApproach(label); expect(screen.getByLabelText("Auto Router Name")).toHaveValue("forecast-router"); expect(screen.queryByTestId("template-selector")).not.toBeInTheDocument(); @@ -575,7 +755,7 @@ describe("AddAutoRouterTab", () => { it("restores the automatic/template/detail flow on the Complexity tab", async () => { const user = userEvent.setup(); mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); await selectAutoRouterApproach("Capability"); expect(screen.queryByTestId("configure-automatically-button")).not.toBeInTheDocument(); @@ -603,9 +783,9 @@ describe("AddAutoRouterTab", () => { { model_group: "capable", mode: "chat" }, { model_group: "judge", mode: "chat" }, ]); - renderWithProviders(); + renderAdvanced(); await selectAutoRouterApproach(label); - await user.type(screen.getByLabelText("Auto Router Name"), "forecast-retry"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "forecast-retry" } }); const capability = label === "Capability"; const policyField = capability ? "Solve probability threshold" : "Efficient solver profile"; fireEvent.change(screen.getByLabelText(policyField), { @@ -627,7 +807,7 @@ describe("AddAutoRouterTab", () => { // Detailed Configuration starts collapsed so the modal opens onto just Name + Template; a caller // opts into the full tier/classifier form rather than always seeing it up front. it("shows models immediately and keeps advanced settings collapsed", async () => { - renderWithProviders(); + renderAdvanced(); expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false"); @@ -641,14 +821,14 @@ describe("AddAutoRouterTab", () => { { model_group: "unknown-model-a", mode: "chat" }, { model_group: "unknown-model-b", mode: "chat" }, ]); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await waitFor(() => expect(optionByLabel("Anthropic Family")).toHaveTextContent("Missing:")); expect(screen.queryByTestId("configure-automatically-button")).not.toBeInTheDocument(); }); - it("mixes preferred tier models even when one complete preset is available", async () => { + it("prefers a same-family pair when a complete family is available", async () => { const anthropicPreset = getPresetByKey("anthropic_family")!; mockFetchAvailableModels.mockResolvedValue( [...getRequiredModelsInPreset(anthropicPreset), "gpt-5.6-luna"].map((model_group) => ({ @@ -657,18 +837,18 @@ describe("AddAutoRouterTab", () => { })), ); mockFetchAllModelDeployments.mockResolvedValue([]); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); - expectTierModel("Simple", "gpt-5.6-luna"); - expectTierModel("Medium", "claude-sonnet-5"); + expectTierModel("Simple", ANTHROPIC_TIERS.SIMPLE[0]); + expectTierModel("Medium", ANTHROPIC_TIERS.SIMPLE[0]); expectTierModel("Complex", "claude-opus-5-5"); expectTierModel("Reasoning", "claude-opus-5-5"); expect(toast.success).not.toHaveBeenCalledWith(expect.stringContaining("Configured with")); }); - it("mixes available models from the preferred tier catalog when no complete template fits", async () => { + it("uses two models within the available family without a complete template", async () => { mockFetchAvailableModels.mockResolvedValue( ["gpt-5.6-luna", "claude-sonnet-5", "gpt-5.6-sol"].map((model_group) => ({ model_group, @@ -676,12 +856,12 @@ describe("AddAutoRouterTab", () => { })), ); mockFetchAllModelDeployments.mockResolvedValue([]); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); expectTierModel("Simple", "gpt-5.6-luna"); - expectTierModel("Medium", "claude-sonnet-5"); + expectTierModel("Medium", "gpt-5.6-luna"); expectTierModel("Complex", "gpt-5.6-sol"); expectTierModel("Reasoning", "gpt-5.6-sol"); }); @@ -689,20 +869,20 @@ describe("AddAutoRouterTab", () => { it("opens Detailed Configuration on the tiers automatic setup just filled in", async () => { const simpleModel = "gpt-5.6-luna"; mockFetchAvailableModels.mockResolvedValue([...ALL_FAMILY_MODELS, { model_group: simpleModel, mode: "chat" }]); - renderWithProviders(); + renderAdvanced(); expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false"); await waitForAutomaticTiers(); expect(screen.getByText("Define your tiers")).toBeInTheDocument(); - expectTierModel("Simple", simpleModel); + expectTierModel("Simple", ANTHROPIC_TIERS.SIMPLE[0]); }); // Nothing is filled in, so there is nothing to submit. The button reports that itself instead of // accepting a click and answering with a toast. it("offers no submit at all until every tier has a model", async () => { - renderWithProviders(); + renderAdvanced(); expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled(); }); @@ -710,7 +890,8 @@ describe("AddAutoRouterTab", () => { it("still flags the router name once the config no longer blocks the submit", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -720,15 +901,27 @@ describe("AddAutoRouterTab", () => { }); it("offers no team selector to a proxy admin, who may create an unscoped router", async () => { - renderWithProviders(); + renderAdvanced(); expect(screen.queryByTestId("team-dropdown")).not.toBeInTheDocument(); }); - it("requires a team admin to pick a team", async () => { + it("waits for a team before showing model inventory in simple setup", async () => { renderWithProviders( , ); + expect(screen.getByText("Select a team to see available models.")).toBeInTheDocument(); + expect(screen.queryByText(/Two deployed models are needed/)).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled(); + await userEvent.selectOptions(screen.getByTestId("team-dropdown"), "team-1"); + expect(await screen.findByText("Chosen models")).toBeInTheDocument(); + expect(screen.queryByText("Select a team to see available models.")).not.toBeInTheDocument(); + }); + + it("requires a team admin to pick a team", async () => { + renderAdvanced( + , + ); expect(screen.getByTestId("team-dropdown")).toBeInTheDocument(); expect(screen.getByText("Select Team")).toBeInTheDocument(); @@ -741,11 +934,11 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders( + renderAdvanced( , ); - await user.type(screen.getByPlaceholderText(/smart_router/i), "team-scoped-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "team-scoped-router" } }); await user.selectOptions(screen.getByTestId("team-dropdown"), "team-1"); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -763,7 +956,7 @@ describe("AddAutoRouterTab", () => { ); vi.mocked(handleAddAutoRouterSubmit).mockImplementationOnce(actualSubmit.handleAddAutoRouterSubmit); const user = userEvent.setup(); - renderWithProviders( + renderAdvanced( { error: "session_affinity cannot be combined with tier_definitions", }); - renderWithProviders(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "rejected-router"); + renderAdvanced(); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "rejected-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -838,7 +1031,7 @@ describe("AddAutoRouterTab", () => { }), ); - const { container } = renderWithProviders(); + const { container } = renderAdvanced(); fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "double-submit-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); @@ -858,8 +1051,8 @@ describe("AddAutoRouterTab", () => { vi.mocked(getMissingTiersError).mockReturnValue(null); validateAutoRouterConfig.mockResolvedValueOnce({ valid: true }); - renderWithProviders(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "accepted-router"); + renderAdvanced(); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "accepted-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -873,9 +1066,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "keyword-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); @@ -890,9 +1083,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "keyword-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); @@ -908,9 +1101,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "orphan-rule-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "orphan-rule-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); @@ -927,9 +1120,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "keyword-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); @@ -944,9 +1137,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "keyword-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); @@ -965,11 +1158,11 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders( + renderAdvanced( , ); - await user.type(screen.getByPlaceholderText(/smart_router/i), "team-scoped-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "team-scoped-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -983,11 +1176,11 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders( + renderAdvanced( , ); - await user.type(screen.getByPlaceholderText(/smart_router/i), "team-scoped-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "team-scoped-router" } }); await user.selectOptions(screen.getByTestId("team-dropdown"), "team-1"); await user.click(screen.getByTestId("team-dropdown-clear")); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); @@ -1001,9 +1194,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "affinity-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Classification Method"); expect(await screen.findByRole("combobox", { name: "How often to classify" })).not.toHaveTextContent( @@ -1022,7 +1215,8 @@ describe("AddAutoRouterTab", () => { it("blocks invalid success thresholds and creates a heuristic v2 router with explicit zero", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "" } }); fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "threshold-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Classification Method"); @@ -1059,7 +1253,7 @@ describe("AddAutoRouterTab", () => { "./handle_add_auto_router_submit", ); vi.mocked(handleAddAutoRouterSubmit).mockImplementationOnce(actualSubmit.handleAddAutoRouterSubmit); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "chained-router" } }); await userEvent.click(screen.getByRole("radio", { name: "LLM" })); @@ -1087,7 +1281,7 @@ describe("AddAutoRouterTab", () => { it("preserves classifier tuning when choosing models automatically", async () => { const user = userEvent.setup(); mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "reset-threshold-router" } }); openAutoRouterAdvanced("Classification Method"); @@ -1112,7 +1306,8 @@ describe("AddAutoRouterTab", () => { it("clears an invalid inactive threshold before creating the router", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "" } }); fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "clear-threshold-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Classification Method"); @@ -1133,13 +1328,15 @@ describe("AddAutoRouterTab", () => { it.each([false, true])("creates a router with cache routing only after an explicit opt-in: %s", async (enabled) => { const user = userEvent.setup(); mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "cache-aware-router" } }); openAutoRouterAdvanced("Cache-aware routing"); const toggle = screen.getByRole("switch", { name: "Cache-aware routing" }); expect(toggle).not.toBeChecked(); if (enabled) { + expect(toggle).toHaveAttribute("aria-disabled", "true"); + await selectAutoRouterOption("How often to classify", "Every request"); await user.click(toggle); fireEvent.change(screen.getByLabelText("Expected output tokens"), { target: { value: "512" } }); fireEvent.change(screen.getByLabelText("Prediction timeout (ms)"), { target: { value: "750" } }); @@ -1158,9 +1355,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-window-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "ctx-window-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Context Window Escalation"); const toggle = await screen.findByRole("switch", { name: "Escalate oversized prompts to a tier that fits" }); @@ -1181,9 +1378,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-buffer-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "ctx-buffer-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Context Window Escalation"); await user.click(screen.getByRole("switch", { name: "Escalate oversized prompts to a tier that fits" })); @@ -1204,9 +1401,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-clear-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "ctx-clear-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Context Window Escalation"); await user.click(screen.getByRole("switch", { name: "Escalate oversized prompts to a tier that fits" })); @@ -1230,9 +1427,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "no-compression-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "no-compression-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -1246,9 +1443,11 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "no-compression-explicit-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { + target: { value: "no-compression-explicit-router" }, + }); expandDetailedConfiguration(); openAutoRouterAdvanced("Compression"); await chooseSelectOption( @@ -1270,9 +1469,11 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "different-compression-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { + target: { value: "different-compression-router" }, + }); expandDetailedConfiguration(); openAutoRouterAdvanced("Compression"); await chooseSelectOption( @@ -1299,9 +1500,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "override-floor-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "override-floor-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Classification Method"); await selectAutoRouterOption("Heuristic", "Rule-based"); @@ -1321,9 +1522,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "affinity-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Classification Method"); await selectAutoRouterOption("How often to classify", "Once per session"); @@ -1346,9 +1547,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "user-turn-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "user-turn-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Classification Method"); await selectAutoRouterOption("How often to classify", "Every new user message"); @@ -1362,16 +1563,19 @@ describe("AddAutoRouterTab", () => { }); }); - it("writes every_request into the create payload when the default frequency stays selected", async () => { + it("allows every request in Advanced while defaulting to every new user message", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "default-timing-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "default-timing-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Classification Method"); - expect(await screen.findByRole("combobox", { name: "How often to classify" })).toHaveTextContent("Every request"); + expect(await screen.findByRole("combobox", { name: "How often to classify" })).toHaveTextContent( + "Every new user message", + ); + await selectAutoRouterOption("How often to classify", "Every request"); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -1386,9 +1590,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "affinity-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Affinity"); expect(await screen.findByRole("switch", { name: "Pin one model deployment per tier" })).toBeChecked(); @@ -1406,9 +1610,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "affinity-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Affinity"); await user.click(await screen.findByRole("switch", { name: "Pin one model deployment per tier" })); @@ -1426,7 +1630,7 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "modality-router" } }); @@ -1444,7 +1648,7 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "modality-router" } }); expandDetailedConfiguration(); @@ -1465,7 +1669,7 @@ describe("AddAutoRouterTab", () => { // Custom is the escape hatch, not the headline choice, so it's listed after every bundled preset // rather than first. it("lists Start from scratch after the bundled presets", async () => { - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); const labels = visibleOptions().map((option) => option.querySelector(".font-medium")?.textContent); @@ -1487,7 +1691,7 @@ describe("AddAutoRouterTab", () => { ); vi.mocked(getMissingTiersError).mockImplementation(actual.getMissingTiersError); - renderWithProviders(); + renderAdvanced(); expect(screen.getByTestId("auto-router-test-routing-btn")).toBeDisabled(); }); @@ -1504,8 +1708,8 @@ describe("AddAutoRouterTab", () => { }, }); - renderWithProviders(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + renderAdvanced(); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "keyword-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); @@ -1540,7 +1744,7 @@ describe("AddAutoRouterTab", () => { }, }); - renderWithProviders(); + renderAdvanced(); await user.click(screen.getByTestId("auto-router-test-routing-btn")); await user.type(await screen.findByTestId("auto-router-routing-test-prompt"), "reconcile this invoice"); await user.click(screen.getByTestId("auto-router-routing-test-send")); @@ -1564,7 +1768,7 @@ describe("AddAutoRouterTab", () => { }), ); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); const anthropicOption = optionByLabel("Anthropic Family")!; @@ -1582,7 +1786,7 @@ describe("AddAutoRouterTab", () => { it("disables every preset and offers a retry when the model list fails to load", async () => { mockFetchAvailableModels.mockRejectedValue(new Error("network error")); - renderWithProviders(); + renderAdvanced(); expect(await screen.findByText("Could not load available models.")).toBeInTheDocument(); openTemplateDropdown(); @@ -1595,7 +1799,7 @@ describe("AddAutoRouterTab", () => { mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); mockFetchAllModelDeployments.mockRejectedValue(new Error("network error")); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await waitForPresetEnabled("OpenAI Family"); @@ -1606,7 +1810,7 @@ describe("AddAutoRouterTab", () => { ALL_FAMILY_MODELS.filter((m) => m.model_group !== ANTHROPIC_ONLY_MODEL), ); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await waitFor(() => { @@ -1618,7 +1822,7 @@ describe("AddAutoRouterTab", () => { it("enables a preset once every model it needs is available", async () => { mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await waitForPresetEnabled("OpenAI Family"); @@ -1626,7 +1830,7 @@ describe("AddAutoRouterTab", () => { it("shows the template's model choices while advanced settings stay collapsed", async () => { mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await selectTemplate("Anthropic Family"); expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false"); @@ -1635,7 +1839,7 @@ describe("AddAutoRouterTab", () => { }); it("expands detailed configuration when Start from scratch is chosen", async () => { - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await selectTemplate("Start from scratch"); @@ -1647,7 +1851,7 @@ describe("AddAutoRouterTab", () => { it("lets a caller manually re-expand a detailed configuration a preset just collapsed", async () => { mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await selectTemplate("Anthropic Family"); expect(screen.queryByText("Keyword/Semantic Matching")).not.toBeInTheDocument(); @@ -1666,11 +1870,11 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await selectTemplate("Anthropic Family"); - await user.type(screen.getByPlaceholderText(/smart_router/i), "anthropic-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "anthropic-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -1689,7 +1893,7 @@ describe("AddAutoRouterTab", () => { it("blocks a form submit when a referenced model disappears after the tiers are filled in", async () => { mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - const { container } = renderWithProviders(); + const { container } = renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await selectTemplate("Anthropic Family"); fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "stale-model-router" } }); @@ -1713,11 +1917,11 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await selectTemplate("Anthropic Family"); - await user.type(screen.getByPlaceholderText(/smart_router/i), "anthropic-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "anthropic-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -1751,10 +1955,10 @@ describe("AddAutoRouterTab", () => { it("submits the pinned model in place of the one the tiers derive", async () => { const user = userEvent.setup(); - renderWithProviders(); + renderAdvanced(); await applyPresetAndPin(user); - await user.type(screen.getByPlaceholderText(/smart_router/i), "pinned-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "pinned-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -1771,7 +1975,7 @@ describe("AddAutoRouterTab", () => { it("blocks a submit whose pinned model is no longer available", async () => { const user = userEvent.setup(); - const { container } = renderWithProviders(); + const { container } = renderAdvanced(); await applyPresetAndPin(user); fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "stale-pin-router" } }); @@ -1796,11 +2000,11 @@ describe("AddAutoRouterTab", () => { it("omits plan_mode_min_tier from the payload when never touched", async () => { const user = userEvent.setup(); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await selectTemplate("Anthropic Family"); - await user.type(screen.getByPlaceholderText(/smart_router/i), "no-plan-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "no-plan-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -1812,7 +2016,7 @@ describe("AddAutoRouterTab", () => { it("carries the enabled override through to the create payload", async () => { const user = userEvent.setup(); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await selectTemplate("Anthropic Family"); @@ -1820,7 +2024,7 @@ describe("AddAutoRouterTab", () => { openAutoRouterAdvanced("Plan-Mode Override"); await user.click(await screen.findByRole("switch", { name: "Route plan-mode requests to a minimum tier" })); - await user.type(screen.getByPlaceholderText(/smart_router/i), "plan-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "plan-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -1847,7 +2051,7 @@ describe("AddAutoRouterTab", () => { mockFetchAvailableModels.mockResolvedValue(groupsFor(ALL_RENAMED_DEPLOYMENTS)); mockFetchAllModelDeployments.mockResolvedValue(ALL_RENAMED_DEPLOYMENTS); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await waitFor(() => { @@ -1874,7 +2078,7 @@ describe("AddAutoRouterTab", () => { mockFetchAvailableModels.mockResolvedValue(groupsFor(deployments)); mockFetchAllModelDeployments.mockResolvedValue(deployments); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await waitFor(() => { expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); @@ -1886,7 +2090,7 @@ describe("AddAutoRouterTab", () => { expect(screen.getByText("Keyword/Semantic Matching")).toBeInTheDocument(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "renamed-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "renamed-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -1908,7 +2112,7 @@ describe("AddAutoRouterTab", () => { mockFetchAvailableModels.mockResolvedValue(groupsFor(anthropicOnly)); mockFetchAllModelDeployments.mockResolvedValue(anthropicOnly); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await waitFor(() => { @@ -1933,7 +2137,7 @@ describe("AddAutoRouterTab", () => { mockFetchAvailableModels.mockResolvedValue(groupsFor(wildcard)); mockFetchAllModelDeployments.mockResolvedValue(wildcard); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await waitFor(() => { @@ -1960,7 +2164,7 @@ describe("AddAutoRouterTab", () => { mockFetchAvailableModels.mockResolvedValue(EXPANDED_HUB_GROUPS); mockFetchAllModelDeployments.mockResolvedValue(WILDCARD_DEPLOYMENTS); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await waitFor(() => { @@ -1974,14 +2178,14 @@ describe("AddAutoRouterTab", () => { mockFetchAvailableModels.mockResolvedValue(EXPANDED_HUB_GROUPS); mockFetchAllModelDeployments.mockResolvedValue(WILDCARD_DEPLOYMENTS); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await waitFor(() => { expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); }); await selectTemplate("Anthropic Family"); - await user.type(screen.getByPlaceholderText(/smart_router/i), "wildcard-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "wildcard-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -2079,7 +2283,7 @@ describe("preset catalog fetch states", () => { }, ], }); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Bounded JEV"); await selectTemplate("Bounded JEV"); await userEvent.click(screen.getByRole("combobox", { name: "Decision model" })); @@ -2100,7 +2304,7 @@ describe("preset catalog fetch states", () => { ...LOADED_PRESETS_QUERY, isError: true, } as never); - renderWithProviders(); + renderAdvanced(); expect(screen.queryByText(/Could not load templates/)).not.toBeInTheDocument(); @@ -2114,7 +2318,7 @@ describe("preset catalog fetch states", () => { data: undefined, isPending: true, } as never); - renderWithProviders(); + renderAdvanced(); expect(screen.getByText("Loading templates...")).toBeInTheDocument(); }); @@ -2127,7 +2331,7 @@ describe("preset catalog fetch states", () => { isError: true, refetch, } as never); - renderWithProviders(); + renderAdvanced(); expect(await screen.findByText(/Could not load templates/)).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 82c30395755..8c3f402df1f 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -13,7 +13,7 @@ import { FormField } from "@/components/shared/form/FormField"; import { Button } from "@/components/ui/button"; import { Card, CardContent } from "@/components/ui/card"; import { Input } from "@/components/ui/input"; -import { Select, SelectContent, SelectItem, SelectSeparator, SelectTrigger, SelectValue } from "@/components/ui/select"; +import AutoRouterPresetSelect, { type PresetAvailability } from "./AutoRouterPresetSelect"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; import { useZodForm } from "@/lib/forms/useZodForm"; @@ -79,7 +79,16 @@ import { } from "@/lib/autorouter_presets"; import { useAutoRouterPresets } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets"; import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; -import { buildAutomaticRouterConfig, buildPreferredTierModels, prefillEmptyTiers } from "./auto_setup"; +import AutoRouterSimpleSetup from "./AutoRouterSimpleSetup"; +import { + deployedRouterModels, + nextRouterName, + simpleSetupError, + recommendSolverPair, + solverPairConfig, + type RouterGoal, + type FamilyPreference, +} from "./auto_setup"; interface AddAutoRouterTabProps { handleOk: () => void; @@ -95,31 +104,6 @@ interface AddAutoRouterTabProps { teams?: Team[] | null; } -type PresetAvailability = - | { kind: "available"; viaDeployments: boolean } - | { kind: "loading" } - | { kind: "unverifiable" } - | { kind: "missing_models"; models: readonly string[] }; - -// Every non-"available" state disables the option. Selection derives from this same function -// (see presetAvailability below), so an option a caller can click is always one that can be applied. -const presetDisabledHint = (availability: PresetAvailability): string | null => { - switch (availability.kind) { - case "available": - return null; - case "loading": - return "Checking model availability..."; - case "unverifiable": - return "Cannot verify these models are available"; - case "missing_models": - return `Missing: ${availability.models.join(", ")}`; - } -}; - -// "loading"/"unverifiable" are transient system states, not a gap specific to this preset; only a -// caller-specific missing-model reason gets the alarming red treatment. -const isPresetHintAlarming = (availability: PresetAvailability): boolean => availability.kind === "missing_models"; - const NO_PRESETS: AutoRouterPreset[] = []; // Why the submit is unavailable, or null when it is available. The button reads this to disable @@ -168,7 +152,7 @@ const autoRouterSchema = (requiresTeamScope: boolean) => type AddAutoRouterFormValues = z.infer>; const EMPTY_FORM_VALUES: AddAutoRouterFormValues = { - auto_router_name: "", + auto_router_name: "auto-router", team_id: null, model_access_group: undefined, }; @@ -210,11 +194,18 @@ const AddAutoRouterTab: React.FC = ({ }); const [modelAccessGroups, setModelAccessGroups] = useState([]); - const [complexityRouterConfig, setComplexityRouterConfig] = useState({ + const [advancedConfig, setComplexityRouterConfig] = useState({ tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic", + classification_mode: "user_turn", }); + const [advanced, setAdvanced] = useState(false); + const [prefillAttempted, setPrefillAttempted] = useState(false); + const [goal, setGoal] = useState("balanced"); + const [family, setFamily] = useState("auto"); + const [solverOverrides, setSolverOverrides] = useState<{ everyday?: string; hard?: string }>({}); + const [customTechnicalKeywords, setCustomTechnicalKeywords] = useState([]); const [keywordTierRules, setKeywordTierRules] = useState([]); const [semanticMatchingEnabled, setSemanticMatchingEnabled] = useState(false); @@ -250,17 +241,30 @@ const AddAutoRouterTab: React.FC = ({ isError: modelsError, refetch: refetchModels, } = useQuery({ - queryKey: ["availableModels", "autoRouter", accessToken, ...(isMemberManaged ? [watchedTeamId] : [])], + queryKey: ["availableModels", "autoRouter", accessToken, ...(requiresTeamScope ? [watchedTeamId] : [])], queryFn: () => - isMemberManaged ? fetchAutoRouterModels(accessToken, watchedTeamId) : fetchAvailableModels(accessToken), - enabled: Boolean(accessToken && (!isMemberManaged || watchedTeamId)), + requiresTeamScope ? fetchAutoRouterModels(accessToken, watchedTeamId) : fetchAvailableModels(accessToken), + enabled: Boolean(accessToken && (!requiresTeamScope || watchedTeamId)), }); - const { data: deployments, isLoading: deploymentsLoading } = useQuery({ + const { + data: deployments, + isLoading: deploymentsLoading, + isError: deploymentsError, + refetch: refetchDeployments, + } = useQuery({ queryKey: autoRouterListKey(userId ?? "", userRole), queryFn: () => fetchAllModelDeployments(accessToken, userId ?? "", userRole), enabled: Boolean(accessToken), }); const modelsLoading = groupsLoading || deploymentsLoading; + useEffect(() => { + if (!deployments || form.getFieldState("auto_router_name").isDirty) return; + form.setValue( + "auto_router_name", + nextRouterName(deployments.flatMap((row) => (row.model_name ? [row.model_name] : []))), + ); + }, [deployments, form]); + const modelInfo = React.useMemo(() => data ?? [], [data]); const { data: presetsData, @@ -269,12 +273,12 @@ const AddAutoRouterTab: React.FC = ({ refetch: refetchPresets, } = useAutoRouterPresets(); const presets = presetsData ?? NO_PRESETS; - const automaticSetupLoading = modelsLoading || presetsPending; const presetsUnavailable = presetsError && presetsData === undefined; // react-query keeps the last successful list around when a later refetch fails, so isError alone // can't tell "never loaded" apart from "loaded, then a background refetch errored" - only the // former leaves us with nothing trustworthy to verify a preset's models against. const modelsUnverifiable = modelsError && data === undefined; + const inventoryUnavailable = modelsUnverifiable || (deploymentsError && deployments === undefined); const isAdmin = all_admin_roles.includes(userRole); @@ -294,14 +298,33 @@ const AddAutoRouterTab: React.FC = ({ ), [modelInfo], ); - const preferredTierModels = React.useMemo( - () => buildPreferredTierModels(presets, availability), - [presets, availability], + const solverModels = React.useMemo( + () => deployedRouterModels(modelInfo, deployments ?? []), + [modelInfo, deployments], ); - const automaticRouterConfig = React.useMemo( - () => buildAutomaticRouterConfig(modelInfo, deployments ?? [], preferredTierModels), - [modelInfo, deployments, preferredTierModels], + const recommended = React.useMemo( + () => recommendSolverPair(solverModels, goal, family), + [solverModels, goal, family], ); + const everyday = solverOverrides.everyday + ? solverModels.find((model) => model.name === solverOverrides.everyday) + : recommended?.everyday; + const hard = solverOverrides.hard + ? solverModels.find((model) => model.name === solverOverrides.hard) + : recommended?.hard; + const simpleConfig = everyday && hard ? solverPairConfig(everyday.name, hard.name) : null; + const complexityRouterConfig = advanced + ? advancedConfig + : simpleConfig ?? { + classifier_type: "heuristic" as const, + classification_mode: "user_turn" as const, + tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }, + }; + const openAdvanced = () => { + setComplexityRouterConfig(complexityRouterConfig); + setPrefillAttempted(simpleConfig !== null); + setAdvanced(true); + }; // A preset's models can only be trusted against a successfully loaded list. Selection and the // greyed-out state derive from this one function, so a preset that cannot be selected can never @@ -322,22 +345,6 @@ const AddAutoRouterTab: React.FC = ({ [modelsLoading, modelsUnverifiable, availability, groupsOnlyAvailability], ); - const sortedPresetOptions = React.useMemo( - () => - presets - .map((preset) => ({ preset, availability: presetAvailability(preset) })) - .sort((a, b) => Number(b.availability.kind === "available") - Number(a.availability.kind === "available")), - [presets, presetAvailability], - ); - - const templateItems = React.useMemo( - () => [ - ...sortedPresetOptions.map(({ preset }) => ({ value: preset.key, label: preset.label })), - { value: "custom", label: "Start from scratch" }, - ], - [sortedPresetOptions], - ); - const applyPrefill = (prefill: PresetPrefill) => { setEditingTiers(false); setComplexityRouterConfig(prefill.complexityRouterConfig); @@ -349,16 +356,20 @@ const AddAutoRouterTab: React.FC = ({ setEscalationKeywords(prefill.escalationKeywords); }; - const [prefillAttempted, setPrefillAttempted] = useState(false); - const automaticPrefill = automaticSetupLoading || modelsUnverifiable ? null : automaticRouterConfig; - if (automaticPrefill && !prefillAttempted) { + if (advanced && !prefillAttempted && !modelsLoading && simpleConfig) { setPrefillAttempted(true); - setComplexityRouterConfig((current) => prefillEmptyTiers(current, automaticPrefill)); + setComplexityRouterConfig((current) => + !current.custom_tier_set && + !isForecastClassifier(current.classifier_type) && + activeTierRows(current).every((row) => row.models.length === 0) + ? { ...current, tiers: simpleConfig.tiers } + : current, + ); } const handlePresetChange = (presetKey: string | undefined) => { + setPrefillAttempted(true); if (!presetKey || presetKey === "custom") { - setPrefillAttempted(true); setSelectedPreset(presetKey); applyPrefill(buildEmptyPrefill()); @@ -372,7 +383,6 @@ const AddAutoRouterTab: React.FC = ({ const presetState = presetAvailability(preset); if (presetState.kind !== "available") return; - setPrefillAttempted(true); setSelectedPreset(presetKey); applyPrefill(buildPresetPrefill(preset.complexity_router_config, availability)); }; @@ -413,7 +423,11 @@ const AddAutoRouterTab: React.FC = ({ }, !requiresTeamScope || Boolean(watchedTeamId), ); - const saveBlockedReason = submitBlockedReason ?? routerAvailability.saveBlockedReason; + const simpleBlockedReason = advanced ? null : simpleSetupError(modelsLoading, inventoryUnavailable, everyday, hard); + const saveBlockedReason = + !advanced && requiresTeamScope && !watchedTeamId + ? "Select a team to see available models." + : simpleBlockedReason ?? submitBlockedReason ?? routerAvailability.saveBlockedReason; const handleClassifierChange = (config: ComplexityRouterConfigValue) => { setSelectedPreset(undefined); setComplexityRouterConfig(config); @@ -540,71 +554,20 @@ const AddAutoRouterTab: React.FC = ({ setIsTestModalVisible(true); }; - const tierSetup = ( -
-
- -

Replaces classifier and model choices

- {presetsPending &&
Loading templates...
} - {presetsUnavailable && ( -
- Could not load templates, so only Start from scratch is shown.{" "} - -
- )} -
-
- ); - const configurationForm = ( void refetchPresets()} + /> + } editingTiers={editingTiers} onEditingTiersChange={setEditingTiers} modelInfo={modelInfo} @@ -646,8 +609,8 @@ const AddAutoRouterTab: React.FC = ({ )}
-
- {isAdmin && ( +
+ {isAdmin && advanced && (
= ({ )} - - } - judgeModel={ - model.mode !== "embedding") - .map((model) => ({ value: model.model_group, label: model.model_group }))} - showValidationErrors={showValidationErrors} - /> - } - > - - {modelsUnverifiable && ( -

- Could not load available models.{" "} - -

- )} - {configurationForm} - -
- - - Need Help? - - } - /> - Get help on our github - -
- - - - +

+ )} + {configurationForm} + + + )} + {!advanced && routerAvailability.isError && ( +

+ Could not check availability.{" "} + +

+ )} + {!advanced && routerAvailability.data?.error && ( +

+ {routerAvailability.data.error} +

+ )} +
+ + - {isTestingConnection && } - Test Connection - - - - -
-
- - + Need Help? + + } + /> + Get help on our github + +
+ + + + + + + +
+
diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts index 7b12a0d54fd..4097afc17fe 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts @@ -1,274 +1,325 @@ -import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; -import { buildModelAvailability, deploymentRefsFromModelInfo } from "@/lib/autorouter_presets"; import { - buildAutomaticRouterConfig, - buildPreferredTierModels, - prefillEmptyTiers, - type PreferredTierModels, + deployedRouterModels, + recommendSolverPair, + solverPairConfig, + nextRouterName, + simpleSetupError, } from "./auto_setup"; -const models = (...names: string[]) => names.map((model_group) => ({ model_group, mode: "chat" })); -const reasoningModel = (model_group: string, supported_reasoning_efforts: string[]) => ({ - model_group, - mode: "chat", - supports_reasoning: true, - supported_reasoning_efforts, -}); -const deployment = (model_name: string, model = model_name): AutoRouterDeployment => ({ - model_name, +const deployment = (name: string, model: string, extra: Partial = {}): AutoRouterDeployment => ({ + model_name: name, litellm_params: { model }, + ...extra, }); -const tierModels = (config: ReturnType) => - config && Object.values(config.tiers).map((tier) => (typeof tier === "string" ? tier : tier[0])); +const resolve = (...rows: AutoRouterDeployment[]) => + deployedRouterModels( + rows.map((row) => ({ model_group: row.model_name!, mode: "chat" })), + rows, + ); +const names = (pair: ReturnType) => pair && [pair.everyday.name, pair.hard.name]; +const ladder = () => + resolve( + deployment("small", "anthropic/claude-haiku-5-5"), + deployment("medium", "anthropic/claude-sonnet-5-5"), + deployment("large", "anthropic/claude-opus-5-5"), + ); -describe("buildPreferredTierModels", () => { - it("recognizes curated models that are not in a preset", () => { - const available = ["gpt-5.6-luna", "claude-sonnet-5", "grok-4.6", "deepseek-v4-pro"]; - const availability = buildModelAvailability(available, []); - const preferred = buildPreferredTierModels([], availability); - const expected: PreferredTierModels = { - SIMPLE: ["gpt-5.6-luna"], - MEDIUM: ["claude-sonnet-5"], - COMPLEX: ["deepseek-v4-pro", "grok-4.6"], - REASONING: ["deepseek-v4-pro", "grok-4.6"], - }; +const catalog: Record = JSON.parse( + readFileSync(new URL("../../../../../model_prices_and_context_window.json", import.meta.url), "utf8"), +); +const catalogOpusModels = Object.entries(catalog).filter( + ([id, metadata]) => metadata.mode === "chat" && /claude-opus-\d/.test(id) && !/-(?:fast|think)$/.test(id), +); - expect(preferred).toEqual(expected); - }); - - it("prefers the current model ladder over older preset entries", () => { - const availability = buildModelAvailability(["gpt-5.6-luna", "gpt-4o-mini"], []); - const preferred = buildPreferredTierModels( - [ - { - key: "old", - label: "Old", - description: "Old model", - complexity_router_config: { - tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] }, - classifier_type: "heuristic_v2", - }, - }, - ], - availability, - ); - - expect(preferred.SIMPLE).toEqual(["gpt-5.6-luna", "gpt-4o-mini"]); - }); -}); - -describe("buildAutomaticRouterConfig", () => { - it("selects native Terra and Sol groups with their reasoning settings, retaining cloud fallback", () => { - const modelNames = ["gpt-5.6-terra", "gpt-5.6-sol"]; - const deployments = modelNames.flatMap((model) => [ - deployment(model, `azure/${model}`), - deployment(`z-native-${model}`, `openai/${model}`), - ]); - const available = deployments.map(({ model_name }) => reasoningModel(model_name!, ["none", "high"])); - const availability = buildModelAvailability( - available.map(({ model_group }) => model_group), - deploymentRefsFromModelInfo(deployments), - ); - const preferred = buildPreferredTierModels([], availability); - const config = buildAutomaticRouterConfig(available, deployments, preferred); - - expect(tierModels(config)).toEqual([ - "z-native-gpt-5.6-terra", - "z-native-gpt-5.6-terra", - "z-native-gpt-5.6-sol", - "z-native-gpt-5.6-sol", - ]); - expect(config?.tier_model_params).toEqual({ - REASONING: { "z-native-gpt-5.6-sol": { reasoning_effort: "high" } }, - }); - - const cloudOnly = models(...modelNames); - const cloudAvailability = buildModelAvailability(modelNames, deploymentRefsFromModelInfo(deployments)); - const cloudPreferred = buildPreferredTierModels([], cloudAvailability); - - expect(tierModels(buildAutomaticRouterConfig(cloudOnly, deployments, cloudPreferred))).toEqual([ - "gpt-5.6-terra", - "gpt-5.6-terra", - "gpt-5.6-sol", - "gpt-5.6-sol", - ]); - }); - - it("selects one preferred model for each tier", () => { - const preferred: PreferredTierModels = { - SIMPLE: ["simple"], - MEDIUM: ["medium"], - COMPLEX: ["complex"], - REASONING: ["reasoning"], - }; - - expect( - tierModels(buildAutomaticRouterConfig(models("simple", "medium", "complex", "reasoning"), [], preferred)), - ).toEqual(["simple", "medium", "complex", "reasoning"]); - }); - - it.each([ - { - provider: "OpenAI", - available: ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-6-astra"], - expected: ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-6-astra", "gpt-6-astra"], - supportedEfforts: ["low", "medium", "high", "xhigh", "max"], - effort: "max", - }, - { - provider: "Anthropic", - available: ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5"], - expected: ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5", "claude-opus-5"], - supportedEfforts: ["low", "medium", "high", "max"], - effort: "max", - }, - { - provider: "Google", - available: ["gemini-3.5-flash-lite", "gemini-3.8-flash", "gemini-3.1-pro-preview"], - expected: ["gemini-3.5-flash-lite", "gemini-3.8-flash", "gemini-3.1-pro-preview", "gemini-3.1-pro-preview"], - supportedEfforts: ["low", "medium", "high"], - effort: "high", - }, - { - provider: "DeepSeek", - available: ["deepseek-v4-flash", "deepseek-v4-pro"], - expected: ["deepseek-v4-flash", "deepseek-v4-flash", "deepseek-v4-pro", "deepseek-v4-pro"], - supportedEfforts: ["none", "high"], - effort: "high", - }, - { - provider: "xAI", - available: ["grok-4.6"], - expected: ["grok-4.6", "grok-4.6", "grok-4.6", "grok-4.6"], - supportedEfforts: ["low", "medium", "high", "xhigh"], - effort: "xhigh", - }, - ])( - "uses the current $provider ladder and strongest advertised reasoning effort", - ({ available, expected, supportedEfforts, effort }) => { - const availability = buildModelAvailability(available, []); - const preferred = buildPreferredTierModels([], availability); - const modelInfo = models(...available).map((model) => - model.model_group === expected[3] ? reasoningModel(model.model_group, supportedEfforts) : model, - ); - - const config = buildAutomaticRouterConfig(modelInfo, [], preferred); - - expect(tierModels(config)).toEqual(expected); - expect(config?.tier_model_params).toEqual({ - REASONING: { [expected[3]]: { reasoning_effort: effort } }, - }); +describe("deployedRouterModels", () => { + it.each(catalogOpusModels)( + "recognizes the catalog's standard Opus deployment %s behind a custom alias", + (id, metadata) => { + const row = { + model_name: "custom-deployment", + litellm_params: { model: id, custom_llm_provider: metadata.litellm_provider }, + }; + const models = deployedRouterModels([{ model_group: "custom-deployment", mode: "chat" }], [row]); + expect(models).toEqual([expect.objectContaining({ name: "custom-deployment", family: "claude", line: "opus" })]); }, ); - it("never exceeds the selected model group's advertised reasoning efforts", () => { - const available = ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-5.6-sol"]; - const availability = buildModelAvailability(available, []); - const preferred = buildPreferredTierModels([], availability); - const modelInfo = [ - ...models("gpt-5.6-luna", "gpt-5.6-terra"), - reasoningModel("gpt-5.6-sol", ["none", "low", "medium", "high", "xhigh"]), - ]; - - const config = buildAutomaticRouterConfig(modelInfo, [], preferred); - - expect(config?.tier_model_params).toEqual({ - REASONING: { "gpt-5.6-sol": { reasoning_effort: "xhigh" } }, - }); + it("resolves underlying identities independently of deployment aliases and serving providers", () => { + const models = resolve( + deployment("everyday", "bedrock/us.openai.gpt-6.1-sol"), + deployment("hard", "bedrock_converse/global.openai.gpt-6-astra"), + ); + const expectedEveryday = { name: "everyday", family: "openai", line: "sol", version: [6, 1], provider: "Bedrock" }; + const expectedHard = { name: "hard", family: "openai", line: "astra", version: [6], provider: "Bedrock" }; + expect(models).toEqual( + expect.arrayContaining([expect.objectContaining(expectedEveryday), expect.objectContaining(expectedHard)]), + ); + expect(names(recommendSolverPair(models, "quality", "openai"))).toEqual(["everyday", "hard"]); }); - it("leaves reasoning effort unset when the proxy does not report supported values", () => { - const available = ["grok-4.6"]; - const availability = buildModelAvailability(available, []); - const preferred = buildPreferredTierModels([], availability); - - const config = buildAutomaticRouterConfig(models(...available), [], preferred); - - expect(config?.tier_model_params).toBeUndefined(); - }); - - it("reuses the closest available tier when a tier has no match", () => { - const preferred: PreferredTierModels = { - SIMPLE: ["simple"], - MEDIUM: [], - COMPLEX: ["complex"], - REASONING: [], - }; - - expect(tierModels(buildAutomaticRouterConfig(models("simple", "complex"), [], preferred))).toEqual([ - "simple", - "simple", - "complex", - "complex", + it("recognizes provider-specific Claude names and a declared base model for opaque deployments", () => { + const models = resolve( + deployment("small", "vertex_ai/claude-haiku-4-5@20251001"), + deployment("large", "bedrock/arn:example:inference-profile/us.anthropic.claude-opus-5-5-v1:0"), + deployment("middle", "azure/private-alias", { + litellm_params: { model: "azure/private-alias" }, + model_info: { base_model: "gpt-6.1-sol" }, + }), + ); + expect(models.map(({ name, family, provider }) => ({ name, family, provider }))).toEqual([ + { name: "large", family: "claude", provider: "Bedrock" }, + { name: "middle", family: "openai", provider: "Azure" }, + { name: "small", family: "claude", provider: "Vertex AI" }, ]); }); - it("returns null when none of the available models are recommended", () => { - const preferred: PreferredTierModels = { - SIMPLE: ["missing-simple"], - MEDIUM: ["missing-medium"], - COMPLEX: ["missing-complex"], - REASONING: ["missing-reasoning"], - }; - - expect(buildAutomaticRouterConfig(models("unknown-model"), [], preferred)).toBeNull(); + it("uses only caller-visible chat models and excludes blocked and recursive targets", () => { + const rows = [ + deployment("hidden", "anthropic/claude-opus-6"), + deployment("blocked", "anthropic/claude-opus-5", { model_info: { blocked: true } }), + deployment("embedding", "anthropic/claude-opus-5"), + deployment("recursive", "auto_router/complexity_router"), + deployment("ready", "anthropic/claude-sonnet-5"), + ]; + expect( + deployedRouterModels( + [ + { model_group: "blocked", mode: "chat" }, + { model_group: "embedding", mode: "embedding" }, + { model_group: "recursive" }, + { model_group: "ready", mode: "chat" }, + ], + rows, + ).map((model) => model.name), + ).toEqual(["ready"]); }); - it("ignores non-chat models and existing auto routers", () => { - const preferred: PreferredTierModels = { - SIMPLE: ["gpt-4o-mini", "smart-router"], - MEDIUM: [], - COMPLEX: [], - REASONING: [], - }; - const available = [ - { model_group: "gpt-4o-mini", mode: "chat" }, - { model_group: "image-model", mode: "image_generation" }, - { model_group: "smart-router", mode: "chat" }, + it("does not claim a mixed-version or unknown model group is a known version", () => { + const rows = [ + deployment("mixed", "anthropic/claude-opus-5"), + deployment("mixed", "anthropic/claude-opus-5-5"), + deployment("claude-sonnet-5", "openai/custom-model"), + deployment("wildcard", "openai/*"), ]; + const models = deployedRouterModels( + [ + { model_group: "mixed", mode: "chat" }, + { model_group: "claude-sonnet-5", mode: "chat" }, + { model_group: "wildcard", mode: "chat" }, + ], + rows, + ); + expect(models.map((model) => model.name)).toEqual(["claude-sonnet-5", "mixed"]); + expect(models.every((model) => model.family === undefined)).toBe(true); + expect(recommendSolverPair(models, "balanced", "auto")).toBeNull(); + }); + it("keeps homogeneous replicas and ignores blocked replicas when resolving identity", () => { + const rows = [ + deployment("group", "anthropic/claude-opus-5-5"), + deployment("group", "bedrock/anthropic.claude-opus-5-5-v1:0"), + deployment("group", "anthropic/claude-opus-5", { model_info: { blocked: true } }), + ]; + const models = deployedRouterModels([{ model_group: "group" }], rows); + expect(models).toHaveLength(1); + expect(models[0]).toMatchObject({ name: "group", version: [5, 5], provider: "Anthropic, Bedrock" }); + }); + + it("keeps arbitrary canonical names available without pretending an unversioned alias is latest", () => { expect( - tierModels( - buildAutomaticRouterConfig(available, [deployment("smart-router", "auto_router/complexity_router")], preferred), + deployedRouterModels([{ model_group: "claude-opus-8-12" }, { model_group: "claude-opus-latest" }], []).map( + (model) => model.name, ), - ).toEqual(["gpt-4o-mini", "gpt-4o-mini", "gpt-4o-mini", "gpt-4o-mini"]); + ).toEqual(["claude-opus-8-12", "claude-opus-latest"]); }); }); -describe("prefillEmptyTiers", () => { - const empty: ComplexityRouterConfigValue = { - classifier_type: "jev", - jev_classifier_config: { model: "jev-custom", timeout_ms: 4200 }, - classification_mode: "user_turn", - tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }, - }; - const automatic: ComplexityRouterConfigValue = { - classifier_type: "heuristic_v2", - tiers: { SIMPLE: ["fast"], MEDIUM: ["mid"], COMPLEX: ["strong"], REASONING: ["strong"] }, - tier_model_params: { REASONING: { strong: { reasoning_effort: "high" } } }, - }; - it("fills tiers while preserving a classifier configured before models load", () => { - expect(prefillEmptyTiers(empty, automatic)).toEqual({ - ...empty, - tiers: automatic.tiers, - tier_model_params: automatic.tier_model_params, +describe("recommendSolverPair", () => { + it.each([ + ["cheapest", ["small", "medium"]], + ["balanced", ["small", "large"]], + ["quality", ["medium", "large"]], + ] as const)("uses the requested %s pair", (goal, expected) => { + expect(names(recommendSolverPair(ladder(), goal, "claude"))).toEqual(expected); + }); + + it("selects future deployed releases numerically within each line, without a version allowlist", () => { + const models = resolve( + deployment("old-small", "anthropic/claude-haiku-8-9"), + deployment("new-small", "bedrock/anthropic.claude-haiku-8-10-v1:0"), + deployment("old-large", "anthropic/claude-opus-8-9"), + deployment("new-large", "anthropic/claude-opus-9"), + ); + expect(names(recommendSolverPair(models, "balanced", "claude"))).toEqual(["new-small", "new-large"]); + }); + + it("orders snapshots within a version without confusing a snapshot with the next version", () => { + const models = resolve( + deployment("old", "anthropic/claude-haiku-4-5-20251001"), + deployment("snapshot", "anthropic/claude-haiku-4-5-20251101"), + deployment("next", "anthropic/claude-haiku-4-6"), + deployment("large", "anthropic/claude-opus-5"), + ); + expect(names(recommendSolverPair(models, "balanced", "claude"))).toEqual(["next", "large"]); + expect( + names( + recommendSolverPair( + models.filter((model) => model.name !== "next"), + "balanced", + "claude", + ), + ), + ).toEqual(["snapshot", "large"]); + }); + + it("prefers rolling aliases over snapshots of the same release", () => { + const models = resolve( + deployment("pin", "anthropic/claude-haiku-4-5-20251101"), + deployment("alias", "anthropic/claude-haiku-4-5"), + deployment("hard", "anthropic/claude-opus-5"), + deployment("luna-pin", "openai/gpt-6-luna-2026-01-01"), + deployment("luna", "openai/gpt-6-luna"), + deployment("astra", "openai/gpt-6-astra"), + ); + expect(names(recommendSolverPair(models, "balanced", "claude"))).toEqual(["alias", "hard"]); + expect(names(recommendSolverPair(models, "balanced", "openai"))).toEqual(["luna", "astra"]); + }); + + it.each([ + { models: ["grok-4.20", "grok-4.20-0309"], expected: "grok-4.20" }, + { models: ["grok-4.20-0309", "grok-4.20-0410"], expected: "grok-4.20-0410" }, + { models: ["grok-4.20-0309", "grok-4.21"], expected: "grok-4.21" }, + ])("orders Grok releases and short-date snapshots: $expected", ({ models, expected }) => { + const available = resolve( + deployment("everyday", "openai/gpt-6-luna"), + ...models.map((model) => deployment(model, `xai/${model}`)), + ); + expect(names(recommendSolverPair(available, "balanced", "grok"))).toEqual(["everyday", expected]); + }); + + it("does not rank a major-release snapshot above a newer minor release", () => { + const models = resolve( + deployment("old", "anthropic/claude-5-haiku-20260101"), + deployment("next", "anthropic/claude-haiku-5-1"), + deployment("hard", "anthropic/claude-opus-5"), + ); + expect(names(recommendSolverPair(models, "balanced", "claude"))).toEqual(["next", "hard"]); + }); + + it("prefers a same-family substitute over a matching rank from another family", () => { + const models = [ + ...ladder().filter((model) => model.line !== "haiku"), + ...resolve(deployment("luna", "openai/gpt-6-luna")), + ]; + const pair = recommendSolverPair(models, "balanced", "claude"); + expect(names(pair)).toEqual(["medium", "large"]); + expect(pair?.notice).toContain("closest available pair"); + }); + + it("prefers a complete family for Auto and explains a cross-family substitute when explicitly requested", () => { + const models = [ + ...ladder().filter((model) => model.line === "sonnet"), + ...resolve( + deployment("luna", "bedrock/us.openai.gpt-6-luna"), + deployment("astra", "bedrock/us.openai.gpt-6-astra"), + ), + ]; + expect(names(recommendSolverPair(models, "balanced", "auto"))).toEqual(["luna", "astra"]); + const selected = recommendSolverPair(models, "quality", "claude"); + expect(names(selected)).toEqual(["medium", "astra"]); + expect(selected?.notice).toContain("two model families"); + }); + + it.each([ + ["glm", "zai/glm-7.3-flash", "zai/glm-7.3"], + ["deepseek", "deepseek/deepseek-v9-flash", "deepseek/deepseek-v9-pro"], + ] as const)("handles the %s two-line family for every goal", (family, small, large) => { + const models = resolve(deployment("small", small), deployment("large", large)); + for (const goal of ["cheapest", "balanced", "quality"] as const) + expect(names(recommendSolverPair(models, goal, family))).toEqual(["small", "large"]); + }); + + it("does not fabricate a second solver or substitute a different family when none of the requested family exists", () => { + expect(recommendSolverPair(ladder().slice(0, 1), "balanced", "auto")).toBeNull(); + expect(recommendSolverPair(ladder(), "balanced", "openai")).toBeNull(); + expect(recommendSolverPair([], "balanced", "auto")).toBeNull(); + }); + + it("retains Gemini family suggestions and recognizes the existing Grok fallback", () => { + const models = resolve( + deployment("lite", "gemini/gemini-3.5-flash-lite"), + deployment("flash-old", "vertex_ai/gemini-3.5-flash"), + deployment("flash", "vertex_ai/gemini-3.8-flash"), + deployment("pro", "gemini/gemini-3.1-pro-preview"), + deployment("grok", "xai/grok-4.6"), + ); + expect(names(recommendSolverPair(models, "cheapest", "gemini"))).toEqual(["lite", "flash"]); + expect(names(recommendSolverPair(models, "balanced", "gemini"))).toEqual(["lite", "pro"]); + expect(names(recommendSolverPair(models, "quality", "gemini"))).toEqual(["flash", "pro"]); + expect(models.find((m) => m.name === "grok")).toMatchObject({ family: "grok", provider: "xAI" }); + expect(names(recommendSolverPair(models, "balanced", "grok"))).toEqual(["lite", "grok"]); + }); + + it("uses the latest deployed Terra as a fallback without replacing the preferred OpenAI pairs", () => { + const models = resolve( + deployment("terra-old", "openai/gpt-5.6-terra"), + deployment("terra", "openai/gpt-6.1-terra"), + deployment("sol", "openai/gpt-6.1-sol"), + ); + expect(names(recommendSolverPair(models, "balanced", "openai"))).toEqual(["terra", "sol"]); + const complete = [ + ...models, + ...resolve(deployment("luna", "openai/gpt-6-luna"), deployment("astra", "openai/gpt-6-astra")), + ]; + expect(names(recommendSolverPair(complete, "cheapest", "openai"))).toEqual(["luna", "sol"]); + expect(names(recommendSolverPair(complete, "balanced", "openai"))).toEqual(["luna", "astra"]); + expect(names(recommendSolverPair(complete, "quality", "openai"))).toEqual(["sol", "astra"]); + }); + + it("suggests a deterministic equal-rank cross-family fallback only when no ordered pair exists", () => { + const models = resolve(deployment("opus", "anthropic/claude-opus-5-5"), deployment("astra", "openai/gpt-6-astra")); + const pair = recommendSolverPair(models, "balanced", "auto"); + expect(names(pair)).toEqual(["astra", "opus"]); + expect(pair?.notice).toContain("two model families"); + expect(names(recommendSolverPair([...models].reverse(), "balanced", "auto"))).toEqual(names(pair)); + const withLuna = [...models, ...resolve(deployment("luna", "openai/gpt-6-luna"))]; + expect(names(recommendSolverPair(withLuna, "balanced", "auto"))).toEqual(["luna", "astra"]); + }); + + it("does not prefer native-provider deployments over a newer cloud deployment", () => { + const models = resolve( + deployment("native", "openai/gpt-6-luna"), + deployment("cloud", "bedrock/global.openai.gpt-6.2-luna"), + deployment("strong", "bedrock/global.openai.gpt-6-astra"), + ); + expect(names(recommendSolverPair(models, "balanced", "openai"))).toEqual(["cloud", "strong"]); + }); +}); + +describe("solverPairConfig", () => { + it("builds two solver references using built-in rules without paid classifiers, custom tiers or output overrides", () => { + expect(solverPairConfig("everyday", "hard")).toEqual({ + classifier_type: "heuristic", + classification_mode: "user_turn", + tiers: { SIMPLE: ["everyday"], MEDIUM: ["everyday"], COMPLEX: ["hard"], REASONING: ["hard"] }, }); }); - it("leaves the draft alone when no recommended models are available", () => { - expect(prefillEmptyTiers(empty, null)).toBe(empty); +}); + +describe("simple setup validation", () => { + it("avoids existing names without requiring a naming decision", () => { + expect(nextRouterName([])).toBe("auto-router"); + expect(nextRouterName(["auto-router", "auto-router-2", "auto-router-4"])).toBe("auto-router-3"); }); - it("preserves a partial manual assignment", () => { - const partial = { ...empty, tiers: { ...empty.tiers, SIMPLE: ["manual"] } }; - expect(prefillEmptyTiers(partial, automatic)).toBe(partial); - }); - it("preserves a custom tier set created before models load", () => { - const custom = { ...empty, custom_tier_set: { tiers: [], fallback_tier_id: "" } }; - expect(prefillEmptyTiers(custom, automatic)).toBe(custom); - }); - it.each(["capability", "llm_v2"] as const)("does not prefill %s solver pools", (classifier_type) => { - const solver = { ...empty, classifier_type }; - expect(prefillEmptyTiers(solver, automatic)).toBe(solver); + it("blocks stale or incomplete inventory and same-line selections", () => { + const [small, large] = [ladder()[0], ladder()[2]]; + expect(simpleSetupError(true, false, small, large)).not.toBeNull(); + expect(simpleSetupError(false, true, small, large)).not.toBeNull(); + expect(simpleSetupError(false, false, small)).not.toBeNull(); + expect(simpleSetupError(false, false, small, small)).not.toBeNull(); + expect(simpleSetupError(false, false, small, large)).toBeNull(); }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts index 8b60336cd27..39476470928 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts @@ -1,110 +1,263 @@ -import { activeTierRows } from "./tier_rows"; -import { isForecastClassifier } from "./forecast_classifier_config"; -import { isAutoRouterDeployment, type AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; +import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; +import { isAutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; import type { ModelGroup } from "@/components/llm_calls/fetch_models"; -import { resolveAvailableModel, type AutoRouterPreset, type ModelAvailability } from "@/lib/autorouter_presets"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; -const TIER_NAMES = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"] as const; -type TierName = (typeof TIER_NAMES)[number]; -export type PreferredTierModels = Record; +export const ROUTER_FAMILIES = { + claude: "Claude", + openai: "OpenAI", + gemini: "Gemini", + grok: "Grok", + glm: "GLM", + deepseek: "DeepSeek", +} as const; +export const ROUTER_GOALS = { cheapest: "Cheapest", balanced: "Balanced", quality: "Best quality" } as const; +export type RouterFamily = keyof typeof ROUTER_FAMILIES; +export type RouterGoal = keyof typeof ROUTER_GOALS; +export type FamilyPreference = RouterFamily | "auto"; -const REASONING_EFFORT_STRENGTH = ["max", "xhigh", "high", "medium", "low", "minimal", "none"] as const; +type ModelIdentity = { + family: RouterFamily; + line: string; + rank: number; + version: readonly number[]; + snapshot: number; + model: string; +}; +export type SolverModel = Partial & { name: string; provider: string; model: string }; +export type RouterModel = ModelIdentity & SolverModel; +export type SolverPair = { everyday: RouterModel; hard: RouterModel; notice: string | null }; -const CURRENT_TIER_MODELS: PreferredTierModels = { - SIMPLE: ["gpt-5.6-luna", "claude-haiku-4-5", "gemini-3.5-flash-lite", "deepseek-v4-flash"], - MEDIUM: ["gpt-5.6-terra", "claude-sonnet-5", "gemini-3.8-flash", "deepseek-v4-flash"], - COMPLEX: ["gpt-6-astra", "gpt-5.6-sol", "claude-opus-5", "gemini-3.1-pro-preview", "deepseek-v4-pro", "grok-4.6"], - REASONING: ["gpt-6-astra", "gpt-5.6-sol", "claude-opus-5", "gemini-3.1-pro-preview", "deepseek-v4-pro", "grok-4.6"], +const releaseVersion = (raw: string, date?: string) => { + const snapshot = raw.match(/[-.](\d{8})$/); + const release = snapshot ? raw.slice(0, -9) : raw; + return { + version: release.split(/[.-]/).map(Number), + // An undated alias follows the current snapshot of its release. + snapshot: date || snapshot ? Number((date ?? snapshot![1]).replaceAll("-", "")) : Number.MAX_SAFE_INTEGER, + }; }; -export const buildPreferredTierModels = ( - presets: AutoRouterPreset[], - availability: ModelAvailability, -): PreferredTierModels => - Object.fromEntries( - TIER_NAMES.map((tier) => [ - tier, - Array.from( - new Set( - [ - ...CURRENT_TIER_MODELS[tier], - ...presets.flatMap((preset) => preset.complexity_router_config.tiers[tier]), - ].flatMap((model) => { - const resolved = resolveAvailableModel(model, availability); - return resolved ? [resolved] : []; - }), - ), +const parseModel = (raw: string): ModelIdentity | null => { + const model = raw + .toLowerCase() + .split("/") + .at(-1)! + .replace(/^(?:us|eu|apac|global|au|jp|us-gov)\./, "") + .replace(/^(?:anthropic|openai|zai|deepseek)\./, "") + .replace(/^databricks-/, "") + .replace(/@default$/, "") + .replace(/-v\d+(?::\d+)?$/, ""); + const versionFirst = model.match(/^claude-(\d+(?:[.-]\d+)*)-(haiku|sonnet|opus)(?:[-@]([0-9]{8}))?$/); + const claude = + model.match(/^claude-(haiku|sonnet|opus)-(\d+(?:[.-]\d+)*)(?:@([0-9]+))?$/) ?? + (versionFirst ? [versionFirst[0], versionFirst[2], versionFirst[1], versionFirst[3]] : null); + if (claude) + return { + family: "claude", + line: claude[1], + rank: ["haiku", "sonnet", "opus"].indexOf(claude[1]), + ...releaseVersion(claude[2], claude[3]), + model, + }; + const openai = model.match(/^gpt-(\d+(?:[.-]\d+)*)-(luna|terra|sol|astra)(?:-(\d{4}-\d{2}-\d{2}))?$/); + if (openai) + return { + family: "openai", + line: openai[2], + rank: openai[2] === "terra" ? 0.5 : ["luna", "sol", "astra"].indexOf(openai[2]), + ...releaseVersion(openai[1], openai[3]), + model, + }; + const gemini = model.match(/^gemini-(\d+(?:[.-]\d+)*)-(flash-lite|flash|pro)(?:-preview)?$/); + if (gemini) + return { + family: "gemini", + line: gemini[2], + rank: ["flash-lite", "flash", "pro"].indexOf(gemini[2]), + ...releaseVersion(gemini[1]), + model, + }; + const grok = model.match(/^grok-(\d+(?:[.-]\d+)*?)(?:-(\d{4}))?$/); + if (grok) return { family: "grok", line: "grok", rank: 2, ...releaseVersion(grok[1], grok[2]), model }; + const glm = model.match(/^glm-(\d+(?:[.-]\d+)*)(-flash)?$/); + if (glm) + return { + family: "glm", + line: glm[2] ? "flash" : "glm", + rank: glm[2] ? 0 : 2, + ...releaseVersion(glm[1]), + model, + }; + const deepseek = model.match(/^deepseek-v(\d+(?:[.-]\d+)*)-(flash|pro)$/); + if (deepseek) + return { + family: "deepseek", + line: deepseek[2], + rank: deepseek[2] === "flash" ? 0 : 2, + ...releaseVersion(deepseek[1]), + model, + }; + return null; +}; + +const identityOf = (deployment: AutoRouterDeployment): ModelIdentity | null => + parseModel(deployment.litellm_params?.model ?? "") ?? + parseModel(deployment.model_info?.base_model ?? deployment.litellm_params?.base_model ?? ""); + +const providerOf = (deployment: AutoRouterDeployment, group: ModelGroup, family?: RouterFamily): string => { + const raw = deployment.litellm_params?.model ?? ""; + const provider = + deployment.litellm_params?.custom_llm_provider || + (raw.includes("/") ? raw.split("/")[0] : group.providers?.join(", ")) || + (family + ? { claude: "anthropic", glm: "zai", openai: "openai", gemini: "gemini", grok: "xai", deepseek: "deepseek" }[ + family + ] + : ""); + if (provider.startsWith("bedrock")) return "Bedrock"; + return ( + ( + { + anthropic: "Anthropic", + openai: "OpenAI", + gemini: "Google AI", + xai: "xAI", + azure: "Azure", + azure_ai: "Azure AI", + databricks: "Databricks", + github_copilot: "GitHub Copilot", + "vertex_ai-anthropic_models": "Vertex AI", + vertex_ai: "Vertex AI", + zai: "Z.ai", + deepseek: "DeepSeek", + } as Record + )[provider] ?? provider + ); +}; + +const compareVersion = (left: readonly number[], right: readonly number[]): number => { + const index = Array.from({ length: Math.max(left.length, right.length) }, (_, i) => i).find( + (i) => (left[i] ?? 0) !== (right[i] ?? 0), + ); + return index === undefined ? 0 : (left[index] ?? 0) - (right[index] ?? 0); +}; + +const compareRelease = (left: ModelIdentity, right: ModelIdentity): number => + compareVersion(left.version, right.version) || left.snapshot - right.snapshot; + +export const deployedRouterModels = ( + groups: readonly ModelGroup[], + deployments: readonly AutoRouterDeployment[], +): SolverModel[] => + groups + .flatMap((group) => { + if (group.mode && group.mode !== "chat") return []; + const rows = deployments.filter((row) => row.model_name === group.model_group); + if ( + rows.some(isAutoRouterDeployment) || + group.model_group.includes("*") || + group.model_group.startsWith("auto_router/") + ) + return []; + const active = rows.filter((row) => !row.model_info?.blocked); + if (rows.length > 0 && active.length === 0) return []; + if (active.some((row) => row.model_info?.mode && row.model_info.mode !== "chat")) return []; + if (active.some((row) => row.litellm_params?.model?.includes("*"))) return []; + const identities = active.length > 0 ? active.map(identityOf) : [parseModel(group.model_group)]; + const first = identities[0]; + const homogeneous = + first && + identities.every((identity) => { + if (!identity) return false; + const sameLine = identity.family === first.family && identity.line === first.line; + return sameLine && compareRelease(identity, first) === 0; + }); + const identity = homogeneous ? first : undefined; + const providers = + active.length > 0 + ? active.map((row) => providerOf(row, group, identity?.family)) + : [providerOf({}, group, identity?.family)]; + return [ + { + ...identity, + name: group.model_group, + model: identity?.model ?? group.model_group, + provider: [...new Set(providers)].toSorted().join(", "), + }, + ]; + }) + .toSorted((a, b) => a.name.localeCompare(b.name)); + +const latestLines = (models: readonly RouterModel[]): RouterModel[] => + models.filter( + (model) => + !models.some( + (other) => other.family === model.family && other.line === model.line && compareRelease(other, model) > 0, ), - ]), - ) as PreferredTierModels; - -const selectPreferredTierModels = ( - preferredByTier: PreferredTierModels, - usableNames: ReadonlySet, -): [string, string, string, string] | null => { - const preferred = TIER_NAMES.map((tier) => preferredByTier[tier].find((name) => usableNames.has(name))); - const candidates = preferred.flatMap((model, tier) => (model ? [{ model, tier }] : [])); - if (candidates.length === 0) return null; - - const nearest = (tier: number): string => - [...candidates].sort( - (left, right) => Math.abs(left.tier - tier) - Math.abs(right.tier - tier) || left.tier - right.tier, - )[0].model; - return preferred.map((model, tier) => model ?? nearest(tier)) as [string, string, string, string]; -}; - -export const buildAutomaticRouterConfig = ( - models: ModelGroup[], - deployments: AutoRouterDeployment[], - preferredByTier: PreferredTierModels, -): ComplexityRouterConfigValue | null => { - const autoRouterNames: ReadonlySet = new Set( - deployments - .filter(isAutoRouterDeployment) - .flatMap((deployment) => (deployment.model_name ? [deployment.model_name] : [])), ); - const names = Array.from( - new Set( - models - .filter((model) => model.mode === undefined || model.mode === "chat") - .map((model) => model.model_group) - .filter((name) => name && !name.startsWith("auto_router/") && !autoRouterNames.has(name)), - ), + +const requestedRanks = (goal: RouterGoal): readonly [number, number] => + (({ cheapest: [0, 1], quality: [1, 2], balanced: [0, 2] }) as const)[goal]; + +export const recommendSolverPair = ( + models: readonly SolverModel[], + goal: RouterGoal, + family: FamilyPreference, +): SolverPair | null => { + const recognized = models.filter((model): model is RouterModel => model.family !== undefined); + const latest = latestLines(recognized); + const [low, high] = requestedRanks(goal); + const pairs = latest.flatMap((everyday) => + latest + .filter((hard) => { + const matchesFamily = family === "auto" || everyday.family === family || hard.family === family; + const ordered = hard.rank > everyday.rank; + const equalRankFallback = hard.rank === everyday.rank && hard.family !== everyday.family; + return (ordered || equalRankFallback) && matchesFamily; + }) + .map((hard) => ({ everyday, hard })), ); - if (names.length === 0) return null; - const usableNames: ReadonlySet = new Set(names); - const selected = selectPreferredTierModels(preferredByTier, usableNames); - if (selected === null) return null; - - const supportedReasoningEfforts = models.find( - (model) => model.model_group === selected[3], - )?.supported_reasoning_efforts; - const reasoningEffort = REASONING_EFFORT_STRENGTH.find((effort) => supportedReasoningEfforts?.includes(effort)); - - return { - tiers: { - SIMPLE: [selected[0]], - MEDIUM: [selected[1]], - COMPLEX: [selected[2]], - REASONING: [selected[3]], - }, - classifier_type: "heuristic_v2", - ...(reasoningEffort && { - tier_model_params: { REASONING: { [selected[3]]: { reasoning_effort: reasoningEffort } } }, - }), - }; + const score = ({ everyday, hard }: { everyday: RouterModel; hard: RouterModel }): number => + (everyday.family === hard.family ? 0 : 100) + + (everyday.rank === hard.rank ? 10 : 0) + + Math.abs(everyday.rank - low) + + Math.abs(hard.rank - high); + const chosen = pairs.toSorted( + (a, b) => + score(a) - score(b) || a.everyday.name.localeCompare(b.everyday.name) || a.hard.name.localeCompare(b.hard.name), + )[0]; + if (!chosen) return null; + const mixed = chosen.everyday.family !== chosen.hard.family; + const substituted = chosen.everyday.rank !== low || chosen.hard.rank !== high; + const mixedNotice = mixed ? "A complete pair is unavailable in one family. Using two model families." : null; + const substituteNotice = substituted ? "Using the closest available pair in this family." : null; + return { ...chosen, notice: mixedNotice ?? substituteNotice }; }; -export const prefillEmptyTiers = ( - current: ComplexityRouterConfigValue, - automatic: ComplexityRouterConfigValue | null, -): ComplexityRouterConfigValue => { - if (!automatic || current.custom_tier_set || isForecastClassifier(current.classifier_type)) return current; - if (activeTierRows(current).some((row) => row.models.length > 0)) return current; - return { - ...current, - tiers: { ...current.tiers, ...automatic.tiers }, - tier_model_params: { ...current.tier_model_params, ...automatic.tier_model_params }, - }; +export const solverPairConfig = (everyday: string, hard: string): ComplexityRouterConfigValue => ({ + classifier_type: "heuristic", + classification_mode: "user_turn", + tiers: { SIMPLE: [everyday], MEDIUM: [everyday], COMPLEX: [hard], REASONING: [hard] }, +}); + +export const simpleSetupError = ( + loading: boolean, + failed: boolean, + everyday?: SolverModel, + hard?: SolverModel, +): string | null => { + if (loading) return "Loading deployed models"; + if (failed) return "Could not verify deployed models"; + if (!everyday || !hard) return "Select two different deployed models"; + if (everyday.name === hard.name) return "Select two different deployed models"; + return null; +}; + +export const nextRouterName = (names: readonly string[]): string => { + const candidates = [ + "auto-router", + ...Array.from({ length: names.length + 1 }, (_, index) => `auto-router-${index + 2}`), + ]; + return candidates.find((name) => !names.includes(name))!; }; From 7045139c3a6fe81f80895cb3887882056b5755fe Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 26 Oct 2026 15:40:19 -0700 Subject: [PATCH 076/155] fix(router): stop concurrent deployment deletes from crashing deployment lookups (#47680) * fix(router): stop concurrent deployment deletes from crashing deployment lookups delete_deployment and upsert_deployment shrink model_list before they repair the index maps, so a lookup on another thread in that window read a stale index: IndexError, or the neighbouring deployment and its credentials. An add racing a delete could corrupt the index for good. One RLock now covers model_list and the three index maps. Delete, upsert, add and the set_model_list reset change them under it, and every lookup that turns an index into a deployment reads under it. Fixes #45242 * test(router): release the held delete when the read reaches the index lock The race tests released the held write after a fixed 0.2 s, so a reader the runner stalled past it read the repaired list and passed on unfixed code. The read now runs on its own thread and the write stays held until the read returns or reaches the router's index lock; the 10 s bound only fails a broken synchronization. The integration test gains the audit cells: chat, streaming, Responses and Anthropic messages calls (sync and async, by model name and team public name) during a held delete, the same for upsert, add and set_model_list, a request for the deployment being deleted, a provider error, a double delete, and thread and async bursts during upsert churn. * test(router): cover the early return of a delete and the team filter of get_model_ids delete_deployment now returns None from inside the index lock, and get_model_ids by name reads its deployments through _deployments_named; pin both: deleting an unknown id returns None and leaves the list alone, and exclude_team_models drops only the team deployment. * fix(router): read deployments by name under the index lock outside the Router The response polling provider check, the complexity router's adaptive pool and the savings baseline turned the name index into list positions themselves, so a concurrent delete could still hand them a stale slot. They now read through Router.deployments_named, which takes the deployment index lock. * fix(router): type the locked deployment lookups as DeploymentTypedDict deployments_named and _team_deployments_named returned tuple[Any, ...]. They now return DeploymentTypedDict rows, which is what model_list holds. The adaptive pool setup in the Router and the complexity router drop their branches for pydantic Deployment rows, which these lookups never return, and should_include_deployment accepts the typed row. --------- Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- .../proxy/response_polling/polling_handler.py | 5 +- litellm/router.py | 243 ++++----- .../complexity_router/complexity_router.py | 15 +- litellm/router_strategy/savings_baseline.py | 11 +- .../router_code_coverage.py | 1 + ...router_concurrent_deployment_delete_sdk.py | 474 ++++++++++++++++++ .../test_response_polling_handler.py | 37 +- .../router_strategy/test_complexity_router.py | 21 +- .../router_strategy/test_savings_baseline.py | 3 +- tests/unit/test_router/test_router.py | 321 +++++++++++- 10 files changed, 964 insertions(+), 167 deletions(-) create mode 100644 tests/integration/sdk/test_router_concurrent_deployment_delete_sdk.py diff --git a/litellm/proxy/response_polling/polling_handler.py b/litellm/proxy/response_polling/polling_handler.py index 787f27f56c8..e50587b1a0e 100644 --- a/litellm/proxy/response_polling/polling_handler.py +++ b/litellm/proxy/response_polling/polling_handler.py @@ -308,10 +308,7 @@ def should_use_polling_for_request( # Otherwise, check ALL deployments for this model_name in router elif llm_router is not None: try: - # Get all deployment indices for this model name - indices: Final = llm_router.model_name_to_deployment_indices.get(model, []) - for idx in indices: - deployment_dict = llm_router.model_list[idx] + for deployment_dict in llm_router.deployments_named(model) or (): litellm_params = deployment_dict.get("litellm_params", {}) # Check custom_llm_provider first diff --git a/litellm/router.py b/litellm/router.py index 7993b5b921c..3af39736e25 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1440,6 +1440,9 @@ class Router: # Maps (team_id, team_public_model_name) -> list of indices in model_list self.team_model_to_deployment_indices: dict[tuple[str, str], list[int]] = {} self.team_public_model_names: frozenset[str] = frozenset() + # Guards model_list and the three index maps above: writers change them, and readers turn an + # index into a deployment, under it, so a lookup never lands between a removal and its re-index + self._deployment_index_lock = threading.RLock() # Initialize cache attributes that ``_invalidate_model_group_info_cache`` # and ``_invalidate_access_groups_cache`` touch *before* the first @@ -2180,8 +2183,8 @@ class Router: public_model_name: Final = deployment.model_info.team_public_model_name if team_id is None or public_model_name is None: return False - sibling_indices: Final = self.team_model_to_deployment_indices.get((team_id, public_model_name)) or () - routable_siblings: Final = self._filter_blocked_deployments([self.model_list[idx] for idx in sibling_indices]) + siblings: Final = self._team_deployments_named(team_id, public_model_name) or () + routable_siblings: Final = self._filter_blocked_deployments([dict(sibling) for sibling in siblings]) return len(routable_siblings) > 1 _OVERRIDABLE_ROUTING_STRATEGIES: frozenset[str] = frozenset({"simple-shuffle", *_DEFAULT_SELECTOR_ATTR_BY_STRATEGY}) @@ -10827,20 +10830,17 @@ class Router: # O(k) via the name→indices map: only touch deployments whose name # is listed in `available_models`, instead of scanning model_list. for name in config.available_models: - indices = self.model_name_to_deployment_indices.get(name, []) - if not indices: + deployments = self.deployments_named(name) + if not deployments: continue - d = (self.model_list or [])[indices[0]] - mi = d.get("model_info") if isinstance(d, dict) else d.model_info - mi_dict: dict[str, Any] = mi if isinstance(mi, dict) else (mi.model_dump() if mi else {}) + d = deployments[0] + mi_dict: dict[str, Any] = d.get("model_info") or {} prefs_raw = mi_dict.get("adaptive_router_preferences") if prefs_raw is not None: model_to_prefs[name] = AdaptiveRouterPreferences(**prefs_raw) # model_info is the conventional pricing location elsewhere in LiteLLM; litellm_params wins if set. - lp = d.get("litellm_params") if isinstance(d, dict) else d.litellm_params - lp_dict: dict[str, Any] = lp if isinstance(lp, dict) else (lp.model_dump() if lp else {}) - cost = lp_dict.get("input_cost_per_token") + cost = d.get("litellm_params", {}).get("input_cost_per_token") if cost is None: cost = mi_dict.get("input_cost_per_token") if cost is not None: @@ -10932,10 +10932,11 @@ class Router: def set_model_list(self, model_list: list): original_model_list: Final = copy.deepcopy(model_list) self._discovered_model_info_cache.flush_cache() - self.model_list = [] - self.model_id_to_deployment_index_map = {} # Reset the index - self.model_name_to_deployment_indices = {} # Reset the model_name index - self.team_model_to_deployment_indices = {} # Reset the team_model index + with self._deployment_index_lock: + self.model_list = [] + self.model_id_to_deployment_index_map = {} # Reset the index + self.model_name_to_deployment_indices = {} # Reset the model_name index + self.team_model_to_deployment_indices = {} # Reset the team_model index self.team_pattern_routers = {} self.team_public_model_names = frozenset() # Reset per-strategy router registries so hot-reload doesn't leave @@ -11326,26 +11327,27 @@ class Router: - model: dict - the model to add to the list - model_id: Optional[str] - the model ID to use for indexing. If None, will try to get from model["model_info"]["id"] """ - idx: Final = len(self.model_list) - self.model_list.append(model) - self._invalidate_model_group_info_cache() - self._invalidate_access_groups_cache() + with self._deployment_index_lock: + idx: Final = len(self.model_list) + self.model_list.append(model) + self._invalidate_model_group_info_cache() + self._invalidate_access_groups_cache() - # Update model_id index for O(1) lookup - if model_id is not None: - self.model_id_to_deployment_index_map[model_id] = idx - elif model.get("model_info", {}).get("id") is not None: - self.model_id_to_deployment_index_map[model["model_info"]["id"]] = idx + # Update model_id index for O(1) lookup + if model_id is not None: + self.model_id_to_deployment_index_map[model_id] = idx + elif model.get("model_info", {}).get("id") is not None: + self.model_id_to_deployment_index_map[model["model_info"]["id"]] = idx - # Update model_name index for O(1) lookup - model_name: Final = model.get("model_name") - if model_name: - if model_name not in self.model_name_to_deployment_indices: - self.model_name_to_deployment_indices[model_name] = [] - self.model_name_to_deployment_indices[model_name].append(idx) + # Update model_name index for O(1) lookup + model_name: Final = model.get("model_name") + if model_name: + if model_name not in self.model_name_to_deployment_indices: + self.model_name_to_deployment_indices[model_name] = [] + self.model_name_to_deployment_indices[model_name].append(idx) - # Update team_model index for O(1) team-scoped lookup - self._update_team_model_index(model, idx) + # Update team_model index for O(1) team-scoped lookup + self._update_team_model_index(model, idx) def upsert_deployment(self, deployment: Deployment) -> Deployment | None: """ @@ -11372,18 +11374,15 @@ class Router: # if there is a new litellm param -> then update the deployment # remove the previous deployment - removal_idx: int | None = None - deployment_id: Final = deployment.model_info.id - deployment_fast_mapping: Final = self.model_id_to_deployment_index_map - - if deployment_id in deployment_fast_mapping: - removal_idx = deployment_fast_mapping[deployment_id] - + with self._deployment_index_lock: + removal_idx: Final = self.model_id_to_deployment_index_map.get(_deployment_model_id) if removal_idx is not None: self.model_list.pop(removal_idx) self._invalidate_model_group_info_cache() self._invalidate_access_groups_cache() - self._update_deployment_indices_after_removal(model_id=deployment_id, removal_idx=removal_idx) + self._update_deployment_indices_after_removal( + model_id=_deployment_model_id, removal_idx=removal_idx + ) # Free the outgoing deployment's pre-routing strategy slot (keyed by the # OLD model_name/tags) before the re-add below re-registers it. @@ -11617,35 +11616,32 @@ class Router: - The deleted deployment - OR None (if deleted deployment not found) """ - deployment_idx = None - if id in self.model_id_to_deployment_index_map: - deployment_idx = self.model_id_to_deployment_index_map[id] - try: - if deployment_idx is not None: + with self._deployment_index_lock: + deployment_idx: Final = self.model_id_to_deployment_index_map.get(id) + if deployment_idx is None: + return None # Pop the item from the list first item: Final = self.model_list.pop(deployment_idx) self._invalidate_model_group_info_cache() self._invalidate_access_groups_cache() self._update_deployment_indices_after_removal(model_id=id, removal_idx=deployment_idx) - _budget_limiter: Final = self.get_router_deployment_budget_limiter() - if _budget_limiter is not None: - _budget_limiter.unregister_deployment_budget(model_id=id) - try: - self._unregister_pre_routing_strategy_for_deployment( - deployment=item if isinstance(item, Deployment) else Deployment(**item) - ) - except Exception: - verbose_router_logger.exception( - "delete_deployment: could not release pre-routing strategies for model_id=%s; " - "the deployment is out of the model_list and its indices are repaired", - id, - ) - return item - else: - return None + _budget_limiter: Final = self.get_router_deployment_budget_limiter() + if _budget_limiter is not None: + _budget_limiter.unregister_deployment_budget(model_id=id) + try: + self._unregister_pre_routing_strategy_for_deployment( + deployment=item if isinstance(item, Deployment) else Deployment(**item) + ) + except Exception: + verbose_router_logger.exception( + "delete_deployment: could not release pre-routing strategies for model_id=%s; " + "the deployment is out of the model_list and its indices are repaired", + id, + ) except Exception: return None + return item def get_router_deployment_budget_limiter( self, @@ -11694,6 +11690,18 @@ class Router: if _budget_limiter is not None: _budget_limiter.register_deployment_budget(deployment=deployment.to_json(exclude_none=True)) + def deployments_named(self, model_name: str) -> tuple[DeploymentTypedDict, ...] | None: + """The deployments indexed under `model_name`, read under the index lock, or None when the name is not indexed.""" + with self._deployment_index_lock: + indices: Final = self.model_name_to_deployment_indices.get(model_name) + return None if indices is None else tuple(self.model_list[idx] for idx in indices) + + def _team_deployments_named(self, team_id: str, public_model_name: str) -> tuple[DeploymentTypedDict, ...] | None: + """The deployments `team_id` serves under `public_model_name`, or None when that pair is not indexed.""" + with self._deployment_index_lock: + indices: Final = self.team_model_to_deployment_indices.get((team_id, public_model_name)) + return None if indices is None else tuple(self.model_list[idx] for idx in indices) + def get_deployment(self, model_id: str) -> Deployment | None: """ Returns -> Deployment or None @@ -11701,9 +11709,10 @@ class Router: Raise Exception -> if model found in invalid format """ # Use O(1) lookup via model_id_to_deployment_index_map only - if model_id in self.model_id_to_deployment_index_map: - idx: Final = self.model_id_to_deployment_index_map[model_id] - model: Final = self.model_list[idx] + with self._deployment_index_lock: + idx: Final = self.model_id_to_deployment_index_map.get(model_id) + model: Final = None if idx is None else self.model_list[idx] + if model is not None: if isinstance(model, dict): return Deployment(**model) elif isinstance(model, Deployment): @@ -11737,17 +11746,17 @@ class Router: Optimized with O(1) index lookup instead of O(n) linear scan. """ # O(1) lookup in model_name index - if model_group_name in self.model_name_to_deployment_indices: - indices: Final = self.model_name_to_deployment_indices[model_group_name] - if indices: - # Return first deployment for this model_name - model: Final = self.model_list[indices[0]] - if isinstance(model, dict): - return Deployment(**model) - elif isinstance(model, Deployment): - return model - else: - raise Exception(f"Model Name invalid - {type(model)}") + with self._deployment_index_lock: + indices: Final = self.model_name_to_deployment_indices.get(model_group_name) + # Return first deployment for this model_name + model: Final = self.model_list[indices[0]] if indices else None + if model is not None: + if isinstance(model, dict): + return Deployment(**model) + elif isinstance(model, Deployment): + return model + else: + raise Exception(f"Model Name invalid - {type(model)}") return None @staticmethod @@ -11770,11 +11779,12 @@ class Router: by other teams so a shared model name never resolves another team's credentials. """ - indices: Final = self.model_name_to_deployment_indices.get(model_group_name) or () - usable: Final = ( - self.model_list[idx] for idx in indices if self.deployment_usable_by_team(self.model_list[idx], team_id) - ) - first_usable: Final = next(usable, None) + with self._deployment_index_lock: + indices: Final = self.model_name_to_deployment_indices.get(model_group_name) or () + usable: Final = ( + self.model_list[idx] for idx in indices if self.deployment_usable_by_team(self.model_list[idx], team_id) + ) + first_usable: Final = next(usable, None) if first_usable is None: return None return Deployment(**first_usable) if isinstance(first_usable, dict) else first_usable @@ -11877,11 +11887,10 @@ class Router: this never triggers pattern matching or deep copies, so it is safe to call per listed model on the /v1/models hot path. """ - indices: Final = self.model_name_to_deployment_indices.get(model_name) - if not indices: + deployments: Final = self.deployments_named(model_name) + if not deployments: return None - deployments: Final = tuple(self.model_list[index] for index in indices) model_infos: Final = tuple( MappingProxyType( { @@ -12054,10 +12063,11 @@ class Router: def _get_team_public_name_deployment(self, model_id: str, team_id: str | None) -> Deployment | None: if team_id is None: return None - team_indices: Final = self.team_model_to_deployment_indices.get((team_id, model_id)) - if not team_indices: + with self._deployment_index_lock: + team_indices: Final = self.team_model_to_deployment_indices.get((team_id, model_id)) + team_model: Final = self.model_list[team_indices[0]] if team_indices else None + if team_model is None: return None - team_model: Final = self.model_list[team_indices[0]] return Deployment(**team_model) if isinstance(team_model, dict) else team_model def _get_wildcard_deployment_usable_by_team(self, model_id: str, team_id: str | None) -> Deployment | None: @@ -12296,10 +12306,9 @@ class Router: Optimized with O(1) index lookup instead of O(n) linear scan. """ # O(1) lookup via model_id_to_deployment_index_map - if id in self.model_id_to_deployment_index_map: - idx: Final = self.model_id_to_deployment_index_map[id] - return self.model_list[idx] - return None + with self._deployment_index_lock: + idx: Final = self.model_id_to_deployment_index_map.get(id) + return None if idx is None else self.model_list[idx] def get_model_group(self, id: str) -> list | None: """ @@ -12969,20 +12978,19 @@ class Router: if model_name is not None: # O(1) lookup in model_name index, then O(k) iteration where k = deployments for this model_name - if model_name in self.model_name_to_deployment_indices: - indices: Final = self.model_name_to_deployment_indices[model_name] - for idx in indices: - model = self.model_list[idx] - if "model_info" in model and "id" in model["model_info"]: - if exclude_team_models and model["model_info"].get("team_id"): - continue - ids.append(model["model_info"]["id"]) + for model in self.deployments_named(model_name) or (): + if "model_info" in model and "id" in model["model_info"]: + if exclude_team_models and model["model_info"].get("team_id"): + continue + ids.append(model["model_info"]["id"]) else: # When model_name is None, return all model IDs # Use the index map keys for O(n) where n = total deployments - for model_id in self.model_id_to_deployment_index_map: - idx = self.model_id_to_deployment_index_map[model_id] - model = self.model_list[idx] + with self._deployment_index_lock: + id_indices: Final = tuple(self.model_id_to_deployment_index_map.items()) + models: Final = tuple(self.model_list) + for model_id, idx in id_indices: + model = models[idx] if "model_info" in model and "id" in model["model_info"]: if exclude_team_models and model["model_info"].get("team_id"): continue @@ -13260,7 +13268,9 @@ class Router: # handled downstream by the pattern_router in _common_checks_available_deployment. return None - def should_include_deployment(self, model_name: str, model: dict, team_id: str | None = None) -> bool: + def should_include_deployment( + self, model_name: str, model: DeploymentTypedDict | dict, team_id: str | None = None + ) -> bool: """ Get the team-specific model name if team_id matches the deployment. """ @@ -13308,12 +13318,10 @@ class Router: # O(1) lookup in team_model index when team_id is provided if team_id is not None: - key: Final = (team_id, model_name) - if key in self.team_model_to_deployment_indices: - indices = self.team_model_to_deployment_indices[key] + team_models: Final = self._team_deployments_named(team_id, model_name) + if team_models is not None: # O(k) where k = team deployments for this model_name (typically 1-10) - for idx in indices: - model = self.model_list[idx] + for model in team_models: if not self.should_include_deployment(model_name=model_name, model=model, team_id=team_id): continue if model_alias is not None: @@ -13326,12 +13334,10 @@ class Router: return returned_models # O(1) lookup in model_name index - if model_name in self.model_name_to_deployment_indices: - indices = self.model_name_to_deployment_indices[model_name] - + named_models: Final = self.deployments_named(model_name) + if named_models is not None: # O(k) where k = deployments for this model_name (typically 1-10) - for idx in indices: - model = self.model_list[idx] + for model in named_models: if self.should_include_deployment(model_name=model_name, model=model, team_id=team_id): if model_alias is not None: # Optimized: Use shallow copy since we only modify top-level model_name @@ -13345,7 +13351,7 @@ class Router: # Fallback: if team_id is provided and model_name not in index, # check if model_name matches any team_public_model_name # O(n) scan but only when team_id lookup fails - for idx, model in enumerate(self.model_list): + for _idx, model in enumerate(self.model_list): if self.should_include_deployment(model_name=model_name, model=model, team_id=team_id): if model_alias is not None: # Optimized: Use shallow copy since we only modify top-level model_name @@ -14610,12 +14616,13 @@ class Router: def _team_deployments_across_teams(self, model: str) -> list[DeploymentTypedDict]: """Every team's deployments under public name `model`, for a proxy admin calling without a team.""" - team_deployments: Final = [ - self.model_list[index] - for (_, public_model_name), indices in self.team_model_to_deployment_indices.items() - if public_model_name == model - for index in indices - ] + with self._deployment_index_lock: + team_deployments: Final = [ + self.model_list[index] + for (_, public_model_name), indices in self.team_model_to_deployment_indices.items() + if public_model_name == model + for index in indices + ] team_ids: Final = { team_id for deployment in team_deployments diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index a475358e1f0..cc592bbcbc5 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -3126,17 +3126,14 @@ class ComplexityRouter(CustomLogger): model_to_prefs: Final[dict[str, AdaptiveRouterPreferences]] = {} model_to_cost: Final[dict[str, float]] = {} - model_list: Final = getattr(self.litellm_router_instance, "model_list", None) or [] - name_to_indices: Final = getattr(self.litellm_router_instance, "model_name_to_deployment_indices", {}) or {} for name in available_models: - indices = name_to_indices.get(name, []) - if not indices: + deployments = self.litellm_router_instance.deployments_named(name) + if not deployments: model_to_prefs[name] = AdaptiveRouterPreferences(quality_tier=2, strengths=[]) model_to_cost[name] = 0.0 continue - deployment = model_list[indices[0]] - mi = deployment.get("model_info") if isinstance(deployment, dict) else deployment.model_info - mi_dict: dict[str, Any] = mi if isinstance(mi, dict) else (mi.model_dump() if mi else {}) + deployment = deployments[0] + mi_dict: dict[str, Any] = deployment.get("model_info") or {} prefs_raw = mi_dict.get("adaptive_router_preferences") if prefs_raw is not None: model_to_prefs[name] = AdaptiveRouterPreferences(**prefs_raw) @@ -3144,9 +3141,7 @@ class ComplexityRouter(CustomLogger): model_to_prefs[name] = AdaptiveRouterPreferences(quality_tier=2, strengths=[]) # model_info is the conventional pricing location elsewhere in LiteLLM; litellm_params wins if set. - lp = deployment.get("litellm_params") if isinstance(deployment, dict) else deployment.litellm_params - lp_dict: dict[str, Any] = lp if isinstance(lp, dict) else (lp.model_dump() if lp else {}) - cost = lp_dict.get("input_cost_per_token") + cost = deployment.get("litellm_params", {}).get("input_cost_per_token") if cost is None: cost = mi_dict.get("input_cost_per_token") model_to_cost[name] = float(cost) if cost is not None else 0.0 diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py index b5b80ddae6d..add0f78fd3a 100644 --- a/litellm/router_strategy/savings_baseline.py +++ b/litellm/router_strategy/savings_baseline.py @@ -14,7 +14,7 @@ bare string with no provider beside them; an operator who writes ``deepseek-r1`` Azure would otherwise be priced against whoever else owns that name. """ -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Final, NamedTuple from litellm._logging import verbose_router_logger @@ -80,12 +80,11 @@ def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]: straight at a provider model rather than at a configured group, and prices under its own name because there is no deployment to override it. """ - indices: Final = router.model_name_to_deployment_indices.get(group_name) - if not indices: + deployments: Final = router.deployments_named(group_name) + if not deployments: return (Baseline(qualified),) if (qualified := canonical_model(group_name)) else () - def candidate(index: int) -> Baseline | None: - deployment: Final = router.model_list[index] + def candidate(deployment: Mapping[str, object]) -> Baseline | None: params: Final = deployment.get("litellm_params") if not isinstance(params, dict): return None @@ -98,7 +97,7 @@ def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]: deployment_id: Final = info.get("id") if isinstance(info, dict) else None return Baseline(qualified, str(deployment_id) if deployment_id else None) - return tuple(c for index in indices if (c := candidate(index)) is not None) + return tuple(c for deployment in deployments if (c := candidate(deployment)) is not None) def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | None: diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index b5e36055354..853b10ad97b 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -171,6 +171,7 @@ ignored_function_names = [ "_deployment_field", # Read by is_batch_capable, tested through the mixed-group tests in test_router.py "_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py "_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py + "_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py ] diff --git a/tests/integration/sdk/test_router_concurrent_deployment_delete_sdk.py b/tests/integration/sdk/test_router_concurrent_deployment_delete_sdk.py new file mode 100644 index 00000000000..dc94c96da05 --- /dev/null +++ b/tests/integration/sdk/test_router_concurrent_deployment_delete_sdk.py @@ -0,0 +1,474 @@ +from __future__ import annotations + +import asyncio +import json +import linecache +import sys +import threading +import uuid +from collections.abc import AsyncIterable, Awaitable, Callable, Iterable +from concurrent.futures import ThreadPoolExecutor +from types import FrameType +from typing import Final, TypeVar + +import pytest +from integration._support.client import object_value +from integration._support.openai_wire import chat_reply, openai_error, responses_reply +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +import litellm +from litellm import Router + +_DEPLOYMENTS: Final = 4 +_TARGET: Final = _DEPLOYMENTS - 1 +_SYNC_SECONDS: Final = 10 +_RATE_LIMITED_KEY: Final = "key-rate-limited" +_BURST_THREADS: Final = 8 +_BURST_CALLS: Final = 24 +_CHURN_UPSERTS: Final = 120 +_UNREPORTED_MODEL_ID: Final = frozenset({"aresponses-stream", "aanthropic-messages"}) +_ROUTER_FILE: Final = Router.__init__.__code__.co_filename +_INDEX_LOCK_LINES: Final = frozenset( + number + for number, line in enumerate(linecache.getlines(_ROUTER_FILE), start=1) + if "with self._deployment_index_lock" in line +) + +T = TypeVar("T") + + +def _peer(request: Request) -> Reply: + if request.headers["authorization"] == f"Bearer {_RATE_LIMITED_KEY}": + return openai_error(429) + body: Final = object_value(json.loads(request.body)) + stream: Final = body.get("stream") is True + if request.target == "/v1/responses": + return responses_reply(f"resp_{uuid.uuid4().hex}", "gpt-4o-mini", "ok", stream=stream) + return chat_reply(f"chatcmpl-{uuid.uuid4().hex}", "gpt-4o-mini", "ok", stream=stream) + + +def _deployment( + index: int, + api_base: str, + team: bool, + *, + model: int | None = None, + public_name: str | None = None, + timeout: float = 10, +) -> dict[str, JsonValue]: + team_info: Final[dict[str, JsonValue]] = ( + {"team_id": "team-1", "team_public_model_name": public_name or f"public-{index}"} if team else {} + ) + return { + "model_name": f"model-{index if model is None else model}", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": api_base, + "api_key": f"key-{index}", + "timeout": timeout, + }, + "model_info": {"id": f"deployment-{index}", **team_info}, + } + + +def _router(wire: Wire, team: bool = False) -> Router: + return Router(model_list=[_deployment(index, f"{wire.url}/v1", team) for index in range(_DEPLOYMENTS)]) + + +def _read_during_held_write(read: Callable[[], T], write: Callable[[], object]) -> T: + """Run `read` on its own thread while `write` waits where the list has shrunk but the index maps have not caught + up. The write resumes once `read` returns or reaches the router's index lock, so a read that skips the lock meets + the half-updated list and a read that takes the lock waits for the write""" + reindex: Final = Router._update_deployment_indices_after_removal.__code__ + held: Final = threading.Event() + release: Final = threading.Event() + settled: Final = threading.Event() + released_in_time: Final[list[bool]] = [] + outcome: Final[list[T]] = [] + failure: Final[list[BaseException]] = [] + + def hold_at_reindex(frame: FrameType, event: str, _arg: object) -> None: + if event == "call" and frame.f_code is reindex and not held.is_set(): + held.set() + released_in_time.append(release.wait(timeout=_SYNC_SECONDS)) + + def run_write() -> None: + previous: Final = sys.getprofile() + sys.setprofile(hold_at_reindex) + try: + write() + finally: + sys.setprofile(previous) + + def watch_read(frame: FrameType, event: str, _arg: object) -> Callable[..., object] | None: + if frame.f_code.co_filename != _ROUTER_FILE: + return None + if event == "line" and frame.f_lineno in _INDEX_LOCK_LINES: + settled.set() + return watch_read + + def run_read() -> None: + previous: Final = sys.gettrace() + sys.settrace(watch_read) + try: + outcome.append(read()) + except BaseException as error: + failure.append(error) + finally: + sys.settrace(previous) + settled.set() + + writer: Final = threading.Thread(target=run_write) + reader: Final = threading.Thread(target=run_read) + writer.start() + try: + assert held.wait(timeout=_SYNC_SECONDS), "the write never reached re-indexing" + reader.start() + assert settled.wait(timeout=_SYNC_SECONDS), "the read neither returned nor reached the index lock" + finally: + release.set() + if reader.ident is not None: + reader.join() + writer.join() + assert released_in_time == [True], "the write was not held for the whole read" + if failure: + raise failure[0] + return outcome[0] + + +def _read_during_held_delete(router: Router, deployment_id: str, read: Callable[[], T]) -> T: + return _read_during_held_write(read, lambda: router.delete_deployment(deployment_id)) + + +def _keys(wire: Wire) -> tuple[str, ...]: + return tuple(request.headers["authorization"] for request in wire.drain()) + + +def _team_metadata(team: bool) -> dict[str, JsonValue]: + return {"user_api_key_team_id": "team-1"} if team else {} + + +def _completion(router: Router, model: str, team: bool) -> str | None: + response: Final = router.completion( + model=model, messages=[{"role": "user", "content": "hi"}], metadata=_team_metadata(team) + ) + return response._hidden_params["model_id"] + + +def _completion_stream(router: Router, model: str, team: bool) -> str | None: + stream: Final = router.completion( + model=model, messages=[{"role": "user", "content": "hi"}], metadata=_team_metadata(team), stream=True + ) + assert isinstance(stream, Iterable), stream + chunks: Final = list(stream) + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "ok", chunks + return stream._hidden_params["model_id"] + + +def _acompletion(router: Router, model: str, team: bool) -> str | None: + async def call() -> str | None: + response: Final = await router.acompletion( + model=model, messages=[{"role": "user", "content": "hi"}], metadata=_team_metadata(team) + ) + return response._hidden_params["model_id"] + + return asyncio.run(call()) + + +def _acompletion_stream(router: Router, model: str, team: bool) -> str | None: + async def call() -> str | None: + stream: Final = await router.acompletion( + model=model, messages=[{"role": "user", "content": "hi"}], metadata=_team_metadata(team), stream=True + ) + assert isinstance(stream, AsyncIterable), stream + chunks: Final = [chunk async for chunk in stream] + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "ok", chunks + return stream._hidden_params["model_id"] + + return asyncio.run(call()) + + +def _responses(router: Router, model: str, team: bool) -> str | None: + response: Final = router.responses(model=model, input="hi", metadata=_team_metadata(team)) + assert response.output_text == "ok", response + return response._hidden_params["model_id"] + + +def _aresponses(router: Router, model: str, team: bool) -> str | None: + async def call() -> str | None: + response: Final = await router.aresponses(model=model, input="hi", metadata=_team_metadata(team)) + assert response.output_text == "ok", response + return response._hidden_params["model_id"] + + return asyncio.run(call()) + + +def _aresponses_stream(router: Router, model: str, team: bool) -> str | None: + async def call() -> None: + stream: Final = await router.aresponses(model=model, input="hi", metadata=_team_metadata(team), stream=True) + assert isinstance(stream, AsyncIterable), stream + events: Final = [event.type async for event in stream] + assert events[-1] == "response.completed", events + + asyncio.run(call()) + return None + + +def _aanthropic_messages(router: Router, model: str, team: bool) -> str | None: + async def call() -> None: + response: Final = await router.aanthropic_messages( + model=model, messages=[{"role": "user", "content": "hi"}], max_tokens=16, metadata=_team_metadata(team) + ) + assert response["content"][0]["text"] == "ok", response + + asyncio.run(call()) + return None + + +_CALLS: Final[dict[str, Callable[[Router, str, bool], str | None]]] = { + "completion": _completion, + "completion-stream": _completion_stream, + "acompletion": _acompletion, + "acompletion-stream": _acompletion_stream, + "responses": _responses, + "aresponses": _aresponses, + "aresponses-stream": _aresponses_stream, + "aanthropic-messages": _aanthropic_messages, +} + + +@pytest.mark.parametrize("call", list(_CALLS)) +@pytest.mark.parametrize("team", [False, True], ids=["model-name", "team-public-name"]) +def test_call_reaches_its_deployment_while_another_thread_deletes_one(call: str, team: bool) -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire, team) + served_by: Final = _read_during_held_delete( + router, + "deployment-0", + lambda: _CALLS[call](router, f"public-{_TARGET}" if team else f"model-{_TARGET}", team), + ) + keys: Final = _keys(wire) + + assert served_by == (None if call in _UNREPORTED_MODEL_ID else f"deployment-{_TARGET}") + assert keys == (f"Bearer key-{_TARGET}",) + assert router.get_model_ids() == [f"deployment-{index}" for index in range(1, _DEPLOYMENTS)] + + +@pytest.mark.parametrize("deleted", [0, _TARGET - 1], ids=["first", "next-to-target"]) +def test_completion_reaches_its_deployment_whichever_earlier_deployment_is_deleted(deleted: int) -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire) + served_by: Final = _read_during_held_delete( + router, f"deployment-{deleted}", lambda: _completion(router, f"model-{_TARGET}", team=False) + ) + keys: Final = _keys(wire) + + assert served_by == f"deployment-{_TARGET}" + assert keys == (f"Bearer key-{_TARGET}",) + + +def test_team_completion_reaches_the_remaining_deployment_of_a_shared_public_name_while_its_sibling_is_deleted() -> ( + None +): + with wire_server(_peer) as wire: + api_base: Final = f"{wire.url}/v1" + router: Final = Router( + model_list=[ + _deployment(0, api_base, team=False), + _deployment(1, api_base, team=False), + _deployment(2, api_base, team=True, public_name="public-shared"), + _deployment(3, api_base, team=True, public_name="public-shared"), + ] + ) + served_by: Final = _read_during_held_delete( + router, "deployment-2", lambda: _completion(router, "public-shared", team=True) + ) + keys: Final = _keys(wire) + + assert served_by == "deployment-3" + assert keys == ("Bearer key-3",) + + +def test_completion_for_the_deployment_being_deleted_fails_cleanly_and_sends_nothing() -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire) + with pytest.raises(litellm.BadRequestError): + _read_during_held_delete( + router, + "deployment-0", + lambda: router.completion(model="model-0", messages=[{"role": "user", "content": "hi"}]), + ) + keys: Final = _keys(wire) + + assert keys == () + assert router.get_model_ids() == [f"deployment-{index}" for index in range(1, _DEPLOYMENTS)] + + +def test_provider_error_during_a_concurrent_delete_comes_from_the_requested_deployment() -> None: + with wire_server(_peer) as wire: + api_base: Final = f"{wire.url}/v1" + rate_limited: Final = _deployment(_TARGET, api_base, team=False) + rate_limited["litellm_params"] = {**object_value(rate_limited["litellm_params"]), "api_key": _RATE_LIMITED_KEY} + router: Final = Router( + model_list=[*(_deployment(index, api_base, team=False) for index in range(_TARGET)), rate_limited], + num_retries=0, + ) + with pytest.raises(litellm.RateLimitError): + _read_during_held_delete( + router, + "deployment-0", + lambda: router.completion(model=f"model-{_TARGET}", messages=[{"role": "user", "content": "hi"}]), + ) + served_by: Final = _completion(router, "model-1", team=False) + keys: Final = _keys(wire) + + assert served_by == "deployment-1" + assert keys == (f"Bearer {_RATE_LIMITED_KEY}", "Bearer key-1") + + +def test_two_threads_deleting_the_same_deployment_remove_it_once() -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire) + second: Final = _read_during_held_delete( + router, "deployment-0", lambda: router.delete_deployment("deployment-0") + ) + served_by: Final = _completion(router, f"model-{_TARGET}", team=False) + keys: Final = _keys(wire) + + assert second is None + assert served_by == f"deployment-{_TARGET}" + assert keys == (f"Bearer key-{_TARGET}",) + assert router.get_model_ids() == [f"deployment-{index}" for index in range(1, _DEPLOYMENTS)] + + +def test_completion_reaches_its_deployment_while_another_thread_upserts_an_earlier_one() -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire) + changed: Final = litellm.Deployment(**_deployment(0, f"{wire.url}/v1", team=False, timeout=11)) + served_by: Final = _read_during_held_write( + lambda: _completion(router, f"model-{_TARGET}", team=False), lambda: router.upsert_deployment(changed) + ) + keys: Final = _keys(wire) + + assert served_by == f"deployment-{_TARGET}" + assert keys == (f"Bearer key-{_TARGET}",) + assert sorted(router.get_model_ids()) == [f"deployment-{index}" for index in range(_DEPLOYMENTS)] + + +def test_deployment_added_during_a_concurrent_delete_serves_its_own_requests() -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire) + added: Final = litellm.Deployment(**_deployment(9, f"{wire.url}/v1", team=False)) + _read_during_held_delete(router, "deployment-0", lambda: router.add_deployment(added)) + served: Final = tuple(_completion(router, f"model-{index}", team=False) for index in (9, _TARGET)) + keys: Final = _keys(wire) + + assert served == ("deployment-9", f"deployment-{_TARGET}") + assert keys == ("Bearer key-9", f"Bearer key-{_TARGET}") + + +def test_model_list_set_during_a_concurrent_delete_serves_its_own_requests() -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire) + replacement: Final = [_deployment(index, f"{wire.url}/v1", team=False) for index in range(10, 14)] + _read_during_held_delete(router, "deployment-0", lambda: router.set_model_list(replacement)) + served: Final = tuple(_completion(router, f"model-{index}", team=False) for index in range(10, 14)) + keys: Final = _keys(wire) + + assert served == tuple(f"deployment-{index}" for index in range(10, 14)) + assert keys == tuple(f"Bearer key-{index}" for index in range(10, 14)) + assert router.get_model_ids() == [f"deployment-{index}" for index in range(10, 14)] + + +def _burst_router(api_base: str) -> Router: + """Two deployments per model name, so an upsert of one leaves its model group served.""" + return Router( + model_list=[ + _deployment(index, api_base, team=False, model=index % _DEPLOYMENTS) for index in range(2 * _DEPLOYMENTS) + ] + ) + + +def _churn(router: Router, api_base: str, stop: threading.Event) -> int: + """Upsert the deployments in turn with a changed timeout; each upsert moves one to the end of the list.""" + upserts = 0 + while upserts < _CHURN_UPSERTS and not stop.is_set(): + index, round_number = upserts % (2 * _DEPLOYMENTS), upserts // (2 * _DEPLOYMENTS) + router.upsert_deployment( + litellm.Deployment( + **_deployment(index, api_base, team=False, model=index % _DEPLOYMENTS, timeout=11 + round_number % 2) + ) + ) + upserts += 1 + return upserts + + +def _served_keys(wire: Wire) -> dict[str, str]: + """Which key reached the upstream for each request marker.""" + return { + object_value(json.loads(request.body))["messages"][0]["content"]: request.headers["authorization"] + for request in wire.drain() + } + + +def _assert_each_request_used_its_models_deployment(served: list[tuple[str, str | None]], keys: dict[str, str]) -> None: + for marker, model_id in served: + model: Final = int(marker.split(":")[0]) + assert model_id in (f"deployment-{model}", f"deployment-{model + _DEPLOYMENTS}"), (marker, model_id) + assert keys[marker] == f"Bearer key-{str(model_id).removeprefix('deployment-')}", (marker, keys[marker]) + assert len(keys) == len(served) + + +def test_thread_burst_during_upsert_churn_sends_every_request_with_its_deployments_key() -> None: + with wire_server(_peer) as wire: + api_base: Final = f"{wire.url}/v1" + router: Final = _burst_router(api_base) + stop: Final = threading.Event() + + def call(number: int) -> tuple[str, str | None]: + marker: Final = f"{number % _DEPLOYMENTS}:{number}" + response: Final = router.completion( + model=f"model-{number % _DEPLOYMENTS}", messages=[{"role": "user", "content": marker}] + ) + return marker, response._hidden_params["model_id"] + + with ThreadPoolExecutor(max_workers=_BURST_THREADS + 1) as pool: + churn: Final = pool.submit(_churn, router, api_base, stop) + served: Final = list(pool.map(call, range(_BURST_THREADS * _BURST_CALLS))) + stop.set() + upserts: Final = churn.result() + keys: Final = _served_keys(wire) + + assert upserts > 0 + _assert_each_request_used_its_models_deployment(served, keys) + assert sorted(router.get_model_ids()) == sorted(f"deployment-{index}" for index in range(2 * _DEPLOYMENTS)) + + +async def _gathered(calls: Iterable[Awaitable[tuple[str, str | None]]]) -> list[tuple[str, str | None]]: + return list(await asyncio.gather(*calls)) + + +def test_async_burst_during_upsert_churn_sends_every_request_with_its_deployments_key() -> None: + with wire_server(_peer) as wire: + api_base: Final = f"{wire.url}/v1" + router: Final = _burst_router(api_base) + stop: Final = threading.Event() + + async def call(number: int) -> tuple[str, str | None]: + marker: Final = f"{number % _DEPLOYMENTS}:{number}" + response: Final = await router.acompletion( + model=f"model-{number % _DEPLOYMENTS}", messages=[{"role": "user", "content": marker}] + ) + return marker, response._hidden_params["model_id"] + + with ThreadPoolExecutor(max_workers=1) as pool: + churn: Final = pool.submit(_churn, router, api_base, stop) + served: Final = asyncio.run(_gathered(call(number) for number in range(_BURST_THREADS * _BURST_CALLS))) + stop.set() + upserts: Final = churn.result() + keys: Final = _served_keys(wire) + + assert upserts > 0 + _assert_each_request_used_its_models_deployment(served, keys) + assert sorted(router.get_model_ids()) == sorted(f"deployment-{index}" for index in range(2 * _DEPLOYMENTS)) diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 979ab499f3b..63e38ce533a 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -1011,10 +1011,9 @@ class TestPollingConditionChecks: # Create mock router mock_router = Mock() - mock_router.model_name_to_deployment_indices = {"gpt-5": [0]} - mock_router.model_list = [ - {"model_name": "gpt-5", "litellm_params": {"model": "openai/gpt-5"}} - ] + mock_router.deployments_named.side_effect = { + "gpt-5": ({"model_name": "gpt-5", "litellm_params": {"model": "openai/gpt-5"}},) + }.get result = should_use_polling_for_request( background_mode=True, @@ -1033,13 +1032,14 @@ class TestPollingConditionChecks: ) mock_router = Mock() - mock_router.model_name_to_deployment_indices = {"claude-3": [0]} - mock_router.model_list = [ - { - "model_name": "claude-3", - "litellm_params": {"model": "anthropic/claude-3-sonnet"}, - } - ] + mock_router.deployments_named.side_effect = { + "claude-3": ( + { + "model_name": "claude-3", + "litellm_params": {"model": "anthropic/claude-3-sonnet"}, + }, + ) + }.get result = should_use_polling_for_request( background_mode=True, @@ -1183,13 +1183,14 @@ class TestPollingConditionChecks: ) mock_router = Mock() - mock_router.model_name_to_deployment_indices = {"deep-research": [0]} - mock_router.model_list = [ - { - "model_name": "deep-research", - "litellm_params": {"model": "openai/o4-mini-deep-research"}, - } - ] + mock_router.deployments_named.side_effect = { + "deep-research": ( + { + "model_name": "deep-research", + "litellm_params": {"model": "openai/o4-mini-deep-research"}, + }, + ) + }.get # Model alias "deep-research" is in native_background_mode result = should_use_polling_for_request( diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index 9b4019bead6..a46f415e10a 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -112,6 +112,13 @@ def _heuristic_v2_artifact() -> TrainedTierArtifact: ) +def _answer_deployments_named(router: MagicMock) -> None: + """Make a mock router answer `deployments_named` from its own `model_list`, the way Router answers from its index.""" + router.deployments_named.side_effect = lambda name: ( + tuple(deployment for deployment in router.model_list if deployment["model_name"] == name) or None + ) + + @pytest.fixture def mock_router_instance(): """Create a mock LiteLLM Router instance.""" @@ -3181,7 +3188,7 @@ class TestCapabilityClassifier: {"model_name": "efficient-model", "litellm_params": {"input_cost_per_token": 0.000001}}, {"model_name": "capable-model", "litellm_params": {"input_cost_per_token": 0.00001}}, ] - mock_router_instance.model_name_to_deployment_indices = {"efficient-model": [0], "capable-model": [1]} + _answer_deployments_named(mock_router_instance) router: Final = self._router( mock_router_instance, adaptive=True, @@ -5670,7 +5677,7 @@ class TestAdaptiveSoftFloors: "model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}}, }, ] - router.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]} + _answer_deployments_named(router) return router @pytest.fixture @@ -6829,7 +6836,7 @@ class TestTierModelAffinity: ) -> ComplexityRouter: mock_router_instance.cache = DualCache() mock_router_instance.model_list = [] - mock_router_instance.model_name_to_deployment_indices = {} + _answer_deployments_named(mock_router_instance) return ComplexityRouter( model_name="affinity-router", litellm_router_instance=mock_router_instance, @@ -7547,7 +7554,7 @@ class TestSessionAffinity: "model_info": {}, }, ] - mock_router_instance.model_name_to_deployment_indices = {"cheap": [0]} + _answer_deployments_named(mock_router_instance) router = ComplexityRouter( model_name="hybrid", litellm_router_instance=mock_router_instance, @@ -11351,7 +11358,7 @@ class TestClientHousekeepingCalls: "model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}}, }, ] - adaptive_instance.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]} + _answer_deployments_named(adaptive_instance) router = ComplexityRouter( model_name="hybrid", litellm_router_instance=adaptive_instance, @@ -13442,7 +13449,7 @@ class TestPlanModeTierFloor: "model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}}, }, ] - adaptive_instance.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]} + _answer_deployments_named(adaptive_instance) router = ComplexityRouter( model_name="hybrid", litellm_router_instance=adaptive_instance, @@ -14941,7 +14948,7 @@ class TestModalityRouting: elif path == "adaptive_pick_rewritten": config["adaptive"] = True mock_router_instance.model_list = [] - mock_router_instance.model_name_to_deployment_indices = {} + _answer_deployments_named(mock_router_instance) router = self._router(mock_router_instance, config, vision) result = await router.async_pre_routing_hook(model="m", request_kwargs=request_kwargs, messages=messages) assert result.model == expected_model diff --git a/tests/unit/router_strategy/test_savings_baseline.py b/tests/unit/router_strategy/test_savings_baseline.py index 5efc73d2dcb..4241a541202 100644 --- a/tests/unit/router_strategy/test_savings_baseline.py +++ b/tests/unit/router_strategy/test_savings_baseline.py @@ -107,8 +107,7 @@ class TestResolveBaseline: served; a dashboard counterfactual must not be able to take routing down.""" class Exploding: - @property - def model_name_to_deployment_indices(self): + def deployments_named(self, model_name): raise RuntimeError("router is mid-reload") assert resolve_baseline(Exploding(), ["anything"]) is None diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 568a05e8ff4..4814d36ee39 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -5,6 +5,7 @@ import functools import gc import itertools import json +import linecache import logging import os import sys @@ -13,8 +14,8 @@ import warnings from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Iterator, Mapping from datetime import datetime, timedelta, timezone from pathlib import Path -from types import MappingProxyType, SimpleNamespace -from typing import TYPE_CHECKING, Final, Literal, cast +from types import CodeType, FrameType, MappingProxyType, SimpleNamespace +from typing import TYPE_CHECKING, Final, Literal, TypeVar, cast from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx @@ -60,7 +61,10 @@ from litellm.router import ( _without_line_breaks, Span, ) +from litellm.proxy.response_polling.polling_handler import should_use_polling_for_request from litellm.router_strategy import simple_shuffle +from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter +from litellm.router_strategy.savings_baseline import Baseline, _models_in from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit from litellm.router_utils.cooldown_handlers import ( _async_get_cooldown_deployments, @@ -30004,6 +30008,319 @@ async def test_responses_fallback_wrapper_disconnect_billing_keeps_the_primary_s assert wrapped.partial_response_on_disconnect().response.usage.output_tokens == primary_tokens + fallback_tokens +_SYNC_SECONDS: Final = 10 +_ROUTER_FILE: Final = Router.__init__.__code__.co_filename +_INDEX_LOCK_LINES: Final = frozenset( + number + for number, line in enumerate(linecache.getlines(_ROUTER_FILE), start=1) + if "with self._deployment_index_lock" in line +) + +_Read = TypeVar("_Read") + + +def _concurrent_delete_deployment(index: int) -> dict[str, object]: + return { + "model_name": f"model-{index}", + "litellm_params": {"model": f"openai/gpt-{index}", "api_key": "k"}, + "model_info": { + "id": f"deployment-{index}", + "team_id": "team-1", + "team_public_model_name": "public-shared" if index >= 2 else f"public-{index}", + }, + } + + +def _concurrent_delete_router() -> Router: + return Router(model_list=[_concurrent_delete_deployment(index) for index in range(4)]) + + +def _read_during_held_write( + read: Callable[[], _Read], + write: Callable[[], object], + event: Literal["call", "c_call"] = "call", + write_after: CodeType | None = None, +) -> _Read: + """Run `read` on its own thread while `write` waits at its first `event` in the re-index, after the list shrank and + before the maps caught up. With `write_after`, the write starts when `read` returns from that function. The write + resumes once `read` returns or reaches the router's index lock, so a read that skips the lock meets the half-updated + list and a read that takes the lock waits for the write""" + reindex: Final = Router._update_deployment_indices_after_removal.__code__ + held: Final = threading.Event() + release: Final = threading.Event() + settled: Final = threading.Event() + released_in_time: Final[list[bool]] = [] + outcome: Final[list[_Read]] = [] + failure: Final[list[BaseException]] = [] + + def hold_in_reindex(frame: FrameType, observed: str, _arg: object) -> None: + if observed == event and frame.f_code is reindex and not held.is_set(): + held.set() + released_in_time.append(release.wait(timeout=_SYNC_SECONDS)) + + def run_write() -> None: + previous: Final = sys.getprofile() + sys.setprofile(hold_in_reindex) + try: + write() + finally: + sys.setprofile(previous) + + writer: Final = threading.Thread(target=run_write) + + def start_write() -> None: + writer.start() + assert held.wait(timeout=_SYNC_SECONDS), "the write never reached re-indexing" + + def watch_read(frame: FrameType, observed: str, _arg: object) -> Callable[..., object] | None: + if frame.f_code.co_filename != _ROUTER_FILE: + return None + if observed == "return" and frame.f_code is write_after and writer.ident is None: + start_write() + elif observed == "line" and held.is_set() and frame.f_lineno in _INDEX_LOCK_LINES: + settled.set() + return watch_read + + def run_read() -> None: + previous: Final = sys.gettrace() + sys.settrace(watch_read) + try: + outcome.append(read()) + except BaseException as error: + failure.append(error) + finally: + sys.settrace(previous) + settled.set() + + reader: Final = threading.Thread(target=run_read) + try: + if write_after is None: + start_write() + reader.start() + assert settled.wait(timeout=_SYNC_SECONDS), "the read neither returned nor reached the index lock" + finally: + release.set() + if reader.ident is not None: + reader.join() + if writer.ident is not None: + writer.join() + assert released_in_time == [True], "the write was not held for the whole read" + if failure: + raise failure[0] + return outcome[0] + + +def _deployment_ids(deployments: list[DeploymentTypedDict]) -> tuple[str, ...]: + return tuple(deployment["model_info"]["id"] for deployment in deployments) + + +@pytest.mark.parametrize( + ("read", "expected"), + [ + pytest.param( + lambda router: router.get_deployment("deployment-3").model_info.id, "deployment-3", id="get_deployment" + ), + pytest.param( + lambda router: router.get_model_info("deployment-3")["model_info"]["id"], + "deployment-3", + id="get_model_info", + ), + pytest.param( + lambda router: _deployment_ids(router.get_all_deployments("model-3")), + ("deployment-3",), + id="get_all_deployments", + ), + pytest.param( + lambda router: _deployment_ids(router.get_all_deployments("public-shared", team_id="team-1")), + ("deployment-2", "deployment-3"), + id="get_all_deployments-team", + ), + pytest.param( + lambda router: router.map_team_model("public-shared", "team-1"), "public-shared", id="map_team_model" + ), + pytest.param(lambda router: router.get_model_ids("model-3"), ["deployment-3"], id="get_model_ids-by-name"), + pytest.param( + lambda router: router.get_model_ids(), ["deployment-1", "deployment-2", "deployment-3"], id="get_model_ids" + ), + pytest.param( + lambda router: router.get_deployment_by_model_group_name("model-3").model_info.id, + "deployment-3", + id="get_deployment_by_model_group_name", + ), + pytest.param( + lambda router: router.get_model_listing_info("model-3").cost_map_keys, + ("openai/gpt-3",), + id="get_model_listing_info", + ), + pytest.param( + lambda router: router.team_model_has_alternatives("deployment-3"), True, id="team_model_has_alternatives" + ), + pytest.param( + lambda router: ( + router._get_model_group_deployment_usable_by_team( + model_group_name="model-3", team_id="team-1" + ).model_info.id + ), + "deployment-3", + id="model_group_deployment_usable_by_team", + ), + pytest.param( + lambda router: ( + router._get_team_public_name_deployment(model_id="public-shared", team_id="team-1").model_info.id + ), + "deployment-2", + id="team_public_name_deployment", + ), + pytest.param( + lambda router: _deployment_ids(router._team_deployments_across_teams("public-shared")), + ("deployment-2", "deployment-3"), + id="team_deployments_across_teams", + ), + pytest.param( + lambda router: _deployment_ids(list(router.deployments_named("model-3") or ())), + ("deployment-3",), + id="deployments_named", + ), + pytest.param( + lambda router: should_use_polling_for_request( + background_mode=True, + polling_via_cache_enabled=["openai"], + redis_cache=MagicMock(), + model="model-3", + llm_router=router, + ), + True, + id="response_polling_provider_lookup", + ), + pytest.param( + lambda router: _models_in(router, "model-3"), + (Baseline("openai/gpt-3", "deployment-3"),), + id="savings_baseline_candidates", + ), + ], +) +def test_lookup_during_a_concurrent_delete_returns_the_requested_deployment( + read: Callable[[Router], object], expected: object +) -> None: + router: Final = _concurrent_delete_router() + + observed: Final = _read_during_held_write(lambda: read(router), lambda: router.delete_deployment("deployment-0")) + + assert observed == expected + + +def test_lookup_during_an_upsert_returns_the_requested_deployment() -> None: + router: Final = _concurrent_delete_router() + rotated: Final = Deployment( + **{**_concurrent_delete_deployment(0), "litellm_params": {"model": "openai/gpt-0", "api_key": "rotated"}} + ) + + observed: Final = _read_during_held_write( + lambda: router.get_deployment("deployment-3").model_info.id, lambda: router.upsert_deployment(rotated) + ) + + assert observed == "deployment-3" + + +def test_add_during_a_concurrent_delete_indexes_the_added_deployment() -> None: + router: Final = _concurrent_delete_router() + + _read_during_held_write( + lambda: router.add_deployment(Deployment(**_concurrent_delete_deployment(4))), + lambda: router.delete_deployment("deployment-0"), + ) + + assert [router.get_deployment(f"deployment-{index}").model_info.id for index in range(1, 5)] == [ + f"deployment-{index}" for index in range(1, 5) + ] + + +def test_set_model_list_during_a_concurrent_delete_indexes_the_new_list() -> None: + router: Final = _concurrent_delete_router() + + _read_during_held_write( + lambda: router.set_model_list([_concurrent_delete_deployment(index) for index in range(10, 14)]), + lambda: router.delete_deployment("deployment-0"), + event="c_call", + ) + + assert router.get_model_ids() == [f"deployment-{index}" for index in range(10, 14)] + + +def test_team_model_has_alternatives_when_a_delete_starts_after_its_deployment_lookup() -> None: + router: Final = _concurrent_delete_router() + + has_alternatives: Final = _read_during_held_write( + lambda: router.team_model_has_alternatives("deployment-3"), + lambda: router.delete_deployment("deployment-0"), + write_after=Router.get_deployment.__code__, + ) + + assert has_alternatives is True + + +def test_init_adaptive_router_deployment_during_a_concurrent_delete_prices_the_listed_model() -> None: + priced: Final = { + **_concurrent_delete_deployment(3), + "litellm_params": {"model": "openai/gpt-3", "api_key": "k", "input_cost_per_token": 3e-06}, + } + router: Final = Router(model_list=[*(_concurrent_delete_deployment(index) for index in range(3)), priced]) + adaptive: Final = Deployment( + model_name="adaptive", + litellm_params=LiteLLM_Params( + model="auto_router/adaptive_router", adaptive_router_config={"available_models": ["model-3"]} + ), + model_info=ModelInfo(id="adaptive"), + ) + + _read_during_held_write( + lambda: router.init_adaptive_router_deployment(deployment=adaptive), + lambda: router.delete_deployment("deployment-0"), + ) + + assert router.adaptive_routers["adaptive"][0].strategy.model_to_cost == {"model-3": 3e-06} + + +def test_complexity_router_adaptive_pool_during_a_concurrent_delete_prices_the_listed_model() -> None: + priced: Final = { + **_concurrent_delete_deployment(3), + "litellm_params": {"model": "openai/gpt-3", "api_key": "k", "input_cost_per_token": 3e-06}, + } + router: Final = Router(model_list=[*(_concurrent_delete_deployment(index) for index in range(3)), priced]) + complexity: Final = ComplexityRouter( + model_name="hybrid", + litellm_router_instance=router, + complexity_router_config={"adaptive": True, "tiers": {"SIMPLE": ["model-3"]}}, + ) + + adaptive: Final = _read_during_held_write( + complexity._ensure_adaptive_router, lambda: router.delete_deployment("deployment-0") + ) + + assert adaptive is not None + assert adaptive.model_to_cost == {"model-3": 3e-06} + + +def test_delete_of_an_unknown_deployment_returns_none_and_keeps_the_list() -> None: + router: Final = _concurrent_delete_router() + + assert router.delete_deployment("deployment-9") is None + assert router.get_model_ids() == [f"deployment-{index}" for index in range(4)] + + +def test_get_model_ids_by_name_skips_team_deployments_only_when_asked() -> None: + shared: Final = {"model_name": "shared", "litellm_params": {"model": "openai/gpt-shared", "api_key": "k"}} + router: Final = Router( + model_list=[ + {**shared, "model_info": {"id": "global"}}, + {**shared, "model_info": {"id": "team", "team_id": "team-1", "team_public_model_name": "public"}}, + ] + ) + + assert router.get_model_ids("shared", exclude_team_models=True) == ["global"] + assert router.get_model_ids("shared") == ["global", "team"] + + _IMAGE_PART = {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}} _IMAGE_CHAT = [{"role": "user", "content": [{"type": "text", "text": "What is in this image?"}, _IMAGE_PART]}] _IMAGE_RESPONSES_INPUT = [ From 4da24fa01e2248a44c3a62e2022b43f5a38794b8 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Mon, 26 Oct 2026 15:42:17 -0700 Subject: [PATCH 077/155] fix(ui): show search tool errors instead of empty results (#48478) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../_components/SearchToolTester.test.tsx | 32 +++++++++++++++++-- .../_components/SearchToolTester.tsx | 2 +- .../src/components/networking.test.ts | 26 +++++++++++++++ .../src/components/networking.tsx | 2 +- 4 files changed, 57 insertions(+), 5 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/SearchToolTester.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/SearchToolTester.test.tsx index 54be24d2401..08a0ab7d02b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/SearchToolTester.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/SearchToolTester.test.tsx @@ -1,4 +1,4 @@ -import { render, screen, waitFor } from "@testing-library/react"; +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { SearchToolTester } from "./SearchToolTester"; @@ -271,7 +271,15 @@ describe("SearchToolTester", () => { }); it("should handle search errors and show notification", async () => { - const error = new Error("Search failed"); + const error = new Error( + JSON.stringify({ + error: { + message: "User not allowed to access search tool: exa-search-2", + type: "user_search_tool_access_denied", + code: "403", + }, + }), + ); vi.mocked(networking.searchToolQueryCall).mockRejectedValue(error); const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {}); const user = userEvent.setup(); @@ -281,11 +289,29 @@ describe("SearchToolTester", () => { const searchButton = screen.getByRole("button", { name: /search/i }); await user.click(searchButton); await waitFor(() => { - expect(toast.fromError).toHaveBeenCalledWith("Failed to query search tool"); + expect(toast.fromError).toHaveBeenCalledWith(error); }); consoleSpy.mockRestore(); }); + it("should not show empty results or add history when search fails", async () => { + vi.mocked(networking.searchToolQueryCall).mockRejectedValue(new Error("Search failed")); + const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const user = userEvent.setup(); + render(); + const input = screen.getByPlaceholderText("Enter your search query..."); + fireEvent.change(input, { target: { value: "test query" } }); + await user.click(screen.getByRole("button", { name: /search/i })); + + await waitFor(() => { + expect(toast.fromError).toHaveBeenCalled(); + }); + expect(screen.queryByText("No results found")).not.toBeInTheDocument(); + expect(screen.queryByText("test query")).not.toBeInTheDocument(); + expect(screen.getByText("Test your search tool")).toBeInTheDocument(); + consoleSpy.mockRestore(); + }); + it("should maintain search history", async () => { const user = userEvent.setup(); render(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/SearchToolTester.tsx b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/SearchToolTester.tsx index 7dcef3fc6ba..646120443c7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/SearchToolTester.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/SearchToolTester.tsx @@ -60,7 +60,7 @@ export const SearchToolTester: React.FC = ({ searchToolNa setSearchHistory((prev) => [historyEntry, ...prev]); } catch (error) { console.error("Error querying search tool:", error); - toast.fromError("Failed to query search tool"); + toast.fromError(error); } finally { setIsLoading(false); } diff --git a/ui/litellm-dashboard/src/components/networking.test.ts b/ui/litellm-dashboard/src/components/networking.test.ts index e7a55870345..8b0f0389772 100644 --- a/ui/litellm-dashboard/src/components/networking.test.ts +++ b/ui/litellm-dashboard/src/components/networking.test.ts @@ -1178,3 +1178,29 @@ describe("getUiConfig runtime backend discovery", () => { }); }); }); + +describe("searchToolQueryCall", () => { + const originalFetch = global.fetch; + + afterEach(() => { + global.fetch = originalFetch; + }); + + it("rejects with the backend error message for a denied search", async () => { + const errorMessage = "User not allowed to access search tool: exa-search-2"; + global.fetch = vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + error: { + message: errorMessage, + type: "user_search_tool_access_denied", + code: "403", + }, + }), + { status: 403 }, + ), + ); + + await expect(Networking.searchToolQueryCall("token", "exa-search-2", "litellm")).rejects.toThrow(errorMessage); + }); +}); diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index fd25242ec34..9aade717566 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -6860,7 +6860,7 @@ export const searchToolQueryCall = async ( if (!response.ok) { const errorData = await response.text(); await handleError(errorData); - return null; + throw new Error(errorData); } const data = await response.json(); From 2e27a02a9fce9bdc6ece13ba3cb8222dc5b6ecfa Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:44:05 -0700 Subject: [PATCH 078/155] feat(chatgpt): forward client subscriptions through the gateway (#41522) * feat(chatgpt): forward each Codex user's ChatGPT OAuth login through the proxy Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): cover _model_group_llm_provider metadata resolution Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(chatgpt): keep the gateway ChatGPT login bound to the authenticator api_base Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(chatgpt): move client credential header selection into the chatgpt provider Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(chatgpt): cover credential recognition and stored account id fallbacks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(chatgpt): support per-model chatgpt_auth_file for multiple ChatGPT accounts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(chatgpt): resolve bare chatgpt_auth_file paths and drop redundant test docstring Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(chatgpt): pass per-model auth file to early provider detection Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(chatgpt): normalize chatgpt_auth_file before caching the authenticator Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(chatgpt): type account selection parameters * fix(chatgpt): preserve account selection through saved prompts * fix(chatgpt): allow administrator account configuration through the proxy * test(chatgpt): construct immutable request parameters * fix(router): preserve typed deployment credential resolution * fix(chatgpt): allow admins to probe unsaved account configurations * fix(chatgpt): retain SDK login headers during provider resolution * fix(chatgpt): allow admins to probe unsaved account configurations * fix(chatgpt): normalize subscription health probe input * fix(chatgpt): normalize health probe input and cover credential destinations * test(chatgpt): keep destination rejection assertions focused * feat(proxy): forward client ChatGPT subscriptions on native Codex routes * docs: lead ChatGPT forwarding with standard model setup * fix(chatgpt): preserve selected credentials in health provider lookup * fix(chatgpt): preserve gateway credentials and workspace selection * refactor(chatgpt): avoid rebinding streaming metadata inputs * fix(chatgpt): keep account lookup token argument immutable * fix(chatgpt): scope forwarded credential routing to ChatGPT * refactor: keep forwarded credential isolation opt-in by provider --------- Co-authored-by: Moe Khalil Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Urbanonymous Co-authored-by: mateo --- .../chatgpt_subscription_passthrough.md | 76 +++ .../litellm_core_utils/get_litellm_params.py | 1 + .../get_llm_provider_logic.py | 12 +- .../get_provider_specific_headers.py | 13 + litellm/llms/chatgpt/authenticator.py | 85 +-- litellm/llms/chatgpt/chat/transformation.py | 83 ++- litellm/llms/chatgpt/common_utils.py | 169 ++++- .../llms/chatgpt/responses/transformation.py | 38 +- litellm/llms/custom_httpx/llm_http_handler.py | 48 +- litellm/main.py | 13 +- litellm/proxy/_lazy_features.py | 5 + litellm/proxy/_lazy_openapi_snapshot.json | 178 +++++ litellm/proxy/_types.py | 1 + litellm/proxy/auth/auth_utils.py | 4 +- .../health_endpoints/_health_endpoints.py | 10 +- litellm/proxy/litellm_pre_call_utils.py | 15 +- .../codex_passthrough_endpoints.py | 87 +++ .../openai_passthrough_logging_handler.py | 9 +- .../pass_through_endpoints.py | 15 +- litellm/proxy/spend_tracking/savings.py | 5 +- litellm/responses/main.py | 53 +- litellm/responses/utils.py | 20 + litellm/router.py | 62 +- litellm/types/litellm_params.py | 1 + litellm/types/router.py | 4 + litellm/types/utils.py | 1 + litellm/utils.py | 6 +- .../test_get_llm_provider_logic.py | 49 ++ .../chat/test_chatgpt_chat_transformation.py | 638 ++++++++++++++++++ .../test_chatgpt_responses_transformation.py | 165 +++++ .../chatgpt/test_chatgpt_authenticator.py | 148 ++++ .../llms/chatgpt/test_chatgpt_common_utils.py | 90 +++ .../custom_httpx/test_llm_http_handler.py | 41 ++ tests/unit/proxy/auth/test_auth_utils.py | 36 + .../health_endpoints/test_health_endpoints.py | 61 ++ .../test_model_management_endpoints.py | 24 +- .../test_codex_passthrough_endpoints.py | 185 +++++ .../unit/proxy/test_litellm_pre_call_utils.py | 92 +++ tests/unit/responses/test_responses_utils.py | 36 + tests/unit/test_router/test_router.py | 9 + tests/unit/types/test_litellm_params.py | 1 + tests/unit/types/test_router.py | 3 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 105 +++ 43 files changed, 2547 insertions(+), 150 deletions(-) create mode 100644 cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md create mode 100644 litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py create mode 100644 tests/unit/llms/chatgpt/chat/test_chatgpt_chat_transformation.py create mode 100644 tests/unit/llms/chatgpt/test_chatgpt_common_utils.py create mode 100644 tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py diff --git a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md new file mode 100644 index 00000000000..c79632e9195 --- /dev/null +++ b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md @@ -0,0 +1,76 @@ +# Use your ChatGPT subscription through LiteLLM + +Sign into ChatGPT in Codex or Pi and send model requests through your existing LiteLLM proxy. The client keeps your login and refreshes it. Each request uses that client's ChatGPT account, while a separate LiteLLM virtual key controls gateway access and usage attribution. + +An administrator adds a `chatgpt/` model using the usual model configuration. No ChatGPT login needs to be added to the proxy. This uses the same separate-key convention as provider passthrough, including Claude Code subscription forwarding. + +## Add a model to the proxy + +Reuse your existing proxy. If you need one, follow the [proxy quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start). Merge this entry into its model list and restart the proxy: + +```yaml +model_list: + - model_name: gpt-6-luna + litellm_params: + model: chatgpt/gpt-6-luna + model_info: + mode: responses +``` + +Use a model available to the caller's subscription. The virtual key must allow the exposed model name. Leave saved ChatGPT logins and API credentials unset for a client-only deployment. Explicit cross-provider fallbacks can incur separate API charges; do not configure them for subscription-only traffic. + +## Configure Codex + +Use your existing LiteLLM URL and a virtual key assigned to your gateway user. Export that key as `LITELLM_API_KEY` in the environment that launches Codex. Keep the value out of committed configuration. + +Add this provider to `~/.codex/config.toml`, preserving your other settings: + +```toml +model = "gpt-6-luna" +model_provider = "litellm_subscription" + +[model_providers.litellm_subscription] +name = "LiteLLM subscription" +base_url = "https://litellm.example.com/v1" +wire_api = "responses" +supports_websockets = false +requires_openai_auth = true +env_http_headers = { "x-litellm-api-key" = "LITELLM_API_KEY" } +``` + +Replace `https://litellm.example.com` with your proxy URL. A local proxy normally uses `http://localhost:4000/v1`. Launch Codex, sign into ChatGPT if needed, and send a prompt with a model available to your subscription. The selected model must match the name exposed by the gateway. + +For a managed workspace rollout, use OpenAI's [gateway configuration guide](https://learn.chatgpt.com/docs/enterprise/sign-in-with-chatgpt-through-a-gateway). + +## Native compatibility and Pi + +For a native Codex model catalog and request format, change Codex's `base_url` to `https://litellm.example.com/backend-api/codex`. This route uses the same credentials but does not need a model deployment or resolve model aliases. Restricted keys must allow the upstream model name. + +### Configure Pi + +Sign into the built-in `openai-codex` provider in Pi. Merge this override into `~/.pi/agent/models.json` and export your LiteLLM virtual key as `LITELLM_API_KEY` before launching Pi: + +```json +{ + "providers": { + "openai-codex": { + "baseUrl": "https://litellm.example.com/backend-api/codex", + "headers": { + "x-litellm-api-key": "LITELLM_API_KEY" + } + } + } +} +``` + +Keep Pi's subscription login as the provider credential. The header value names an environment variable, not a key to paste into the file. + +## Verify account usage + +Send a prompt and check the completed request in LiteLLM's Logs page. Its gateway user/key and token counts should match the caller. Give each person their own LiteLLM virtual key for separate attribution. The gateway credential and ChatGPT login are independently authenticated; LiteLLM does not establish that both identities belong to the same person. + +The native routes are `GET /backend-api/codex/models` and `POST /backend-api/codex/responses`. Model requests use the upstream model name, so a restricted LiteLLM key must allow that name. OpenAI's model catalog is returned in its native format; seeing a model there does not override gateway permissions. + +The proxy forwards the ChatGPT bearer and account header to the fixed OpenAI Codex endpoint. It removes gateway credentials and cookies from the upstream request and redacts credential headers in logging callbacks. The client remains responsible for refresh and reauthentication. Native passthrough returns authentication and quota errors without switching subscriptions. Standard model routes retain the caller login across ChatGPT retries; explicitly configured cross-provider fallbacks use their own credentials. An explicit workspace header is preserved for OpenAI to validate. + +This flow does not connect accounts through the dashboard or pool subscription allowances. Use a Codex build that supports ChatGPT authentication with a custom model provider. The integration forwards HTTP and SSE requests; it does not add native WebSocket or other Codex backend routes. diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 85557090ae9..ca8e2af18e6 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -76,6 +76,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "itpm", "otpm", "use_xai_oauth", + "chatgpt_auth_file", "github_copilot_auth_type", "github_copilot_host", "github_copilot_user_session", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index de51ecd0b21..19c1d5c414c 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -158,6 +158,14 @@ def declared_authenticating_provider(model: str | None, custom_llm_provider: str return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None +def resolve_model_and_provider_for_metadata(model: str, custom_llm_provider: str | None = None) -> tuple[str, str]: + declared: Final = declared_authenticating_provider(model, custom_llm_provider) + if declared is not None: + return model.removeprefix(f"{declared}/"), declared + resolved_model, resolved_provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + return resolved_model, resolved_provider + + def inferred_provider(model: str | None) -> str | None: if not model: return None @@ -842,7 +850,9 @@ def _get_openai_compatible_provider_info( api_base, dynamic_api_key, custom_llm_provider, - ) = litellm.ChatGPTConfig().get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider) + ) = litellm.ChatGPTConfig().get_openai_compatible_provider_info( + model, api_base, api_key, custom_llm_provider, litellm_params + ) elif custom_llm_provider == "novita": api_base = api_base or get_secret("NOVITA_API_BASE") or "https://api.novita.ai/v3/openai" dynamic_api_key = api_key or get_secret_str("NOVITA_API_KEY") diff --git a/litellm/litellm_core_utils/get_provider_specific_headers.py b/litellm/litellm_core_utils/get_provider_specific_headers.py index bc325c3c702..62dfcb073e5 100644 --- a/litellm/litellm_core_utils/get_provider_specific_headers.py +++ b/litellm/litellm_core_utils/get_provider_specific_headers.py @@ -1,10 +1,23 @@ from collections.abc import Mapping, Sequence from typing import Final +from pydantic import TypeAdapter + from litellm.types.utils import ProviderSpecificHeader +_PROVIDER_HEADERS: Final[TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None]] = TypeAdapter( + ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None +) + class ProviderSpecificHeaderUtils: + @staticmethod + def forwarded_authorization(provider_specific_header: object, custom_llm_provider: str | None) -> str | None: + headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers( + _PROVIDER_HEADERS.validate_python(provider_specific_header), custom_llm_provider + ) + return next((value for key, value in headers.items() if key.lower() == "authorization"), None) + @staticmethod def get_provider_specific_headers( provider_specific_header: ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None, diff --git a/litellm/llms/chatgpt/authenticator.py b/litellm/llms/chatgpt/authenticator.py index ff9c6476cba..a5fb1c9c6b9 100644 --- a/litellm/llms/chatgpt/authenticator.py +++ b/litellm/llms/chatgpt/authenticator.py @@ -1,9 +1,9 @@ -import base64 import json import os import time from collections.abc import Mapping -from typing import Final, TypeAlias +from functools import lru_cache +from typing import TYPE_CHECKING, Final import httpx from pydantic import JsonValue, TypeAdapter, ValidationError @@ -24,19 +24,21 @@ from .common_utils import ( CHATGPT_OAUTH_TOKEN_URL, GetAccessTokenError, GetDeviceCodeError, + JsonObject, RefreshAccessTokenError, + decode_jwt_claims, + extract_chatgpt_account_id, ) +if TYPE_CHECKING: + from litellm.types.router import GenericLiteLLMParams + TOKEN_EXPIRY_SKEW_SECONDS: Final = 60 TOKEN_REFRESH_TIMEOUT_SECONDS: Final = 30 DEVICE_CODE_TIMEOUT_SECONDS: Final = 15 * 60 DEVICE_CODE_COOLDOWN_SECONDS: Final = 5 * 60 DEVICE_CODE_POLL_SLEEP_SECONDS: Final = 5 -OPENAI_AUTH_CLAIM_KEY: Final = "https://api.openai.com/auth" - -JsonObject: TypeAlias = Mapping[str, JsonValue] - _JSON_OBJECT_ADAPTER: Final = TypeAdapter(JsonObject) @@ -44,6 +46,26 @@ def _optional_str(value: JsonValue | None) -> str | None: return value if isinstance(value, str) else None +def get_chatgpt_auth_file( + litellm_params: "Mapping[str, object] | GenericLiteLLMParams | None", +) -> str | None: + if litellm_params is None: + return None + value: Final = ( + litellm_params.get("chatgpt_auth_file") + if isinstance(litellm_params, Mapping) + else litellm_params.chatgpt_auth_file + ) + if not isinstance(value, str) or not value: + return None + return os.path.abspath(os.path.expanduser(value)) + + +@lru_cache(maxsize=128) +def get_cached_authenticator(auth_file: str) -> "Authenticator": + return Authenticator(auth_file=auth_file) + + def _token_refresh_timeout() -> httpx.Timeout: configured: Final = get_configured_request_timeout() seconds: Final = ( @@ -53,13 +75,13 @@ def _token_refresh_timeout() -> httpx.Timeout: class Authenticator: - def __init__(self) -> None: - self.token_dir = os.getenv( - "CHATGPT_TOKEN_DIR", - os.path.expanduser("~/.config/litellm/chatgpt"), + def __init__(self, auth_file: str | None = None) -> None: + default_auth_file: Final = os.path.join( + os.getenv("CHATGPT_TOKEN_DIR", os.path.expanduser("~/.config/litellm/chatgpt")), + os.getenv("CHATGPT_AUTH_FILE", "auth.json"), ) - self.auth_file = os.path.join(self.token_dir, os.getenv("CHATGPT_AUTH_FILE", "auth.json")) - self._ensure_token_dir() + self.auth_file = os.path.abspath(os.path.expanduser(auth_file or default_auth_file)) + self.token_dir = os.path.dirname(self.auth_file) def get_api_base(self) -> str: return os.getenv("CHATGPT_API_BASE") or os.getenv("OPENAI_CHATGPT_API_BASE") or CHATGPT_API_BASE @@ -99,16 +121,18 @@ class Authenticator: tokens: Final = self._login_device_code() return tokens["access_token"] - def get_account_id(self) -> str | None: + def get_account_id(self, access_token: str | None = None) -> str | None: auth_data: Final = self._read_auth_file() + if access_token and (not auth_data or auth_data.get("access_token") != access_token): + return extract_chatgpt_account_id(access_token) if not auth_data: return None account_id: Final = _optional_str(auth_data.get("account_id")) if account_id: return account_id id_token: Final = auth_data.get("id_token") - access_token: Final = auth_data.get("access_token") - derived: Final = self._extract_account_id(_optional_str(id_token or access_token)) + saved_access_token: Final = auth_data.get("access_token") + derived: Final = extract_chatgpt_account_id(_optional_str(id_token or saved_access_token)) if derived: self._write_auth_file({**auth_data, "account_id": derived}) return derived @@ -129,6 +153,7 @@ class Authenticator: def _write_auth_file(self, data: JsonObject) -> None: try: + self._ensure_token_dir() with open(self.auth_file, "w") as f: json.dump(data, f) except OSError as exc: @@ -146,34 +171,11 @@ class Authenticator: return time.time() >= float(derived_expires_at) - TOKEN_EXPIRY_SKEW_SECONDS def _get_expires_at(self, token: str) -> int | None: - claims: Final = self._decode_jwt_claims(token) - exp: Final = claims.get("exp") + exp: Final = decode_jwt_claims(token).get("exp") if isinstance(exp, (int, float)): return int(exp) return None - def _decode_jwt_claims(self, token: str) -> JsonObject: - try: - parts: Final = token.split(".") - if len(parts) < 2: - return {} - payload_b64: Final = parts[1] + "=" * (-len(parts[1]) % 4) - payload_bytes: Final = base64.urlsafe_b64decode(payload_b64) - return _JSON_OBJECT_ADAPTER.validate_python(json.loads(payload_bytes.decode("utf-8"))) - except Exception: - return {} - - def _extract_account_id(self, token: str | None) -> str | None: - if not token: - return None - claims: Final = self._decode_jwt_claims(token) - auth_claims: Final = claims.get(OPENAI_AUTH_CLAIM_KEY) - if isinstance(auth_claims, dict): - account_id: Final = auth_claims.get("chatgpt_account_id") - if isinstance(account_id, str) and account_id: - return account_id - return None - def _login_device_code(self) -> dict[str, str]: cooldown_remaining: Final = self._get_device_code_cooldown_remaining(self._read_auth_file()) if cooldown_remaining > 0: @@ -361,15 +363,16 @@ class Authenticator: "refresh_token": _optional_str(data.get("refresh_token")) or refresh_token, "id_token": id_token, } + account_id: Final = self.get_account_id() auth_data: Final = self._build_auth_record(refreshed) - self._write_auth_file(auth_data) + self._write_auth_file({**auth_data, "account_id": account_id} if account_id else auth_data) return refreshed def _build_auth_record(self, tokens: dict[str, str]) -> JsonObject: access_token: Final = tokens.get("access_token") id_token: Final = tokens.get("id_token") expires_at: Final = self._get_expires_at(access_token) if access_token else None - account_id: Final = self._extract_account_id(id_token or access_token) + account_id: Final = extract_chatgpt_account_id(id_token or access_token) return { "access_token": access_token, "refresh_token": tokens.get("refresh_token"), diff --git a/litellm/llms/chatgpt/chat/transformation.py b/litellm/llms/chatgpt/chat/transformation.py index a1aac7f315b..9005185931f 100644 --- a/litellm/llms/chatgpt/chat/transformation.py +++ b/litellm/llms/chatgpt/chat/transformation.py @@ -1,14 +1,20 @@ +from collections.abc import Mapping from typing import Any, Final from litellm.exceptions import AuthenticationError from litellm.llms.openai.openai import OpenAIConfig from litellm.types.llms.openai import AllMessageValues +from litellm.types.router import GenericLiteLLMParams -from ..authenticator import Authenticator +from ..authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file from ..common_utils import ( GetAccessTokenError, + chatgpt_client_api_base, ensure_chatgpt_session_id, + get_chatgpt_client_credential, get_chatgpt_default_headers, + get_chatgpt_request_credential, + strip_chatgpt_client_credential_headers, ) from .streaming_utils import ChatGPTToolCallNormalizer @@ -19,9 +25,27 @@ class ChatGPTConfig(OpenAIConfig): api_key: str | None = None, api_base: str | None = None, custom_llm_provider: str = "openai", + authenticator: Authenticator | None = None, ) -> None: super().__init__() - self.authenticator = Authenticator() + self.authenticator = authenticator if authenticator is not None else Authenticator() + + def _resolve_authenticator( + self, litellm_params: Mapping[str, object] | GenericLiteLLMParams | None + ) -> Authenticator: + auth_file: Final = get_chatgpt_auth_file(litellm_params) + return get_cached_authenticator(auth_file) if auth_file else self.authenticator + + @staticmethod + def _get_access_token_or_raise(authenticator: Authenticator, model: str, llm_provider: str) -> str: + try: + return authenticator.get_access_token() + except GetAccessTokenError as e: + raise AuthenticationError( + model=model, + llm_provider=llm_provider, + message=str(e), + ) def api_base_without_login(self) -> str: return self.authenticator.get_api_base() @@ -32,16 +56,16 @@ class ChatGPTConfig(OpenAIConfig): api_base: str | None, api_key: str | None, custom_llm_provider: str, + litellm_params: Mapping[str, object] | GenericLiteLLMParams | None = None, ) -> tuple[str | None, str | None, str]: - dynamic_api_base: Final = self.api_base_without_login() - try: - dynamic_api_key: Final = self.authenticator.get_access_token() - except GetAccessTokenError as e: - raise AuthenticationError( - model=model, - llm_provider=custom_llm_provider, - message=str(e), - ) + client_credential: Final = get_chatgpt_request_credential(litellm_params) + if client_credential is not None: + return chatgpt_client_api_base(api_base), client_credential.access_token, custom_llm_provider + if api_key: + return api_base or self.api_base_without_login(), api_key, custom_llm_provider + authenticator: Final = self._resolve_authenticator(litellm_params) + dynamic_api_base: Final = authenticator.get_api_base() + dynamic_api_key: Final = self._get_access_token_or_raise(authenticator, model, custom_llm_provider) return dynamic_api_base, dynamic_api_key, custom_llm_provider def get_openai_compatible_provider_info( @@ -50,26 +74,49 @@ class ChatGPTConfig(OpenAIConfig): api_base: str | None, api_key: str | None, custom_llm_provider: str, + litellm_params: Mapping[str, object] | GenericLiteLLMParams | None = None, ) -> tuple[str | None, str | None, str]: - return self._get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider) + return self._get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider, litellm_params) def validate_environment( self, - headers: dict, + headers: dict[str, str], model: str, messages: list[AllMessageValues], optional_params: dict, - litellm_params: dict, + litellm_params: dict[str, object], api_key: str | None = None, api_base: str | None = None, - ) -> dict: + ) -> dict[str, str]: + client_credential: Final = get_chatgpt_client_credential(headers) + if client_credential is not None: + return dict( + get_chatgpt_default_headers( + client_credential.access_token, + client_credential.account_id, + ensure_chatgpt_session_id(litellm_params), + ), + **strip_chatgpt_client_credential_headers(headers), + ) + authenticator: Final = self._resolve_authenticator(litellm_params) + resolved_api_key: Final = ( + self._get_access_token_or_raise(authenticator, model, "chatgpt") + if get_chatgpt_auth_file(litellm_params) + else api_key + ) validated_headers: Final = super().validate_environment( - headers, model, messages, optional_params, litellm_params, api_key, api_base + dict(strip_chatgpt_client_credential_headers(headers)), + model, + messages, + optional_params, + litellm_params, + resolved_api_key, + api_base, ) - account_id: Final = self.authenticator.get_account_id() + account_id: Final = authenticator.get_account_id(resolved_api_key) session_id: Final = ensure_chatgpt_session_id(litellm_params) - default_headers: Final = get_chatgpt_default_headers(api_key or "", account_id, session_id) + default_headers: Final = get_chatgpt_default_headers(resolved_api_key or "", account_id, session_id) return {**default_headers, **validated_headers} def post_stream_processing(self, stream: Any) -> ChatGPTToolCallNormalizer: diff --git a/litellm/llms/chatgpt/common_utils.py b/litellm/llms/chatgpt/common_utils.py index fe33219f110..ec7fbfa92da 100644 --- a/litellm/llms/chatgpt/common_utils.py +++ b/litellm/llms/chatgpt/common_utils.py @@ -2,14 +2,32 @@ Constants and helpers for ChatGPT subscription OAuth. """ +import base64 +import json import os import platform -from typing import Any, Final +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, TypeAlias +from urllib.parse import urlsplit from uuid import uuid4 import httpx +from pydantic import JsonValue, TypeAdapter +from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.types.utils import ProviderSpecificHeader + +if TYPE_CHECKING: + from litellm.types.router import GenericLiteLLMParams + +_SCOPED_HEADERS: Final[TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None]] = TypeAdapter( + ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None +) +_STRING_HEADERS: Final = TypeAdapter(Mapping[str, str]) +_PARAMS: Final = TypeAdapter(Mapping[str, object]) # OAuth + API constants (derived from openai/codex) CHATGPT_AUTH_BASE: Final = "https://auth.openai.com" @@ -19,6 +37,12 @@ CHATGPT_OAUTH_TOKEN_URL: Final = f"{CHATGPT_AUTH_BASE}/oauth/token" CHATGPT_DEVICE_VERIFY_URL: Final = f"{CHATGPT_AUTH_BASE}/codex/device" CHATGPT_API_BASE: Final = "https://chatgpt.com/backend-api/codex" CHATGPT_CLIENT_ID: Final = "app_EMoamEEZ73f0CkXaXp7hrann" +CHATGPT_ACCOUNT_ID_HEADER: Final = "ChatGPT-Account-Id" +OPENAI_AUTH_CLAIM_KEY: Final = "https://api.openai.com/auth" + +JsonObject: TypeAlias = Mapping[str, JsonValue] +_JSON_OBJECT_ADAPTER: Final = TypeAdapter(JsonObject) +_EMPTY_CLAIMS: Final[JsonObject] = MappingProxyType({}) DEFAULT_ORIGINATOR: Final = "codex_cli_rs" DEFAULT_USER_AGENT: Final = "codex_cli_rs/0.0.0 (Unknown 0; unknown) unknown" @@ -229,7 +253,7 @@ def get_chatgpt_default_headers( access_token: str, account_id: str | None, session_id: str | None = None, -) -> dict: +) -> dict[str, str]: originator: Final = get_chatgpt_originator() user_agent: Final = get_chatgpt_user_agent(originator) headers: Final = { @@ -242,30 +266,133 @@ def get_chatgpt_default_headers( if session_id: headers["session_id"] = session_id if account_id: - headers["ChatGPT-Account-Id"] = account_id + headers[CHATGPT_ACCOUNT_ID_HEADER] = account_id return headers +def decode_jwt_claims(token: str) -> JsonObject: + try: + parts: Final = token.split(".") + if len(parts) < 2: + return _EMPTY_CLAIMS + payload_b64: Final = parts[1] + "=" * (-len(parts[1]) % 4) + payload_bytes: Final = base64.urlsafe_b64decode(payload_b64) + return _JSON_OBJECT_ADAPTER.validate_python(json.loads(payload_bytes.decode("utf-8"))) + except Exception: + return _EMPTY_CLAIMS + + +def extract_chatgpt_account_id(token: str | None) -> str | None: + if not token: + return None + auth_claims: Final = decode_jwt_claims(token).get(OPENAI_AUTH_CLAIM_KEY) + if not isinstance(auth_claims, dict): + return None + account_id: Final = auth_claims.get("chatgpt_account_id") + return account_id if isinstance(account_id, str) and account_id else None + + +def _bearer_token(value: str) -> str: + scheme, separator, token = value.strip().partition(" ") + if not separator: + return scheme + return token.strip() if scheme.lower() == "bearer" else "" + + +def is_chatgpt_oauth_key(value: str | None) -> bool: + if value is None: + return False + return OPENAI_AUTH_CLAIM_KEY in decode_jwt_claims(_bearer_token(value)) + + +@dataclass(frozen=True, slots=True) +class ChatGPTClientCredential: + access_token: str + account_id: str | None + + +def _header_value(headers: Mapping[str, str], name: str) -> str | None: + return next((value for key, value in headers.items() if key.lower() == name), None) + + +def get_chatgpt_client_credential(headers: Mapping[str, str]) -> ChatGPTClientCredential | None: + auth_header: Final = _header_value(headers, "authorization") + if auth_header is None or not is_chatgpt_oauth_key(auth_header): + return None + access_token: Final = _bearer_token(auth_header) + account_id: Final = _header_value(headers, CHATGPT_ACCOUNT_ID_HEADER.lower()) or extract_chatgpt_account_id( + access_token + ) + return ChatGPTClientCredential(access_token=access_token, account_id=account_id) + + +def select_chatgpt_client_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]: + if get_chatgpt_client_credential(headers) is None: + return MappingProxyType({}) + return MappingProxyType( + { + key: value + for key, value in headers.items() + if (key.lower() == "authorization" and is_chatgpt_oauth_key(value)) + or key.lower() == CHATGPT_ACCOUNT_ID_HEADER.lower() + } + ) + + +def strip_chatgpt_client_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]: + return MappingProxyType( + { + key: value + for key, value in headers.items() + if key.lower() not in ("authorization", CHATGPT_ACCOUNT_ID_HEADER.lower()) + } + ) + + +def get_chatgpt_request_credential( + params: "Mapping[str, object] | GenericLiteLLMParams | None", +) -> ChatGPTClientCredential | None: + normalized: Final = _normalize_litellm_params(params) + scoped: Final = normalized.get("provider_specific_header") + extra: Final = normalized.get("extra_headers") + headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header=_SCOPED_HEADERS.validate_python(scoped), custom_llm_provider="chatgpt" + ) + return get_chatgpt_client_credential(headers) or get_chatgpt_client_credential( + _STRING_HEADERS.validate_python(extra) if extra is not None else MappingProxyType({}) + ) + + +def chatgpt_client_api_base(api_base: str | None) -> str: + base: Final = api_base or CHATGPT_API_BASE + parsed: Final = urlsplit(base) + if ( + parsed.scheme != "https" + or parsed.hostname != "chatgpt.com" + or parsed.port not in (None, 443) + or parsed.username + or parsed.password + ): + raise ChatGPTAuthError( + status_code=400, message="Client ChatGPT credentials require the official HTTPS ChatGPT endpoint" + ) + return base + + def get_chatgpt_default_instructions() -> str: return os.getenv("CHATGPT_DEFAULT_INSTRUCTIONS") or CHATGPT_DEFAULT_INSTRUCTIONS -def _normalize_litellm_params(litellm_params: Any | None) -> dict: - if litellm_params is None: - return {} - if isinstance(litellm_params, dict): - return litellm_params - if hasattr(litellm_params, "model_dump"): - try: - return litellm_params.model_dump() - except Exception: - return {} - if hasattr(litellm_params, "dict"): - try: - return litellm_params.dict() - except Exception: - return {} - return {} +def _normalize_litellm_params(litellm_params: object) -> Mapping[str, object]: + if isinstance(litellm_params, Mapping): + return _PARAMS.validate_python(litellm_params) + serializer: Final = getattr(litellm_params, "model_dump", None) or getattr(litellm_params, "dict", None) + if not callable(serializer): + return MappingProxyType({}) + try: + return _PARAMS.validate_python(serializer()) + except Exception: + return MappingProxyType({}) def get_chatgpt_session_id(litellm_params: object) -> str | None: @@ -274,8 +401,8 @@ def get_chatgpt_session_id(litellm_params: object) -> str | None: value = params.get(key) if value: return str(value) - metadata: Final = params.get("metadata") - if isinstance(metadata, dict): + metadata: Final = _normalize_litellm_params(params.get("metadata")) + if metadata: value = metadata.get("session_id") if value: return str(value) diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index 34953a79c60..2352de35b5b 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -25,13 +25,15 @@ from litellm.types.llms.openai import ( from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders -from ..authenticator import Authenticator +from ..authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file from ..common_utils import ( CHATGPT_API_BASE, GetAccessTokenError, ensure_chatgpt_session_id, + get_chatgpt_client_credential, get_chatgpt_default_headers, get_chatgpt_default_instructions, + strip_chatgpt_client_credential_headers, ) if TYPE_CHECKING: @@ -49,14 +51,35 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): def custom_llm_provider(self) -> LlmProviders: return LlmProviders.CHATGPT + def _resolve_authenticator(self, litellm_params: GenericLiteLLMParams | None) -> Authenticator: + auth_file: Final = get_chatgpt_auth_file(litellm_params) + return get_cached_authenticator(auth_file) if auth_file else self.authenticator + def validate_environment( self, - headers: dict, + headers: dict[str, str], model: str, litellm_params: GenericLiteLLMParams | None, - ) -> dict: + ) -> dict[str, str]: + session_id: Final = ensure_chatgpt_session_id(litellm_params) + client_credential: Final = get_chatgpt_client_credential(headers) + if client_credential is not None: + return { + **get_chatgpt_default_headers(client_credential.access_token, client_credential.account_id, session_id), + **strip_chatgpt_client_credential_headers(headers), + } + authenticator: Final = self._resolve_authenticator(litellm_params) + safe_headers: Final = strip_chatgpt_client_credential_headers(headers) + configured_api_key: Final = litellm_params.api_key if litellm_params is not None else None + if configured_api_key and not get_chatgpt_auth_file(litellm_params): + return { + **get_chatgpt_default_headers( + configured_api_key, authenticator.get_account_id(configured_api_key), session_id + ), + **safe_headers, + } try: - access_token: Final = self.authenticator.get_access_token() + access_token: Final = authenticator.get_access_token() except GetAccessTokenError as e: raise AuthenticationError( model=model, @@ -64,10 +87,9 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): message=str(e), ) - account_id: Final = self.authenticator.get_account_id() - session_id: Final = ensure_chatgpt_session_id(litellm_params) + account_id: Final = authenticator.get_account_id() default_headers: Final = get_chatgpt_default_headers(access_token, account_id, session_id) - return {**default_headers, **headers} + return {**default_headers, **safe_headers} def transform_responses_api_request( self, @@ -79,7 +101,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): ) -> dict: request: Final = super().transform_responses_api_request( model, - input, + [{"role": "user", "content": input}] if isinstance(input, str) else input, response_api_optional_request_params, litellm_params, headers, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 1b491f1614d..7b808cf2be2 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -198,6 +198,26 @@ from litellm.utils import ( ) +def _merge_headers_case_insensitive( + headers: Mapping[str, object], extra_headers: Mapping[str, object] | None +) -> dict[str, str]: # mutable-ok: legacy HTTP handler methods require a dict of headers + overridden: Final = frozenset(name.lower() for name in extra_headers or {}) + return { + **{name: str(value) for name, value in headers.items() if name.lower() not in overridden}, + **{name: str(value) for name, value in (extra_headers or {}).items()}, + } + + +def _client_request_headers( + response_api_optional_request_params: Mapping[str, object], + extra_headers: Mapping[str, object] | None, +) -> dict[str, str]: # mutable-ok: provider validate_environment methods require a dict + from litellm.litellm_core_utils.core_helpers import as_str_mapping + + optional_headers: Final = as_str_mapping(response_api_optional_request_params.get("extra_headers")) + return _merge_headers_case_insensitive(optional_headers or {}, extra_headers) + + def _rust_responses_websocket_enabled( custom_llm_provider: str | None, ) -> bool: @@ -2906,14 +2926,18 @@ class BaseLLMHTTPHandler: else: sync_httpx_client = client - headers = responses_api_provider_config.validate_environment( - headers=response_api_optional_request_params.get("extra_headers", {}) or {}, - model=model, - litellm_params=litellm_params, - ) - - if extra_headers: - headers.update(extra_headers) + headers = { # rebind-ok: validation replaces the method's incoming headers + **responses_api_provider_config.validate_environment( + headers=( + _client_request_headers(response_api_optional_request_params, extra_headers) + if custom_llm_provider == "chatgpt" + else response_api_optional_request_params.get("extra_headers", {}) or {} + ), + model=model, + litellm_params=litellm_params, + ), + **((extra_headers or {}) if custom_llm_provider != "chatgpt" else {}), + } # Check if streaming is requested stream = response_api_optional_request_params.get("stream", False) @@ -3104,11 +3128,15 @@ class BaseLLMHTTPHandler: validated_headers: Final = { **await asyncio.to_thread( responses_api_provider_config.validate_environment, - headers=response_api_optional_request_params.get("extra_headers", {}) or {}, + headers=( + _client_request_headers(response_api_optional_request_params, extra_headers) + if custom_llm_provider == "chatgpt" + else response_api_optional_request_params.get("extra_headers", {}) or {} + ), model=model, litellm_params=litellm_params, ), - **(extra_headers or {}), + **((extra_headers or {}) if custom_llm_provider != "chatgpt" else {}), } # Check if streaming is requested diff --git a/litellm/main.py b/litellm/main.py index f30d93361e5..b3d62daa4d8 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -677,7 +677,11 @@ async def acompletion( } if custom_llm_provider is None: _supplemental: Final = cast( # cast-ok: kwargs values are request-scoped objects - "dict[str, object]", {k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS if k in kwargs} + "dict[str, object]", + { + **{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs}, + "extra_headers": extra_headers, + }, ) _, custom_llm_provider, _, _ = get_llm_provider( model=model, @@ -5678,7 +5682,10 @@ def completion( if deployment_id is not None: # azure llms model = deployment_id custom_llm_provider = "azure" - _supplemental_provider_params: Final = {k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS if k in kwargs} + _supplemental_provider_params: Final = { + **{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs}, + "extra_headers": extra_headers, + } model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, @@ -5998,6 +6005,7 @@ def completion( tpm=kwargs.get("tpm"), rpm=kwargs.get("rpm"), use_xai_oauth=kwargs.get("use_xai_oauth", False), + chatgpt_auth_file=routing_kwargs.get("chatgpt_auth_file"), gigachat_scope=kwargs.get("gigachat_scope"), gigachat_auth_url=kwargs.get("gigachat_auth_url"), gigachat_access_token=kwargs.get("gigachat_access_token"), @@ -9140,6 +9148,7 @@ async def ahealth_check( custom_llm_provider=custom_llm_provider_from_params, api_base=api_base_from_params, api_key=api_key_from_params, + litellm_params=GenericLiteLLMParams(**model_params), ) model_params["cache"] = {"no-cache": True} # don't used cached responses for making health check calls diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index aa67d36af4f..4e54d6ac733 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -207,6 +207,11 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( module_path="litellm.proxy.management_endpoints.config_override_endpoints", path_prefixes=("/config_overrides",), ), + LazyFeature( + name="codex_passthrough", + module_path="litellm.proxy.pass_through_endpoints.codex_passthrough_endpoints", + path_prefixes=("/backend-api/codex/",), + ), LazyFeature( name="llm_passthrough", module_path="litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 4e183bf9d5c..166072e9613 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -8673,6 +8673,184 @@ } } }, + "codex_passthrough": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ValidationError": { + "properties": { + "ctx": { + "title": "Context", + "type": "object" + }, + "input": { + "title": "Input" + }, + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/backend-api/codex/{endpoint}": { + "get": { + "description": "Forward a client's ChatGPT subscription with separate LiteLLM authentication.", + "operationId": "codex_passthrough_route_backend_api_codex__endpoint__get", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "enum": [ + "models", + "responses" + ], + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Codex Passthrough Route", + "tags": [ + "codex_passthrough" + ] + }, + "post": { + "description": "Forward a client's ChatGPT subscription with separate LiteLLM authentication.", + "operationId": "codex_passthrough_route_backend_api_codex__endpoint__post", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "enum": [ + "models", + "responses" + ], + "title": "Endpoint", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + }, + "text/event-stream": { + "schema": { + "type": "string" + } + } + }, + "description": "Unmodified Codex response or server-sent events" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Codex Passthrough Route", + "tags": [ + "codex_passthrough" + ] + } + } + } + }, "compliance": { "components": { "schemas": { diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index db433f07f3b..6280e2caf87 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -391,6 +391,7 @@ class LiteLLMRoutes(enum.Enum): ] mapped_pass_through_routes = [ + "/backend-api/codex", "/bedrock", "/comprehendmedical", "/azure_speech", diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 25f900a5f9f..30ede935cfa 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -736,7 +736,9 @@ def is_request_body_safe( def _reject_request_credential_params(litellm_params: Mapping[str, object], *, manages_deployments: bool) -> None: if manages_deployments: - reject_server_owned_wif_fields(litellm_params) + reject_server_owned_wif_fields( + MappingProxyType({key: value for key, value in litellm_params.items() if key != "chatgpt_auth_file"}) + ) else: reject_server_owned_wif_params(litellm_params) reject_federated_credential_reference(litellm_params) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 684f1619512..53c54fe15bf 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -2304,7 +2304,15 @@ async def test_model_connection( ) selected_by_id: Final = deployment_by_id is not None - reject_server_owned_wif_fields(request_litellm_params) + reject_server_owned_wif_fields( + MappingProxyType( + { + key: value + for key, value in _OBJECT_MAPPING.validate_python(request_litellm_params).items() + if key != "chatgpt_auth_file" + } + ) + ) # Merge: config params (from proxy config) as base, request params override litellm_params = { **_config_base_for_health_check( diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 8dc7501bc3b..b51eea7a876 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1417,13 +1417,14 @@ def clean_headers( Cleaned headers dict """ from litellm.llms.anthropic.common_utils import is_anthropic_oauth_key + from litellm.llms.chatgpt.common_utils import is_chatgpt_oauth_key clean_headers: Final = {} litellm_key_lower: Final = litellm_key_header_name.lower() if litellm_key_header_name is not None else None for header, value in headers.items(): header_lower = header.lower() - if header_lower == "authorization" and is_anthropic_oauth_key(value): + if header_lower == "authorization" and (is_anthropic_oauth_key(value) or is_chatgpt_oauth_key(value)): if authenticated_with_header is None or authenticated_with_header.lower() != "authorization": clean_headers[header] = value continue @@ -1452,7 +1453,7 @@ def clean_headers( def _is_credential_header(header: str) -> bool: """Whether `header` carries a caller credential rather than request context.""" - return header.lower() in _CREDENTIAL_HEADER_NAMES + return header.lower().removeprefix("x-pass-") in _CREDENTIAL_HEADER_NAMES def redact_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]: @@ -4282,6 +4283,7 @@ _NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join( (LlmProviders.AZURE_AI.value, LlmProviders.DATABRICKS.value) ) _ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = ",".join(sorted(ANTHROPIC_OAUTH_FORWARD_PROVIDERS)) +_CHATGPT_OAUTH_CREDENTIAL_PROVIDERS: Final = LlmProviders.CHATGPT.value def add_provider_specific_headers_to_request( @@ -4289,6 +4291,7 @@ def add_provider_specific_headers_to_request( headers: Mapping[str, str], ) -> bool: from litellm.llms.anthropic.common_utils import is_anthropic_oauth_key + from litellm.llms.chatgpt.common_utils import select_chatgpt_client_credential_headers anthropic_api_headers: Final = {header: headers[header] for header in ANTHROPIC_API_HEADERS if header in headers} anthropic_oauth_credential_headers: Final = { @@ -4296,6 +4299,8 @@ def add_provider_specific_headers_to_request( for header, value in headers.items() if header.lower() == "authorization" and is_anthropic_oauth_key(value) } + chatgpt_oauth_credential_headers: Final = dict(select_chatgpt_client_credential_headers(headers)) + typesafe_danger_zone_headers: Final = { header: value for header, value in headers.items() @@ -4320,13 +4325,17 @@ def add_provider_specific_headers_to_request( ProviderSpecificHeader( custom_llm_provider=LlmProviders.TYPESAFE.value, extra_headers=typesafe_danger_zone_headers ), + ProviderSpecificHeader( + custom_llm_provider=_CHATGPT_OAUTH_CREDENTIAL_PROVIDERS, + extra_headers=chatgpt_oauth_credential_headers, + ), ) if scoped_header["extra_headers"] ] if scoped_headers: data["provider_specific_header"] = scoped_headers[0] if len(scoped_headers) == 1 else scoped_headers - return bool(anthropic_oauth_credential_headers) + return bool(anthropic_oauth_credential_headers or chatgpt_oauth_credential_headers) def _add_otel_traceparent_to_data(data: dict, request: Request): diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py new file mode 100644 index 00000000000..3087faa9f67 --- /dev/null +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -0,0 +1,87 @@ +import hmac +from types import MappingProxyType +from typing import Annotated, Final, Literal + +from fastapi import APIRouter, Depends, HTTPException, Request, Response + +from litellm.llms.chatgpt.common_utils import CHATGPT_API_BASE +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.pass_through_endpoints.pass_through_endpoints import pass_through_request + +router: Final = APIRouter() +_HEADERS: Final = frozenset( + { + "authorization", + "chatgpt-account-id", + "content-type", + "accept", + "user-agent", + "originator", + "openai-beta", + "session-id", + "thread-id", + "session_id", + "conversation_id", + "x-codex-turn-metadata", + "x-codex-turn-state", + } +) + + +@router.get("/backend-api/codex/{endpoint}", tags=["ChatGPT Pass-through", "pass-through"]) +@router.post( + "/backend-api/codex/{endpoint}", + tags=["ChatGPT Pass-through", "pass-through"], + openapi_extra={ + "requestBody": { + "required": True, + "content": {"application/json": {"schema": {"type": "object", "additionalProperties": True}}}, + } + }, + responses={ + 200: { + "description": "Unmodified Codex response or server-sent events", + "content": { + "application/json": {"schema": {"type": "object", "additionalProperties": True}}, + "text/event-stream": {"schema": {"type": "string"}}, + }, + } + }, +) +async def codex_passthrough_route( + endpoint: Literal["models", "responses"], + request: Request, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> Response: + """Forward a client's ChatGPT subscription with separate LiteLLM authentication.""" + expected_method: Final = "GET" if endpoint == "models" else "POST" + if request.method != expected_method: + raise HTTPException(status_code=405, detail="Method not allowed", headers={"Allow": expected_method}) + + authorization: Final = request.headers.get("authorization", "") + scheme, _, token = authorization.partition(" ") + gateway_key: Final = request.headers.get("x-litellm-api-key", "") + gateway_parts: Final = gateway_key.split() + gateway_token: Final = ( + gateway_parts[-1] + if len(gateway_parts) == 1 or (len(gateway_parts) == 2 and gateway_parts[0].lower() == "bearer") + else "" + ) + if scheme.lower() != "bearer" or not token.strip() or not gateway_token: + raise HTTPException( + status_code=401, + detail="Send the ChatGPT bearer in Authorization and the LiteLLM key in x-litellm-api-key", + ) + if hmac.compare_digest(token.strip().encode(), gateway_token.encode()): + raise HTTPException(status_code=401, detail="ChatGPT and LiteLLM credentials must be separate") + + headers: Final = MappingProxyType({name: value for name, value in request.headers.items() if name in _HEADERS}) + return await pass_through_request( + request=request, + target=f"{CHATGPT_API_BASE}/{endpoint}", + custom_headers=dict(headers), + user_api_key_dict=user_api_key_dict, + custom_llm_provider="chatgpt", + upstream_headers_filter=lambda _: headers, + ) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index c763b5dc26e..c410150df70 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -10,6 +10,7 @@ from typing import Final, Literal, TypeAlias from urllib.parse import urlparse import httpx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -95,6 +96,8 @@ def is_openai_compatible_url(url_route: str | None) -> bool: hostname: Final = parsed_url.hostname if not hostname: return False + if hostname == "chatgpt.com" and parsed_url.path.startswith("/backend-api/codex/"): + return True if _hostname_matches(hostname, _OPENAI_HOSTNAMES): return True if _hostname_matches(hostname, _AZURE_OPENAI_HOSTNAMES): @@ -198,7 +201,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): if not url_route: return False parsed_url: Final = urlparse(url_route) - return _is_openai_compatible_host(parsed_url.hostname) and ( + return is_openai_compatible_url(url_route) and ( "/v1/responses" in parsed_url.path or "/responses" in parsed_url.path ) @@ -424,7 +427,9 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): ) = None handler_instance: Final = OpenAIPassthroughLoggingHandler() - custom_llm_provider: Final = kwargs.get("custom_llm_provider", "openai") + custom_llm_provider: Final = TypeAdapter(str).validate_python( + kwargs.get("custom_llm_provider", logging_obj.model_call_details.get("custom_llm_provider", "openai")) + ) if is_chat_completions: # Handle chat completions with existing logic diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index e1e8864678e..fc0e5be3f4c 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -118,6 +118,7 @@ from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy modul _key_or_team_allows_client_pricing_override, # pyright: ignore[reportPrivateUsage] # reuse the proxy's pricing trust policy _strip_client_pricing_overrides, # pyright: ignore[reportPrivateUsage] # sanitize before trusted hooks add guardrail costs get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage] # shared proxy helper, same import style as _read_request_body above + redact_credential_headers, ) from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.route_priority import ( @@ -409,6 +410,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): @staticmethod def get_endpoint_type(url: str, custom_llm_provider: str | None = None) -> EndpointType: parsed_url: Final = urlparse(url) + if custom_llm_provider == "chatgpt": + return EndpointType.OPENAI if custom_llm_provider in ("typesafe", "laya", "bespoke") and parsed_url.path.removesuffix("/").endswith( "/v1/systemone" ): @@ -723,7 +726,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): "url": str(request.url), "method": request.method, "body": copy.copy(_parsed_body), # use copy instead of deepcopy - "headers": request.headers, + "headers": redact_credential_headers(request.headers), }, }, "call_type": "pass_through_endpoint", @@ -1490,7 +1493,7 @@ async def pass_through_request( additional_args={ "complete_input_dict": _parsed_body, "api_base": str(logging_url), - "headers": upstream_headers, + "headers": redact_credential_headers(upstream_headers), }, ) stream = HttpPassThroughEndpointHelpers.update_stream_param_based_on_request_body( @@ -1561,7 +1564,7 @@ async def pass_through_request( data=_parsed_body or {}, user_api_key_dict=user_api_key_dict, response=relay_response, - request_headers=dict(request.headers), + request_headers=dict(redact_credential_headers(request.headers)), ) if callback_headers: _response_headers.update(callback_headers) @@ -1653,7 +1656,7 @@ async def pass_through_request( data=_parsed_body or {}, user_api_key_dict=user_api_key_dict, response=detected_relay_response, - request_headers=dict(request.headers), + request_headers=dict(redact_credential_headers(request.headers)), ) if callback_headers: _response_headers.update(callback_headers) @@ -1703,7 +1706,7 @@ async def pass_through_request( data=_parsed_body or {}, user_api_key_dict=user_api_key_dict, response=response, - request_headers=dict(request.headers), + request_headers=dict(redact_credential_headers(request.headers)), ) if relay_callback_headers: relay_custom_headers.update(relay_callback_headers) @@ -1878,7 +1881,7 @@ async def pass_through_request( data=_parsed_body or {}, user_api_key_dict=user_api_key_dict, response=response, - request_headers=dict(request.headers), + request_headers=dict(redact_credential_headers(request.headers)), ) if callback_headers: custom_headers.update(callback_headers) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 2aab8deb034..604a7a389e6 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -19,6 +19,7 @@ from pydantic import ConfigDict, Field import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_prompt_caching_savings, generic_cost_per_token, @@ -69,7 +70,9 @@ def _resolve_model(model: str | None, custom_llm_provider: str | None) -> _Model if not model: return None try: - resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + resolved_model, provider = resolve_model_and_provider_for_metadata( + model=model, custom_llm_provider=custom_llm_provider + ) except Exception as e: # noqa: BLE001 # get_llm_provider raises for unroutable names; degrade to an unavailable estimate verbose_proxy_logger.debug( "savings: cannot resolve provider for model=%s custom_llm_provider=%s (%s)", model, custom_llm_provider, e diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 1ff80b0d71d..d993b05f5b7 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -705,10 +705,12 @@ def _api_base_kwarg(kwargs: Mapping[str, object]) -> str | None: return api_base if isinstance(api_base, str) else None -def _dispatched_model_name(model: str, custom_llm_provider: str, api_base: str | None) -> str: - provider_model, _, _, _ = litellm.get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider, api_base=api_base - ) +def _dispatched_model_name( + model: str, custom_llm_provider: str, api_base: str | None, litellm_params: GenericLiteLLMParams | None = None +) -> str: + from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata + + provider_model, _ = resolve_model_and_provider_for_metadata(model=model, custom_llm_provider=custom_llm_provider) return _strip_responses_routing_prefix(provider_model) @@ -718,6 +720,7 @@ def _will_bridge_to_chat_completions( use_chat_completions_api: bool, model_info: object, api_base: str | None, + litellm_params: GenericLiteLLMParams | None = None, ) -> bool: """``_bridges_to_chat_completions`` for callers running before the provider config is resolved. @@ -734,7 +737,7 @@ def _will_bridge_to_chat_completions( return True return _bridges_to_chat_completions( _resolve_responses_api_provider_config( - _dispatched_model_name(normalized_model[0], custom_llm_provider, api_base), + _dispatched_model_name(normalized_model[0], custom_llm_provider, api_base, litellm_params), custom_llm_provider, model_info, api_base, @@ -750,12 +753,15 @@ def provider_stores_the_conversation(model: str, custom_llm_provider: str | None first MCP round runs as an internal call that writes no spend log at all. """ api_base: Final = _api_base_kwarg(kwargs) + litellm_params: Final = GenericLiteLLMParams(**kwargs) return not _will_bridge_to_chat_completions( model, - custom_llm_provider or litellm.get_llm_provider(model=model, api_base=api_base)[1], + custom_llm_provider + or litellm.get_llm_provider(model=model, api_base=api_base, litellm_params=litellm_params)[1], bool(kwargs.get("use_chat_completions_api")), kwargs.get("model_info"), api_base, + litellm_params, ) @@ -905,7 +911,8 @@ async def aresponses( model=model, api_base=local_vars.get("base_url", None), litellm_params=GenericLiteLLMParams( - **cast("dict[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict + extra_headers=extra_headers, + **cast("dict[str, object]", kwargs), # cast-ok: kwargs is the untyped request dict ), ) # Update local_vars with detected provider (fixes #19782) @@ -936,6 +943,7 @@ async def aresponses( bool(kwargs.get("use_chat_completions_api")), kwargs.get("model_info"), _api_base_kwarg(kwargs), + GenericLiteLLMParams(**kwargs), ), ): _stamp_injection_points_for_the_hook(kwargs, model, custom_llm_provider, input, tools) @@ -967,7 +975,7 @@ async def aresponses( original_model=original_model, swapped_model=model, custom_llm_provider=custom_llm_provider, - kwargs=kwargs, + kwargs={**kwargs, "extra_headers": extra_headers}, prompt_id=prompt_id, ) kwargs.pop("prompt_id", None) @@ -1051,11 +1059,14 @@ def _resolve_prompt_swapped_provider( kwargs: Mapping[str, object], prompt_id: str | None, ) -> str: - swapped_provider: Final = litellm.get_llm_provider(model=swapped_model)[1] + litellm_params: Final = GenericLiteLLMParams(**kwargs) + swapped_provider: Final = litellm.get_llm_provider(model=swapped_model, litellm_params=litellm_params)[1] if kwargs.get("api_key") is None and kwargs.get("api_base") is None: return swapped_provider try: - original_provider: Final = custom_llm_provider or litellm.get_llm_provider(model=original_model)[1] + original_provider: Final = ( + custom_llm_provider or litellm.get_llm_provider(model=original_model, litellm_params=litellm_params)[1] + ) except litellm.BadRequestError: return swapped_provider if swapped_provider == original_provider: @@ -1108,6 +1119,7 @@ def _apply_prompt_management_to_responses_call( use_chat_completions_api, kwargs.get("model_info"), _api_base_kwarg(kwargs), + GenericLiteLLMParams(**kwargs), ), ): _stamp_injection_points_for_the_hook(kwargs, model, custom_llm_provider, input, tools) @@ -1142,7 +1154,7 @@ def _apply_prompt_management_to_responses_call( original_model=original_model, swapped_model=model, custom_llm_provider=custom_llm_provider, - kwargs=kwargs, + kwargs={**kwargs, "extra_headers": local_vars.get("extra_headers")}, prompt_id=prompt_id, ) ) @@ -1498,7 +1510,8 @@ def responses( model=model, api_base=local_vars.get("base_url", None), litellm_params=GenericLiteLLMParams( - **cast("dict[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict + extra_headers=extra_headers, + **cast("dict[str, object]", kwargs), # cast-ok: kwargs is the untyped request dict ), ) local_vars["custom_llm_provider"] = custom_llm_provider @@ -1514,8 +1527,16 @@ def responses( use_chat_completions_api=use_chat_completions_api, ) - # get llm provider logic - litellm_params: Final = GenericLiteLLMParams(**kwargs) + from litellm.litellm_core_utils.core_helpers import attribute_or_none + + litellm_params: Final = GenericLiteLLMParams(extra_headers=extra_headers, **kwargs) + provider_headers: Final = ResponsesAPIRequestUtils.merge_provider_specific_headers( + extra_headers=extra_headers, + provider_specific_header=attribute_or_none(litellm_params, "provider_specific_header"), + custom_llm_provider=custom_llm_provider, + ) + local_vars["extra_headers"] = provider_headers + affinity_api_base: Final = ( get_openai_credentials(api_base=litellm_params.api_base).api_base if custom_llm_provider == "openai" @@ -1525,7 +1546,7 @@ def responses( try: effective_extra_headers: Final = ( add_provider_affinity_header( - headers=extra_headers or MappingProxyType({}), + headers=provider_headers or MappingProxyType({}), litellm_params=MappingProxyType( { "provider_affinity_header": litellm_params.provider_affinity_header, @@ -1538,7 +1559,7 @@ def responses( default_header=default_affinity_header, ) if litellm_params.provider_affinity_header is not None or default_affinity_header is not None - else extra_headers + else provider_headers ) except ValueError as affinity_error: raise litellm.BadRequestError( diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index dc75e77d685..2d0873a218a 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -12,6 +12,7 @@ import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, iter_items from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path +from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel from litellm.types.llms.openai import ( AllMessageValues, @@ -29,11 +30,16 @@ from litellm.types.utils import ( CacheCreationTokenDetails, CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, + ProviderSpecificHeader, SpecialEnums, Usage, text_tokens_without_nested_reasoning, ) +_PROVIDER_SPECIFIC_HEADER_ADAPTER: Final[ + TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None] +] = TypeAdapter(ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None) + class _UsageExtraFields(ExtraItemsTypedDict, total=False, extra_items=object): pass @@ -233,6 +239,20 @@ class ResponsesAPIRequestUtils: **extra_headers, } + @staticmethod + def merge_provider_specific_headers( + extra_headers: Mapping[str, object] | None, + provider_specific_header: object, + custom_llm_provider: str | None, + ) -> dict[str, object] | None: # mutable-ok: legacy Responses request fields require a dict + scoped_headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header=_PROVIDER_SPECIFIC_HEADER_ADAPTER.validate_python(provider_specific_header), + custom_llm_provider=custom_llm_provider, + ) + if extra_headers is None and not scoped_headers: + return None + return {**(extra_headers or {}), **scoped_headers} + @staticmethod def _check_valid_arg( supported_params: list[str] | None, diff --git a/litellm/router.py b/litellm/router.py index 3af39736e25..c76e4ec843f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -98,7 +98,9 @@ from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, r from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_llm_provider_logic import ( declared_authenticating_provider, + inferred_provider, is_registered_custom_provider, + resolve_model_and_provider_for_metadata, ) from litellm.litellm_core_utils.hidden_params import ( HIDDEN_PARAMS_ATTR as _HIDDEN_PARAMS_ATTR, @@ -561,6 +563,7 @@ _ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset( {"model", "api_base", "api_key", "api_version", "responses_api_version"} ) _ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params" +_FORWARDED_CREDENTIAL_ISOLATION_PROVIDERS: Final = frozenset({"chatgpt"}) _CLAUDE_CODE_SESSION_ID_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$") _CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS: Final = 3600 CLAUDE_CODE_SESSION_ROUTER_BINDING_TARGET: Final = "claude_code_session_router_binding" @@ -4801,9 +4804,39 @@ class Router: deployment_litellm_model_name = deployment["litellm_params"]["model"] deployment_api_base = deployment["litellm_params"].get("api_base") deployment_model_name: Final = deployment["model_name"] - if is_clientside_credential(request_kwargs=kwargs): + from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils + + request_context: Final = cast( # cast-ok: the router's legacy request dict has string keys + Mapping[str, object], kwargs + ) + raw_deployment_params: Final[object] = deployment["litellm_params"] + deployment_context: Final = _LITELLM_PARAMS_ADAPTER.validate_python( + raw_deployment_params.model_dump(exclude_none=True) + if isinstance(raw_deployment_params, LiteLLM_Params) + else raw_deployment_params + ) + declared_provider: Final = deployment_context.get("custom_llm_provider") + declared_model: Final = deployment_context.get("model") + deployment_provider: Final = ( + declared_provider + if isinstance(declared_provider, str) + else inferred_provider(declared_model if isinstance(declared_model, str) else None) + ) + forwarded_authorization: Final = ( + ProviderSpecificHeaderUtils.forwarded_authorization( + request_context.get("provider_specific_header"), deployment_provider + ) + if deployment_provider in _FORWARDED_CREDENTIAL_ISOLATION_PROVIDERS + else None + ) + if is_clientside_credential(request_kwargs=kwargs) or forwarded_authorization is not None: deployment_pydantic_obj: Final = self._handle_clientside_credential( - deployment=deployment, kwargs=kwargs, function_name=function_name + deployment=deployment, + kwargs={ + **kwargs, + **({"api_key": forwarded_authorization} if forwarded_authorization is not None else {}), + }, + function_name=function_name, ) model_info = deployment_pydantic_obj.model_info.model_dump() deployment_litellm_model_name = deployment_pydantic_obj.litellm_params.model @@ -11048,6 +11081,10 @@ class Router: custom_llm_provider = None dynamic_api_key = None api_base = None + elif declared_authenticating_provider(litellm_model, deployment.litellm_params.custom_llm_provider) is not None: + _model, custom_llm_provider = ( # rebind-ok: each branch resolves the deployment provider + self._model_group_llm_provider(deployment.litellm_params) + ) else: # check if model provider in supported providers ( @@ -12401,6 +12438,17 @@ class Router: ) return model_info + @staticmethod + def _model_group_llm_provider(litellm_params: LiteLLM_Params) -> tuple[str, str]: + try: + return resolve_model_and_provider_for_metadata( + model=litellm_params.model, + custom_llm_provider=litellm_params.custom_llm_provider, + ) + except litellm.exceptions.BadRequestError as e: + verbose_router_logger.error("litellm.router.py::get_model_group_info() - %s", e) + return "", "" + def _set_model_group_info( self, model_group: str, user_facing_model_group_name: str, team_id: str | None = None ) -> ModelGroupInfo | None: @@ -12489,15 +12537,7 @@ class Router: deployment_is_mapped = deployment_is_catalog_mapped(model_info, model_info_dict) # get llm provider - litellm_model, llm_provider = "", "" - try: - litellm_model, llm_provider, _, _ = litellm.get_llm_provider( - model=litellm_params.model, - custom_llm_provider=litellm_params.custom_llm_provider, - litellm_params=litellm_params, - ) - except litellm.exceptions.BadRequestError as e: - verbose_router_logger.error("litellm.router.py::get_model_group_info() - %s", e) + litellm_model, llm_provider = self._model_group_llm_provider(litellm_params) if model_info is None: supported_openai_params = litellm.get_supported_openai_params( diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 089f21a33c1..717b74e5741 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -91,6 +91,7 @@ class ProviderConnection: litellm_credential_name: str | None = None configurable_clientside_auth_params: "Sequence[str | ConfigurableClientsideParamsCustomAuth] | None" = None use_xai_oauth: bool | None = None + chatgpt_auth_file: str | None = None github_copilot_auth_type: str | None = None github_copilot_host: str | None = None github_copilot_user_session: object | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index cd1389aea9d..ba9b82e2a57 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -572,6 +572,10 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): default=False, description="Use stored xAI OAuth credentials when no xAI API key is configured.", ) + chatgpt_auth_file: str | None = Field( + default=None, + description="Path to a ChatGPT OAuth auth.json for this deployment.", + ) fireworks_forward_user_id: bool | None = Field( default=None, description="Send the LiteLLM user id of the calling key as the `user` field on Fireworks AI chat, responses and messages requests.", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index cc7ef851111..4f39741a079 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4360,6 +4360,7 @@ server_owned_wif_litellm_params: Final = ( + openai_wif_litellm_params + oauth_token_exchange_litellm_params + github_copilot_oauth_litellm_params + + ("chatgpt_auth_file",) ) secret_bearing_wif_litellm_params: Final = tuple(sorted(WIF_SECRET_BEARING_KEYS)) diff --git a/litellm/utils.py b/litellm/utils.py index 4315ae426cf..1cc7c63e6a5 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2964,12 +2964,14 @@ def supports_native_streaming(model: str, custom_llm_provider: str | None) -> bo Raises: Exception: If the given model is not found in model_prices_and_context_window.json. """ + from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata + try: - model, custom_llm_provider, _, _ = litellm.get_llm_provider( + resolved_model, resolved_provider = resolve_model_and_provider_for_metadata( model=model, custom_llm_provider=custom_llm_provider ) - model_info: Final = get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + model_info: Final = get_model_info_helper(model=resolved_model, custom_llm_provider=resolved_provider) supports_native_streaming = model_info.get("supports_native_streaming", True) if supports_native_streaming is None: supports_native_streaming = True diff --git a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py index 57e66805678..e9e3f7b8174 100644 --- a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py +++ b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py @@ -1,5 +1,9 @@ +from collections.abc import Iterator +from pathlib import Path from typing import Final +import respx + import asyncio, httpx, importlib, os import pytest @@ -8,6 +12,7 @@ from litellm import CustomLLM from litellm.litellm_core_utils import get_llm_provider_logic from litellm.litellm_core_utils.get_llm_provider_logic import ( get_llm_provider, + resolve_model_and_provider_for_metadata, get_provider_affinity_header, inferred_provider, is_registered_custom_provider, @@ -715,3 +720,47 @@ def test_opencode_affinity_resolution_preserves_existing_transport(prefix: str) model, provider, _, destination = get_llm_provider(model=f"{prefix}/test-model", api_base=api_base) assert (model, provider, destination) == ("test-model", prefix, api_base) assert get_provider_affinity_header(destination) == "x-opencode-session" + + +@pytest.fixture +def chatgpt_login_tripwire(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[None]: + blocked: Final = tmp_path / "blocked" + blocked.write_text("ordinary file") + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(blocked / "unused")) + monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False) + with respx.mock as transport: + yield + assert len(transport.calls) == 0 + + +@pytest.mark.parametrize( + ("model", "custom_llm_provider", "expected"), + [ + ("chatgpt/gpt-5.4", None, ("gpt-5.4", "chatgpt")), + ("gpt-5.4", "chatgpt", ("gpt-5.4", "chatgpt")), + ("github_copilot/gpt-4o", None, ("gpt-4o", "github_copilot")), + ("openai/gpt-5.4", None, ("gpt-5.4", "openai")), + ], +) +def test_resolve_model_and_provider_for_metadata_never_logs_in( + chatgpt_login_tripwire: None, model: str, custom_llm_provider: str | None, expected: tuple[str, str] +) -> None: + assert resolve_model_and_provider_for_metadata(model, custom_llm_provider) == expected + + +def test_metadata_callers_never_log_in_for_a_declared_chatgpt_deployment( + chatgpt_login_tripwire: None, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base + from litellm.proxy.spend_tracking.savings import _resolve_model + from litellm.router import Router + from litellm.types.router import LiteLLM_Params + from litellm.utils import supports_native_streaming + + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + + assert supports_native_streaming("chatgpt/gpt-5.4", None) is True + assert get_api_base("chatgpt/gpt-5.4", {"model": "chatgpt/gpt-5.4"}) == "https://chatgpt.com/backend-api/codex" + identity = _resolve_model("chatgpt/gpt-5.4", None) + assert identity is not None and identity.provider == "chatgpt" + assert Router._model_group_llm_provider(LiteLLM_Params(model="chatgpt/gpt-5.4")) == ("gpt-5.4", "chatgpt") diff --git a/tests/unit/llms/chatgpt/chat/test_chatgpt_chat_transformation.py b/tests/unit/llms/chatgpt/chat/test_chatgpt_chat_transformation.py new file mode 100644 index 00000000000..f9d1dc28d12 --- /dev/null +++ b/tests/unit/llms/chatgpt/chat/test_chatgpt_chat_transformation.py @@ -0,0 +1,638 @@ +from unittest.mock import MagicMock + +import pytest + +from litellm.exceptions import AuthenticationError +from litellm.llms.chatgpt.chat.transformation import ChatGPTConfig +from litellm.llms.chatgpt.common_utils import GetAccessTokenError + +GATEWAY_BASE = "https://chatgpt.example/backend-api/codex" + + +def _gateway_authenticator() -> MagicMock: + authenticator = MagicMock() + authenticator.get_api_base.return_value = GATEWAY_BASE + authenticator.get_access_token.return_value = "gateway-access-token" + return authenticator + + +def _provider_info( + config: ChatGPTConfig, api_base: str | None, api_key: str | None +) -> tuple[str | None, str | None, str]: + return config._get_openai_compatible_provider_info( + model="gpt-5.4", api_base=api_base, api_key=api_key, custom_llm_provider="chatgpt" + ) + + +class TestChatGPTChatProviderInfo: + def test_configured_key_uses_configured_base_without_gateway_login(self): + authenticator = _gateway_authenticator() + config = ChatGPTConfig(authenticator=authenticator) + + assert _provider_info(config, "http://127.0.0.1:9999", "configured-key") == ( + "http://127.0.0.1:9999", + "configured-key", + "chatgpt", + ) + authenticator.get_access_token.assert_not_called() + + def test_configured_key_without_base_uses_gateway_base(self): + config = ChatGPTConfig(authenticator=_gateway_authenticator()) + + assert _provider_info(config, None, "configured-key") == (GATEWAY_BASE, "configured-key", "chatgpt") + + def test_gateway_login_is_never_sent_to_a_custom_base(self): + config = ChatGPTConfig(authenticator=_gateway_authenticator()) + + assert _provider_info(config, "http://attacker.example", None) == ( + GATEWAY_BASE, + "gateway-access-token", + "chatgpt", + ) + + def test_missing_gateway_login_raises_authentication_error(self): + authenticator = _gateway_authenticator() + authenticator.get_access_token.side_effect = GetAccessTokenError(status_code=401, message="not logged in") + config = ChatGPTConfig(authenticator=authenticator) + + with pytest.raises(AuthenticationError, match="not logged in"): + _provider_info(config, None, None) + + +import asyncio +import json +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import httpx +import respx + +import litellm +from litellm.integrations.dotprompt.dotprompt_manager import DotpromptManager +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.chatgpt.chat.transformation import ChatGPTConfig +from litellm.types.router import GenericLiteLLMParams + + +def _write_auth_record(path: Path, token: str, account_id: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "access_token": token, + "account_id": account_id, + "expires_at": 4102444800, + } + ) + ) + + +@pytest.fixture +def auth_files(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Path, Path]: + monkeypatch.setattr("time.time", lambda: 1700000000) + default_dir: Final = tmp_path / "default" + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(default_dir)) + monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False) + _write_auth_record(default_dir / "auth.json", "token-default", "acct-default") + file_a: Final = tmp_path / "account-a" / "auth.json" + file_b: Final = tmp_path / "account-b" / "auth.json" + _write_auth_record(file_a, "token-a", "acct-a") + _write_auth_record(file_b, "token-b", "acct-b") + return file_a, file_b + + +class TestChatGPTMultiAccountChat: + def test_validate_environment_uses_per_model_auth_file(self, auth_files: tuple[Path, Path]) -> None: + file_a, file_b = auth_files + config: Final = ChatGPTConfig() + + headers_a: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + messages=[], + optional_params={}, + litellm_params={"chatgpt_auth_file": str(file_a)}, + api_key="token-default", + ) + headers_b: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + messages=[], + optional_params={}, + litellm_params={"chatgpt_auth_file": str(file_b)}, + api_key="token-default", + ) + + assert headers_a["Authorization"] == "Bearer token-a" + assert headers_a["ChatGPT-Account-Id"] == "acct-a" + assert headers_b["Authorization"] == "Bearer token-b" + assert headers_b["ChatGPT-Account-Id"] == "acct-b" + + def test_validate_environment_defaults_to_global_auth_file(self, auth_files: tuple[Path, Path]) -> None: + config: Final = ChatGPTConfig() + + headers: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + messages=[], + optional_params={}, + litellm_params={}, + api_key="token-default", + ) + + assert headers["Authorization"] == "Bearer token-default" + assert headers["ChatGPT-Account-Id"] == "acct-default" + + def test_provider_info_uses_per_model_auth_file(self, auth_files: tuple[Path, Path]) -> None: + file_a, _ = auth_files + config: Final = ChatGPTConfig() + + _, per_model_key, _ = config._get_openai_compatible_provider_info( + "gpt-6.1-sol", None, None, "chatgpt", {"chatgpt_auth_file": str(file_a)} + ) + _, default_key, _ = config._get_openai_compatible_provider_info("gpt-6.1-sol", None, None, "chatgpt") + + assert per_model_key == "token-a" + assert default_key == "token-default" + + def test_get_llm_provider_threads_auth_file(self, auth_files: tuple[Path, Path]) -> None: + file_a, _ = auth_files + + model, provider, dynamic_api_key, _ = get_llm_provider( + model="chatgpt/gpt-6.1-sol", + litellm_params=GenericLiteLLMParams(chatgpt_auth_file=str(file_a)), + ) + + assert model == "gpt-6.1-sol" + assert provider == "chatgpt" + assert dynamic_api_key == "token-a" + + +def _subscription_response(request: httpx.Request) -> httpx.Response: + account: Final = request.headers["ChatGPT-Account-Id"] + assert request.headers["Authorization"] in ( + f"Bearer token-{account.removeprefix('acct-')}", + f"Bearer {_client_token(account)}", + ) + assert "chatgpt_auth_file" not in request.content.decode() + message: Final = { + "id": "msg_test", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": account, "annotations": []}], + } + response: Final = { + "id": "resp_test", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-6.1-sol", + "output": [message], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + events: Final = ( + {"type": "response.created", "response": {**response, "status": "in_progress", "output": []}}, + { + "type": "response.output_text.delta", + "item_id": "msg_test", + "output_index": 0, + "content_index": 0, + "delta": account, + }, + {"type": "response.output_item.done", "output_index": 0, "item": message}, + {"type": "response.completed", "response": response}, + ) + return httpx.Response( + 200, + text="".join(f"data: {json.dumps(event)}\n\n" for event in events), + headers={"content-type": "text/event-stream"}, + ) + + +@pytest.mark.parametrize("blocked_default", [False, True]) +@pytest.mark.parametrize("with_prompt", [False, True]) +@pytest.mark.parametrize("endpoint", ["chat", "responses"]) +@pytest.mark.parametrize("through_router", [False, True]) +@pytest.mark.asyncio +async def test_requests_use_selected_subscription( + auth_files: tuple[Path, Path], + endpoint: str, + through_router: bool, + with_prompt: bool, + blocked_default: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + default_dir: Final = auth_files[0].parent / "missing-default" + if blocked_default: + default_dir.write_text("ordinary file") + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(default_dir / "unused")) + if with_prompt: + monkeypatch.setattr( + litellm, + "callbacks", + [ + DotpromptManager( + prompt_id="account-check", + prompt_data={"content": "Saved account prompt", "metadata": {"model": "chatgpt/gpt-6.1-sol"}}, + ) + ], + ) + router: Final = ( + litellm.Router( + model_list=[ + { + "model_name": f"account-{index}", + "litellm_params": {"model": "chatgpt/gpt-6.1-sol", "chatgpt_auth_file": str(path)}, + } + for index, path in enumerate(auth_files) + ] + ) + if through_router + else None + ) + + async def request(index: int, path: Path) -> str: + params: Final = MappingProxyType( + { + "model": f"account-{index}" if router else "chatgpt/gpt-6.1-sol", + **({"chatgpt_auth_file": str(path)} if router is None else {}), + **({"prompt_id": "account-check"} if with_prompt else {}), + } + ) + if endpoint == "chat": + result: Final = await (router.acompletion if router else litellm.acompletion)( + **params, messages=[{"role": "user", "content": "Hello"}] + ) + return result.choices[0].message.content + response: Final = await (router.aresponses if router else litellm.aresponses)( + **params, input=[{"role": "user", "content": "Hello"}] + ) + return response.output[0].content[0].text + + with respx.mock as transport: + transport.post("https://chatgpt.com/backend-api/codex/responses").mock(side_effect=_subscription_response) + results: Final = await asyncio.gather(*(request(index, path) for index, path in enumerate(auth_files))) + assert results == ["acct-a", "acct-b"] + if with_prompt: + assert all("Saved account prompt" in call.request.content.decode() for call in transport.calls) + + +def _client_token(account: str) -> str: + import base64 + + payload: Final = ( + base64.urlsafe_b64encode(json.dumps({"https://api.openai.com/auth": {"chatgpt_account_id": account}}).encode()) + .rstrip(b"=") + .decode() + ) + return f"header.{payload}.signature" + + +@pytest.mark.parametrize("endpoint", ["chat", "responses"]) +@pytest.mark.parametrize("configured_login", [False, True]) +@pytest.mark.parametrize("with_prompt", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.asyncio +async def test_client_login_keeps_its_account_across_gateway_deployments( + auth_files: tuple[Path, Path], + endpoint: str, + configured_login: bool, + with_prompt: bool, + stream: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(auth_files[0].parent / "no-gateway-login")) + if with_prompt: + monkeypatch.setattr( + litellm, + "callbacks", + [ + DotpromptManager( + prompt_id="client-account-check", + prompt_data={"content": "Saved client prompt", "metadata": {"model": "chatgpt/gpt-6.1-sol"}}, + ) + ], + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "subscriptions", + "litellm_params": { + "model": "chatgpt/gpt-6.1-sol", + **({"chatgpt_auth_file": str(path)} if configured_login else {}), + }, + } + for path in auth_files + ], + num_retries=0, + ) + + def reply(request: httpx.Request) -> httpx.Response: + account: Final = request.headers["ChatGPT-Account-Id"] + assert request.headers["Authorization"] == f"Bearer {_client_token(account)}" + return _subscription_response( + httpx.Request( + "POST", + request.url, + content=request.content, + headers={ + **{key: value for key, value in request.headers.items() if key.lower() != "authorization"}, + "Authorization": f"Bearer token-{account.removeprefix('acct-')}", + }, + ) + ) + + async def call(account: str) -> str: + params: Final = MappingProxyType( + { + "model": "subscriptions", + "stream": stream, + **({"prompt_id": "client-account-check"} if with_prompt else {}), + "provider_specific_header": { + "custom_llm_provider": "chatgpt", + "extra_headers": {"Authorization": f"Bearer {_client_token(account)}"}, + }, + } + ) + response: Final = await ( + router.acompletion(**params, messages=[{"role": "user", "content": "Hello"}]) + if endpoint == "chat" + else router.aresponses(**params, input=[{"role": "user", "content": "Hello"}]) + ) + if stream: + chunks: Final = [part async for part in response] + return ( + "".join(part.choices[0].delta.content or "" for part in chunks) + if endpoint == "chat" + else "".join( + getattr(part, "delta", "") + for part in chunks + if getattr(part, "type", "") == "response.output_text.delta" + ) + ) + return response.choices[0].message.content if endpoint == "chat" else response.output[0].content[0].text + + with respx.mock as transport: + transport.post("https://chatgpt.com/backend-api/codex/responses").mock(side_effect=reply) + results: Final = await asyncio.gather(call("acct-client-a"), call("acct-client-b")) + assert results == ["acct-client-a", "acct-client-b"] + assert len(transport.calls) == 2 + + +@pytest.mark.asyncio +async def test_one_client_limit_does_not_cool_down_other_subscriptions( + auth_files: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + {"model_name": "pool", "litellm_params": {"model": "chatgpt/gpt-6.1-sol", "chatgpt_auth_file": str(path)}} + for path in auth_files + ], + num_retries=0, + allowed_fails=0, + cooldown_time=60, + ) + headers: Final = { + "custom_llm_provider": "chatgpt", + "extra_headers": {"Authorization": f"Bearer {_client_token('acct-limited')}"}, + } + with respx.mock as transport: + route: Final = transport.post("https://chatgpt.com/backend-api/codex/responses") + route.respond( + 429, + headers={"retry-after": "60"}, + json={"error": {"message": "Usage limit reached", "type": "rate_limit_error"}}, + ) + for _ in range(2): + with pytest.raises(litellm.RateLimitError): + await router.aresponses(model="pool", input="Hello", provider_specific_header=headers) + await asyncio.sleep(0) + route.mock(side_effect=_subscription_response) + response: Final = await router.aresponses(model="pool", input="Hello") + assert response.output[0].content[0].text in ("acct-a", "acct-b") + + +@pytest.mark.parametrize("endpoint", ["chat", "responses"]) +@pytest.mark.parametrize("async_call", [False, True]) +@pytest.mark.parametrize("direct_headers", [False, True]) +@pytest.mark.parametrize("configured_key", [False, True]) +@pytest.mark.parametrize( + "target", ["https://example.com/v1", "http://chatgpt.com/backend-api/codex", "https://chatgpt.com.example.com/v1"] +) +@pytest.mark.asyncio +async def test_client_login_cannot_be_redirected_to_another_host( + endpoint: str, target: str, async_call: bool, direct_headers: bool, configured_key: bool +) -> None: + headers: Final = {"Authorization": f"Bearer {_client_token('acct-client')}"} + credential: Final = ( + {"extra_headers": headers} + if direct_headers + else {"provider_specific_header": {"custom_llm_provider": "chatgpt", "extra_headers": headers}} + ) + params: Final = MappingProxyType( + { + "model": "chatgpt/gpt-6.1-sol", + "api_base": target, + **({"api_key": "configured-test-key"} if configured_key else {}), + **credential, + } + ) + + async def request() -> None: + if endpoint == "chat": + if async_call: + await litellm.acompletion(**params, messages=[{"role": "user", "content": "Hello"}]) + else: + litellm.completion(**params, messages=[{"role": "user", "content": "Hello"}]) + elif async_call: + await litellm.aresponses(**params, input="Hello") + else: + litellm.responses(**params, input="Hello") + + with respx.mock as transport: + with pytest.raises(litellm.BadRequestError, match="official HTTPS ChatGPT endpoint"): + await request() + assert len(transport.calls) == 0 + + +@pytest.mark.asyncio +async def test_cross_provider_fallback_does_not_receive_the_chatgpt_login( + auth_files: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "subscription", + "litellm_params": {"model": "chatgpt/gpt-6.1-sol", "chatgpt_auth_file": str(auth_files[0])}, + }, + { + "model_name": "api-fallback", + "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "sk-example-fallback"}, + }, + ], + fallbacks=[{"subscription": ["api-fallback"]}], + num_retries=0, + ) + with respx.mock as transport: + transport.post("https://chatgpt.com/backend-api/codex/responses").respond( + 429, json={"error": {"message": "Usage limit reached", "type": "rate_limit_error"}} + ) + fallback: Final = transport.post("https://api.openai.com/v1/responses").respond( + 200, + json={ + "id": "resp_fallback", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-6.1-sol", + "output": [ + { + "id": "msg_fallback", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "fallback", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + ) + response: Final = await router.aresponses( + model="subscription", + input="Hello", + provider_specific_header={ + "custom_llm_provider": "chatgpt", + "extra_headers": { + "Authorization": f"Bearer {_client_token('acct-client')}", + "ChatGPT-Account-Id": "acct-client", + }, + }, + ) + assert response.output[0].content[0].text == "fallback" + assert fallback.calls.last.request.headers["Authorization"] == "Bearer sk-example-fallback" + assert "ChatGPT-Account-Id" not in fallback.calls.last.request.headers + + +@pytest.mark.parametrize("endpoint", ["chat", "responses"]) +@pytest.mark.parametrize("async_call", [False, True]) +@pytest.mark.parametrize("with_prompt", [False, True]) +@pytest.mark.asyncio +async def test_sdk_extra_headers_use_client_login_without_gateway_auth( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, endpoint: str, async_call: bool, with_prompt: bool +) -> None: + blocked: Final = tmp_path / "blocked" + blocked.write_text("ordinary file") + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(blocked / "unused")) + monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + headers: Final = {"Authorization": f"Bearer {_client_token('acct-client-a')}"} + prompt_params: Final = {"prompt_id": "sdk-client"} if with_prompt else {} + if with_prompt: + monkeypatch.setattr( + litellm, + "callbacks", + [ + DotpromptManager( + prompt_id="sdk-client", + prompt_data={"content": "Hello", "metadata": {"model": "chatgpt/gpt-6-luna"}}, + ) + ], + ) + with respx.mock as transport: + transport.post("https://chatgpt.com/backend-api/codex/responses").mock(side_effect=_subscription_response) + if endpoint == "chat": + params: Final = { + "model": "chatgpt/gpt-6.1-sol", + "messages": [{"role": "user", "content": "Hello"}], + "extra_headers": headers, + **prompt_params, + } + result: Final = await litellm.acompletion(**params) if async_call else litellm.completion(**params) + assert result.choices[0].message.content == "acct-client-a" + else: + response: Final = ( + await litellm.aresponses( + model="chatgpt/gpt-6.1-sol", input="Hello", extra_headers=headers, **prompt_params + ) + if async_call + else litellm.responses( + model="chatgpt/gpt-6.1-sol", input="Hello", extra_headers=headers, **prompt_params + ) + ) + assert response.output[0].content[0].text == "acct-client-a" + assert transport.calls.last.request.headers["Authorization"] == headers["Authorization"] + assert len(transport.calls) == 1 + + +@pytest.mark.parametrize("endpoint", ["chat", "responses"]) +@pytest.mark.parametrize("sync", [False, True]) +@pytest.mark.parametrize("credential_source", ["default", "file", "api_key"]) +@pytest.mark.parametrize("authorization_header", [None, "Authorization", "authorization"]) +@pytest.mark.asyncio +async def test_requests_preserve_gateway_token_and_selected_workspace( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + endpoint: str, + sync: bool, + credential_source: str, + authorization_header: str | None, +) -> None: + monkeypatch.setattr("time.time", lambda: 1700000000) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("CHATGPT_AUTH_FILE", str(tmp_path / "default.json")) + token: Final = _client_token("acct-token-default") + auth_file: Final = tmp_path / ("selected.json" if credential_source == "file" else "default.json") + _write_auth_record(auth_file, token, "acct-saved-workspace") + configured_token: Final = _client_token("acct-configured") + expected_token: Final = configured_token if credential_source == "api_key" else token + expected_account: Final = "acct-configured" if credential_source == "api_key" else "acct-saved-workspace" + + def reply(request: httpx.Request) -> httpx.Response: + assert request.headers.get_list("authorization") == [f"Bearer {expected_token}"] + assert request.headers.get_list("chatgpt-account-id") == [expected_account] + assert request.headers["originator"] == "codex_cli_rs" + return _subscription_response( + httpx.Request( + "POST", + request.url, + content=request.content, + headers={ + "Authorization": f"Bearer token-{expected_account.removeprefix('acct-')}", + "ChatGPT-Account-Id": expected_account, + }, + ) + ) + + params: Final = MappingProxyType( + { + "model": "chatgpt/gpt-6.1-sol", + **({"chatgpt_auth_file": str(auth_file)} if credential_source == "file" else {}), + **({"api_key": configured_token} if credential_source == "api_key" else {}), + "extra_headers": { + "originator": "codex_cli_rs", + **({authorization_header: "Bearer sk-unrelated-example"} if authorization_header else {}), + }, + } + ) + with respx.mock as transport: + transport.post("https://chatgpt.com/backend-api/codex/responses").mock(side_effect=reply) + if endpoint == "chat": + result: Final = ( + await asyncio.to_thread(litellm.completion, **params, messages=[{"role": "user", "content": "Hello"}]) + if sync + else await litellm.acompletion(**params, messages=[{"role": "user", "content": "Hello"}]) + ) + assert result.choices[0].message.content == expected_account + else: + response: Final = ( + await asyncio.to_thread(litellm.responses, **params, input=[{"role": "user", "content": "Hello"}]) + if sync + else await litellm.aresponses(**params, input=[{"role": "user", "content": "Hello"}]) + ) + assert response.output[0].content[0].text == expected_account diff --git a/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py b/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py index e47d7fc702c..d88bdfc0b41 100644 --- a/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py +++ b/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py @@ -4,8 +4,10 @@ Tests for ChatGPT subscription Responses API transformation Source: litellm/llms/chatgpt/responses/transformation.py """ +import base64 import json from collections.abc import Generator +from pathlib import Path from typing import Final from unittest.mock import MagicMock, patch @@ -13,6 +15,7 @@ import httpx import pytest import litellm +from litellm.llms.chatgpt.authenticator import Authenticator from litellm.llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig from litellm.llms.openai.common_utils import OpenAIError from litellm.main import responses_api_bridge_check @@ -395,3 +398,165 @@ class TestChatGPTResponsesAPITransformation: assert exc_info.value.message == "ChatGPT upstream failed" assert exc_info.value.status_code == 502 + + +def _fake_chatgpt_oauth_token(account_id: str) -> str: + header = base64.urlsafe_b64encode(b'{"alg":"RS256"}').rstrip(b"=").decode() + payload = ( + base64.urlsafe_b64encode( + json.dumps({"https://api.openai.com/auth": {"chatgpt_account_id": account_id}}).encode() + ) + .rstrip(b"=") + .decode() + ) + return f"{header}.{payload}.fake-signature" + + +def _gateway_authenticator() -> MagicMock: + authenticator = MagicMock() + authenticator.get_access_token.return_value = "gateway-access-token" + authenticator.get_account_id.return_value = "acct-gateway" + return authenticator + + +def _config_with(authenticator: MagicMock) -> ChatGPTResponsesAPIConfig: + return ChatGPTResponsesAPIConfig(authenticator=authenticator) + + +class TestChatGPTClientCredentialForwarding: + def test_client_chatgpt_oauth_wins_over_configured_key_and_gateway_login(self): + authenticator = _gateway_authenticator() + token = _fake_chatgpt_oauth_token("acct-alice") + + headers = _config_with(authenticator).validate_environment( + headers={"Authorization": f"Bearer {token}", "originator": "codex_cli_rs"}, + model="gpt-5.4", + litellm_params=GenericLiteLLMParams(api_key="configured-gateway-key", litellm_session_id="session-1"), + ) + + assert headers["Authorization"] == f"Bearer {token}" + assert headers["ChatGPT-Account-Id"] == "acct-alice" + assert headers["originator"] == "codex_cli_rs" + assert headers["accept"] == "text/event-stream" + assert [name for name in headers if name.lower() == "authorization"] == ["Authorization"] + authenticator.get_access_token.assert_not_called() + + def test_explicit_account_header_wins_over_jwt_claim(self): + token = _fake_chatgpt_oauth_token("acct-from-jwt") + + headers = _config_with(_gateway_authenticator()).validate_environment( + headers={"authorization": f"Bearer {token}", "chatgpt-account-id": "acct-from-header"}, + model="gpt-5.4", + litellm_params=GenericLiteLLMParams(), + ) + + assert headers["ChatGPT-Account-Id"] == "acct-from-header" + assert [name for name in headers if name.lower() == "chatgpt-account-id"] == ["ChatGPT-Account-Id"] + + @pytest.mark.parametrize( + "bearer", ["Bearer sk-proj-fake-platform-key", "Bearer not-a-jwt", "Bearer a.bm90LWpzb24.c"] + ) + def test_non_chatgpt_bearer_falls_back_to_configured_key(self, bearer, tmp_path): + authenticator = Authenticator(auth_file=str(tmp_path / "missing-default.json")) + + headers = ChatGPTResponsesAPIConfig(authenticator=authenticator).validate_environment( + headers={"Authorization": bearer}, + model="gpt-5.4", + litellm_params=GenericLiteLLMParams(api_key="configured-gateway-key"), + ) + + assert "ChatGPT-Account-Id" not in headers + assert headers["Authorization"] == "Bearer configured-gateway-key" + + def test_configured_key_is_used_without_client_credential(self): + authenticator = _gateway_authenticator() + + headers = _config_with(authenticator).validate_environment( + headers={}, + model="gpt-5.4", + litellm_params=GenericLiteLLMParams(api_key="configured-gateway-key"), + ) + + assert headers["Authorization"] == "Bearer configured-gateway-key" + authenticator.get_access_token.assert_not_called() + + def test_gateway_login_is_used_without_client_credential_or_configured_key(self): + headers = _config_with(_gateway_authenticator()).validate_environment( + headers={}, + model="gpt-5.4", + litellm_params=GenericLiteLLMParams(), + ) + + assert headers["Authorization"] == "Bearer gateway-access-token" + assert headers["ChatGPT-Account-Id"] == "acct-gateway" + + def test_two_clients_never_share_a_credential(self): + config = _config_with(_gateway_authenticator()) + alice = _fake_chatgpt_oauth_token("acct-alice") + bob = _fake_chatgpt_oauth_token("acct-bob") + + forwarded = [ + config.validate_environment( + headers={"Authorization": f"Bearer {token}"}, model="gpt-5.4", litellm_params=GenericLiteLLMParams() + ) + for token in (alice, bob, alice) + ] + + assert [h["Authorization"] for h in forwarded] == [f"Bearer {alice}", f"Bearer {bob}", f"Bearer {alice}"] + assert [h["ChatGPT-Account-Id"] for h in forwarded] == ["acct-alice", "acct-bob", "acct-alice"] + + +class TestChatGPTResponsesMultiAccount: + @pytest.fixture(autouse=True) + def fixed_clock(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("time.time", lambda: 1700000000) + + @staticmethod + def _write_auth_record(path: Path, token: str, account_id: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "access_token": token, + "account_id": account_id, + "expires_at": 4102444800, + } + ) + ) + + def test_validate_environment_uses_per_model_auth_file( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + default_dir: Final = tmp_path / "default" + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(default_dir)) + monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False) + self._write_auth_record(default_dir / "auth.json", "token-default", "acct-default") + file_a: Final = tmp_path / "account-a" / "auth.json" + file_b: Final = tmp_path / "account-b" / "auth.json" + self._write_auth_record(file_a, "token-a", "acct-a") + self._write_auth_record(file_b, "token-b", "acct-b") + + config: Final = ChatGPTResponsesAPIConfig() + + headers_a: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + litellm_params=GenericLiteLLMParams(chatgpt_auth_file=str(file_a)), + ) + headers_b: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + litellm_params=GenericLiteLLMParams(chatgpt_auth_file=str(file_b)), + ) + headers_default: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + litellm_params=GenericLiteLLMParams(), + ) + + assert headers_a["Authorization"] == "Bearer token-a" + assert headers_a["ChatGPT-Account-Id"] == "acct-a" + assert headers_b["Authorization"] == "Bearer token-b" + assert headers_b["ChatGPT-Account-Id"] == "acct-b" + assert headers_default["Authorization"] == "Bearer token-default" + assert headers_default["ChatGPT-Account-Id"] == "acct-default" diff --git a/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py b/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py index 43a4c26b6d2..2e9e3462bee 100644 --- a/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py +++ b/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py @@ -2,17 +2,24 @@ import base64 import json import time from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final from unittest.mock import MagicMock, mock_open, patch import pytest +import httpx +import respx import litellm from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS, HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS from litellm.llms.chatgpt.authenticator import ( TOKEN_REFRESH_TIMEOUT_SECONDS, Authenticator, + get_cached_authenticator, + get_chatgpt_auth_file, ) from litellm.llms.chatgpt.common_utils import GetAccessTokenError +from litellm.types.router import GenericLiteLLMParams def _make_jwt(payload: dict) -> str: @@ -161,3 +168,144 @@ class TestChatGPTAuthenticator: assert account_id == "acct-123" mock_write.assert_called_once() assert mock_write.call_args[0][0]["account_id"] == "acct-123" + + def test_get_account_id_prefers_stored_value_without_rewriting_file(self, authenticator): + id_token = _make_jwt({"https://api.openai.com/auth": {"chatgpt_account_id": "acct-from-token"}}) + auth_data = json.dumps({"account_id": "acct-stored", "id_token": id_token}) + + with ( + patch("builtins.open", mock_open(read_data=auth_data)), + patch.object(authenticator, "_write_auth_file") as mock_write, + ): + assert authenticator.get_account_id() == "acct-stored" + mock_write.assert_not_called() + + def test_get_account_id_without_login_returns_none(self, authenticator): + with ( + patch("builtins.open", mock_open(read_data="{}")), + patch.object(authenticator, "_write_auth_file") as mock_write, + ): + assert authenticator.get_account_id() is None + mock_write.assert_not_called() + + +def _write_auth_record(path: Path, token: str, account_id: str) -> None: + path.write_text( + json.dumps( + { + "access_token": token, + "account_id": account_id, + "expires_at": 4102444800, + } + ) + ) + + +class TestChatGPTMultiAccountAuthenticator: + @pytest.fixture(autouse=True) + def fixed_clock(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("time.time", lambda: 1700000000) + + def test_explicit_auth_file_overrides_env(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path / "env-dir")) + monkeypatch.setenv("CHATGPT_AUTH_FILE", "env.json") + custom_file: Final = tmp_path / "account-a" / "auth.json" + + authenticator: Final = Authenticator(auth_file=str(custom_file)) + + assert authenticator.auth_file == str(custom_file) + assert authenticator.token_dir == str(custom_file.parent) + assert not custom_file.parent.exists() + authenticator._write_auth_file({"access_token": "token-a"}) + assert json.loads(custom_file.read_text())["access_token"] == "token-a" + + def test_default_auth_file_from_env(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path / "env-dir")) + monkeypatch.setenv("CHATGPT_AUTH_FILE", "env.json") + + authenticator: Final = Authenticator() + + assert authenticator.auth_file == str(tmp_path / "env-dir" / "env.json") + + def test_bare_filename_auth_file_resolves_against_cwd( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.chdir(tmp_path) + + authenticator: Final = Authenticator(auth_file="auth.json") + + assert authenticator.auth_file == str(tmp_path / "auth.json") + assert authenticator.token_dir == str(tmp_path) + + def test_two_auth_files_are_isolated(self, tmp_path: Path) -> None: + file_a: Final = tmp_path / "account-a.json" + file_b: Final = tmp_path / "account-b.json" + _write_auth_record(file_a, "token-a", "acct-a") + _write_auth_record(file_b, "token-b", "acct-b") + + authenticator_a: Final = Authenticator(auth_file=str(file_a)) + authenticator_b: Final = Authenticator(auth_file=str(file_b)) + + assert authenticator_a.get_access_token() == "token-a" + assert authenticator_a.get_account_id() == "acct-a" + assert authenticator_b.get_access_token() == "token-b" + assert authenticator_b.get_account_id() == "acct-b" + + def test_get_cached_authenticator_reuses_instance_per_path(self, tmp_path: Path) -> None: + file_a: Final = tmp_path / "account-a.json" + file_b: Final = tmp_path / "account-b.json" + + authenticator_a: Final = get_cached_authenticator(str(file_a)) + + assert get_cached_authenticator(str(file_a)) is authenticator_a + assert get_cached_authenticator(str(file_b)) is not authenticator_a + + def test_get_chatgpt_auth_file(self) -> None: + assert get_chatgpt_auth_file(None) is None + assert get_chatgpt_auth_file({}) is None + assert get_chatgpt_auth_file({"chatgpt_auth_file": ""}) is None + assert get_chatgpt_auth_file({"chatgpt_auth_file": "/a/auth.json"}) == "/a/auth.json" + assert get_chatgpt_auth_file(GenericLiteLLMParams()) is None + assert get_chatgpt_auth_file(GenericLiteLLMParams(chatgpt_auth_file="/b/auth.json")) == "/b/auth.json" + + def test_relative_and_absolute_auth_file_share_cached_authenticator( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.chdir(tmp_path) + get_cached_authenticator.cache_clear() + auth_file: Final = tmp_path / "x" / "auth.json" + auth_file.parent.mkdir(parents=True) + auth_file.write_text("{}") + + relative_key: Final = get_chatgpt_auth_file({"chatgpt_auth_file": "x/auth.json"}) + absolute_key: Final = get_chatgpt_auth_file({"chatgpt_auth_file": str(auth_file)}) + assert relative_key is not None and absolute_key is not None + + assert get_cached_authenticator(relative_key) is get_cached_authenticator(absolute_key) + + +def test_refresh_keeps_saved_workspace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("time.time", lambda: 1700000000) + auth_file: Final = tmp_path / "selected.json" + auth_file.write_text( + json.dumps( + { + "access_token": "expired-example-token", + "refresh_token": "example-refresh-token", + "expires_at": 1, + "account_id": "acct-selected-workspace", + } + ) + ) + token: Final = _make_jwt({"exp": 4102444800, "https://api.openai.com/auth": {"chatgpt_account_id": "acct-default"}}) + authenticator: Final = Authenticator(auth_file=str(auth_file)) + with respx.mock as transport: + transport.post("https://auth.openai.com/oauth/token").mock( + return_value=httpx.Response( + 200, + json={"access_token": token, "id_token": token, "refresh_token": "example-rotated-refresh"}, + ) + ) + assert authenticator.get_access_token() == token + assert authenticator.get_account_id(token) == "acct-selected-workspace" + assert json.loads(auth_file.read_text())["refresh_token"] == "example-rotated-refresh" diff --git a/tests/unit/llms/chatgpt/test_chatgpt_common_utils.py b/tests/unit/llms/chatgpt/test_chatgpt_common_utils.py new file mode 100644 index 00000000000..6b280d97089 --- /dev/null +++ b/tests/unit/llms/chatgpt/test_chatgpt_common_utils.py @@ -0,0 +1,90 @@ +import base64 +import json + +import pytest + +from litellm.llms.chatgpt.common_utils import ( + extract_chatgpt_account_id, + get_chatgpt_client_credential, + is_chatgpt_oauth_key, + select_chatgpt_client_credential_headers, +) + + +def _make_jwt(payload: dict) -> str: + def _b64(obj: dict) -> str: + raw = json.dumps(obj, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("utf-8").rstrip("=") + + return f"{_b64({'alg': 'none', 'typ': 'JWT'})}.{_b64(payload)}." + + +CHATGPT_JWT = _make_jwt({"https://api.openai.com/auth": {"chatgpt_account_id": "acct-alice"}}) +CHATGPT_JWT_WITHOUT_ACCOUNT = _make_jwt({"https://api.openai.com/auth": {"user_id": "user-1"}}) +NON_CHATGPT_JWT = _make_jwt({"sub": "someone", "https://api.openai.com/auth": "not-an-object"}) + + +class TestExtractChatGPTAccountId: + def test_reads_account_id_from_auth_claim(self): + assert extract_chatgpt_account_id(CHATGPT_JWT) == "acct-alice" + + @pytest.mark.parametrize( + "token", + [None, "", "sk-proj-not-a-jwt", "header.only", NON_CHATGPT_JWT, CHATGPT_JWT_WITHOUT_ACCOUNT], + ) + def test_returns_none_without_a_usable_account_claim(self, token): + assert extract_chatgpt_account_id(token) is None + + +class TestIsChatGPTOAuthKey: + def test_recognizes_bearer_and_raw_chatgpt_jwt(self): + assert is_chatgpt_oauth_key(CHATGPT_JWT) + assert is_chatgpt_oauth_key(f"Bearer {CHATGPT_JWT}") + assert is_chatgpt_oauth_key(f"Bearer {CHATGPT_JWT_WITHOUT_ACCOUNT}") + + @pytest.mark.parametrize( + "value", [None, "", "Bearer sk-proj-platform-key", "Bearer not.a.jwt", _make_jwt({"sub": "x"})] + ) + def test_rejects_non_chatgpt_credentials(self, value): + assert not is_chatgpt_oauth_key(value) + + +class TestSelectChatGPTClientCredentialHeaders: + def test_account_header_alone_is_not_forwarded(self): + selected = select_chatgpt_client_credential_headers( + {"chatgpt-account-id": "acct-alice", "authorization": "Bearer sk-proj-platform-key"} + ) + assert dict(selected) == {} + assert get_chatgpt_client_credential({"chatgpt-account-id": "acct-alice"}) is None + + def test_forwards_only_bearer_and_account_context(self): + selected = select_chatgpt_client_credential_headers( + { + "Authorization": f"Bearer {CHATGPT_JWT}", + "ChatGPT-Account-Id": "acct-explicit", + "x-litellm-api-key": "sk-gateway", + "user-agent": "codex", + } + ) + assert dict(selected) == {"Authorization": f"Bearer {CHATGPT_JWT}", "ChatGPT-Account-Id": "acct-explicit"} + credential = get_chatgpt_client_credential(selected) + assert credential is not None + assert credential.access_token == CHATGPT_JWT + assert credential.account_id == "acct-explicit" + + def test_account_id_falls_back_to_jwt_claim(self): + credential = get_chatgpt_client_credential({"authorization": f"Bearer {CHATGPT_JWT}"}) + assert credential is not None + assert credential.account_id == "acct-alice" + + +@pytest.mark.parametrize("scheme", ["Bearer", "bearer", "BEARER"]) +def test_bearer_scheme_is_case_insensitive(scheme): + credential = get_chatgpt_client_credential({"authorization": f"{scheme} {CHATGPT_JWT}"}) + assert credential is not None + assert credential.access_token == CHATGPT_JWT + assert credential.account_id == "acct-alice" + + +def test_other_authentication_schemes_are_not_chatgpt_logins(): + assert not is_chatgpt_oauth_key(f"Basic {CHATGPT_JWT}") diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index bbcd7f70e63..f7850aaf9a1 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -5654,3 +5654,44 @@ def test_vector_store_retrieve_handler_sends_id_before_azure_api_version_query() sent: Final = received[0] assert str(sent.url) == "https://x.openai.azure.com/openai/vector_stores/vs_abc?api-version=2025-04-01-preview" assert len(received) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", (False, True)) +@pytest.mark.parametrize("provider", ("openai", "azure")) +async def test_non_chatgpt_responses_preserves_caller_headers_after_validation( + is_async: bool, provider: str, respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url__regex=r"https://responses\.example\.com/.*").respond( + 200, + json={ + "id": "resp_headers", "object": "response", "created_at": 1, + "model": "gpt-6-luna", "status": "completed", "output": [], + "parallel_tool_calls": False, "tool_choice": "auto", "tools": [], + }, + ) + client: Final = AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport()) if is_async else HTTPHandler(client=httpx.Client()) + kwargs: Final = { + "model": f"{provider}/gpt-6-luna", + "input": "hello", + "api_key": "fake-configured-key", + "api_base": "https://responses.example.com/v1", + "api_version": "2025-04-01-preview", + "extra_headers": {"Authorization": "Bearer fake-caller-key", "content-type": "application/vnd.example+json"}, + "client": client, + } + try: + response: Final = await litellm.aresponses(**kwargs) if is_async else litellm.responses(**kwargs) + assert response.status == "completed" + assert route.calls[0].request.headers["authorization"] == "Bearer fake-caller-key" + expected_content_types: Final = ( + ["application/json", "application/vnd.example+json"] + if provider == "openai" + else ["application/vnd.example+json"] + ) + assert route.calls[0].request.headers.get_list("content-type") == expected_content_types + finally: + if isinstance(client, AsyncHTTPHandler): + await client.close() + else: + client.client.close() diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index ee1ae43fd3a..10a0f974a42 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -3626,6 +3626,18 @@ class TestIsRequestBodySafeBlocksEndpointTargetingFields: ) +@pytest.mark.parametrize("allow_client_side_credentials", [False, True]) +@pytest.mark.parametrize("nested", [False, True]) +def test_request_cannot_select_chatgpt_auth_file(allow_client_side_credentials: bool, nested: bool) -> None: + selector: Final = {"chatgpt_auth_file": "/tokens/account-b/auth.json"} + with pytest.raises(ValueError, match="chatgpt_auth_file"): + is_request_body_safe( + request_body={"model": "chatgpt-account-a", **({"extra_body": selector} if nested else selector)}, + general_settings={"allow_client_side_credentials": allow_client_side_credentials}, + llm_router=None, + model="chatgpt-account-a", + ) + class TestIsRequestBodySafeBlocksBedrockProjectOverride: """``aws_bedrock_project_id`` pins a deployment to a Bedrock project so that project's data-retention policy applies to its requests. A @@ -5304,3 +5316,27 @@ async def test_a_non_numeric_max_request_size_mb_is_ignored_without_premium(): patch("litellm.proxy.proxy_server.premium_user", False), ): assert await check_if_request_size_is_safe(request=_request_with_content_length(1024)) is True + + +@pytest.mark.parametrize("allow_client_side_credentials", [False, True]) +@pytest.mark.parametrize("route", ["/model/new", "/model/update", "/model/account-a/update", "/health/test_connection"]) +def test_chatgpt_model_configuration_reaches_role_authorization( + allow_client_side_credentials: bool, route: str +) -> None: + body: Final = {"litellm_params": {"model": "chatgpt/gpt-6-luna", "chatgpt_auth_file": "/tokens/account-a/auth.json"}} + assert is_request_body_safe( + request_body=body, + general_settings={"allow_client_side_credentials": allow_client_side_credentials}, + llm_router=None, + model="", + route=route, + ) + for inference_route in ("/v1/chat/completions", "/v1/responses"): + with pytest.raises(ValueError, match="chatgpt_auth_file"): + is_request_body_safe( + request_body=body, + general_settings={"allow_client_side_credentials": allow_client_side_credentials}, + llm_router=None, + model="", + route=inference_route, + ) diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 950d1742d49..67b280eb379 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -5437,3 +5437,64 @@ async def test_health_services_endpoint_langfuse_missing_keys_errors(monkeypatch with pytest.raises(ProxyException, match="auth_check failed") as raised: await health_services_endpoint(service="langfuse") assert "connection refused by lf.internal.example" in str(raised.value.message) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("admin", [False, True]) +async def test_unsaved_chatgpt_account_probe_requires_proxy_admin( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, admin: bool +) -> None: + from starlette.requests import Request + from litellm.proxy import proxy_server + + monkeypatch.setenv("CHATGPT_AUTH_FILE", str(tmp_path / "missing-default-auth.json")) + auth_file: Final = tmp_path / "auth.json" + auth_file.write_text( + json.dumps({"access_token": "token-probe", "account_id": "acct-probe", "expires_at": 4102444800}) + ) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + output: Final = { + "id": "msg_probe", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + response: Final = { + "id": "resp_probe", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-6.1-sol", + "output": [output], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + with respx.mock(assert_all_called=False) as upstream: + upstream.post("https://chatgpt.com/backend-api/codex/responses").respond( + 200, + text="data: " + json.dumps({"type": "response.completed", "response": response}) + "\n\n", + headers={"content-type": "text/event-stream"}, + ) + call: Final = health_test_model_connection( + request=Request({"type": "http", "headers": []}), + mode="responses", + litellm_params={"model": "chatgpt/gpt-6.1-sol", "chatgpt_auth_file": str(auth_file)}, + model_info={}, + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN if admin else LitellmUserRoles.INTERNAL_USER, token="test-key" + ), + ) + if admin: + result: Final = await call + assert result["status"] == "success" + assert upstream.calls.last.request.headers["Authorization"] == "Bearer token-probe" + assert upstream.calls.last.request.headers["ChatGPT-Account-Id"] == "acct-probe" + assert isinstance(json.loads(upstream.calls.last.request.content)["input"], list) + else: + with pytest.raises(ProxyException) as error: + await call + assert error.value.code == "403" + assert len(upstream.calls) == 0 diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 303b8e060ed..a718426eea6 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -8933,7 +8933,14 @@ class TestNonAdminCannotPersistWifFieldsOnModel: mock_prisma.db.litellm_proxymodeltable.update.assert_not_called() @pytest.mark.asyncio - async def test_patch_model_non_admin_cannot_set_wif_field(self): + @pytest.mark.parametrize( + "field,value", + [ + ("anthropic_keycloak_token_url", "https://identity.example.com/token"), + ("chatgpt_auth_file", "/tokens/account-b/auth.json"), + ], + ) + async def test_patch_model_non_admin_cannot_set_wif_field(self, field: str, value: str) -> None: from litellm.proxy.management_endpoints.model_management_endpoints import ( patch_model, ) @@ -8979,13 +8986,13 @@ class TestNonAdminCannotPersistWifFieldsOnModel: model_id="m1", patch_data=updateDeployment( litellm_params=updateLiteLLMParams( - anthropic_keycloak_token_url="https://attacker.example/token", + **{field: value}, ) ), user_api_key_dict=non_admin, ) err = exc_info.value - assert getattr(err, "param", "") == "anthropic_keycloak_token_url" + assert getattr(err, "param", "") == field mock_prisma.db.litellm_proxymodeltable.update.assert_not_called() @pytest.mark.asyncio @@ -9044,7 +9051,14 @@ class TestNonAdminCannotPersistWifFieldsOnModel: mock_prisma.db.litellm_proxymodeltable.update.assert_not_called() @pytest.mark.asyncio - async def test_patch_model_admin_can_set_wif_field(self): + @pytest.mark.parametrize( + "field,value", + [ + ("anthropic_keycloak_token_url", "https://identity.example.com/token"), + ("chatgpt_auth_file", "/tokens/account-b/auth.json"), + ], + ) + async def test_patch_model_admin_can_set_wif_field(self, field: str, value: str) -> None: from litellm.proxy.management_endpoints.model_management_endpoints import ( patch_model, ) @@ -9095,7 +9109,7 @@ class TestNonAdminCannotPersistWifFieldsOnModel: model_id="m1", patch_data=updateDeployment( litellm_params=updateLiteLLMParams( - anthropic_keycloak_token_url="https://keycloak.internal/token", + **{field: value}, ) ), user_api_key_dict=admin, diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py new file mode 100644 index 00000000000..354c65776dc --- /dev/null +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -0,0 +1,185 @@ +import asyncio +import json +from collections.abc import Iterator +from datetime import datetime +from typing import Final + +import httpx +import pytest +import respx +from fastapi.testclient import TestClient + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.proxy import proxy_server +from tests._master_key import MASTER_KEY + +_BASE: Final = "https://chatgpt.com/backend-api/codex" +_TOKEN: Final = "test-subscription-token" +_HEADERS: Final = {"authorization": f"Bearer {_TOKEN}", "x-litellm-api-key": MASTER_KEY} + + +@pytest.fixture +def client(monkeypatch: pytest.MonkeyPatch, httpx_transport: None) -> Iterator[TestClient]: + monkeypatch.setattr(proxy_server, "master_key", MASTER_KEY) + yield TestClient(proxy_server.app) + + +def test_native_catalog_preserves_account_and_filters_gateway_credentials( + client: TestClient, respx_mock: respx.MockRouter +) -> None: + catalog: Final = {"models": [{"slug": "subscription-model", "supported_reasoning_levels": ["high"]}]} + upstream: Final = respx_mock.get(f"{_BASE}/models", params={"client_version": "1.0.0"}).respond(json=catalog) + response: Final = client.get( + "/backend-api/codex/models?client_version=1.0.0", + headers={ + **_HEADERS, + "chatgpt-account-id": "account-a", + "cookie": "session=test-cookie", + "x-pass-authorization": "Bearer test-injected-token", + "x-pass-x-litellm-api-key": "test-injected-key", + }, + ) + assert response.status_code == 200 + assert response.json() == catalog + assert upstream.calls.last.request.headers["authorization"] == f"Bearer {_TOKEN}" + assert upstream.calls.last.request.headers["chatgpt-account-id"] == "account-a" + assert "x-litellm-api-key" not in upstream.calls.last.request.headers + assert "cookie" not in upstream.calls.last.request.headers + + +@pytest.mark.parametrize("account", ["account-a", "account-b"]) +def test_native_responses_preserves_body_and_stream( + client: TestClient, respx_mock: respx.MockRouter, account: str +) -> None: + body: Final = { + "model": "subscription-model", + "input": [{"role": "user", "content": "Hello"}], + "reasoning": {"effort": "high"}, + "store": False, + "stream": True, + } + stream: Final = ( + b'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"Hello"}\n\n' + ) + upstream: Final = respx_mock.post(f"{_BASE}/responses").respond( + content=stream, headers={"content-type": "text/event-stream"} + ) + response: Final = client.post( + "/backend-api/codex/responses", + json=body, + headers={**_HEADERS, "authorization": f"Bearer test-{account}", "chatgpt-account-id": account}, + ) + assert response.status_code == 200 + assert response.content == stream + assert response.headers["content-type"].startswith("text/event-stream") + assert json.loads(upstream.calls.last.request.content) == body + assert upstream.calls.last.request.headers["authorization"] == f"Bearer test-{account}" + assert upstream.calls.last.request.headers["chatgpt-account-id"] == account + + +@pytest.mark.parametrize("gateway_key", [None, "sk-invalid-test-key"]) +def test_gateway_auth_rejects_before_upstream( + client: TestClient, respx_mock: respx.MockRouter, gateway_key: str | None +) -> None: + headers: Final = { + "authorization": f"Bearer {_TOKEN}", + **({"x-litellm-api-key": gateway_key} if gateway_key else {}), + } + response: Final = client.get("/backend-api/codex/models", headers=headers) + assert response.status_code in (400, 401, 403) + assert not respx_mock.calls + + +@pytest.mark.parametrize("authorization", ["", "Basic test", "Bearer ", f"Bearer {MASTER_KEY}"]) +def test_subscription_credential_required(client: TestClient, respx_mock: respx.MockRouter, authorization: str) -> None: + response: Final = client.get("/backend-api/codex/models", headers={**_HEADERS, "authorization": authorization}) + assert response.status_code == 401 + assert not respx_mock.calls + + +@pytest.mark.parametrize("status", [401, 403, 429]) +def test_provider_errors_do_not_fall_back(client: TestClient, respx_mock: respx.MockRouter, status: int) -> None: + upstream: Final = respx_mock.post(f"{_BASE}/responses").respond( + status, json={"error": {"message": "subscription unavailable"}}, headers={"retry-after": "60"} + ) + response: Final = client.post( + "/backend-api/codex/responses", headers=_HEADERS, json={"model": "subscription-model", "stream": True} + ) + assert response.status_code == status + assert upstream.call_count == 1 + + +@pytest.mark.parametrize("method,path", [("POST", "models"), ("GET", "responses"), ("GET", "deviceauth")]) +def test_only_native_inference_routes_are_forwarded( + client: TestClient, respx_mock: respx.MockRouter, method: str, path: str +) -> None: + response: Final = client.request(method, f"/backend-api/codex/{path}", headers=_HEADERS) + assert response.status_code in (405, 422) + assert not respx_mock.calls + + +class UsageRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.events: Final[asyncio.Queue[dict[str, object]]] = asyncio.Queue() + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + await self.events.put(kwargs) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_usage_logged_without_credentials( + monkeypatch: pytest.MonkeyPatch, httpx_transport: None, respx_mock: respx.MockRouter, stream: bool +) -> None: + monkeypatch.setattr(proxy_server, "master_key", MASTER_KEY) + recorder: Final = UsageRecorder() + monkeypatch.setattr(litellm, "_async_success_callback", [recorder]) + litellm.register_model( + { + "chatgpt/subscription-model": { + "litellm_provider": "chatgpt", + "mode": "responses", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + } + ) + result: Final = { + "id": "resp_test", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "subscription-model", + "output": [], + "usage": {"input_tokens": 12, "output_tokens": 3, "total_tokens": 15}, + } + event: Final = {"type": "response.completed", "response": result} + response_body: Final = ( + httpx.Response( + 200, + content=f"event: response.completed\ndata: {json.dumps(event)}\n\n", + headers={"content-type": "text/event-stream"}, + ) + if stream + else httpx.Response(200, json=result) + ) + respx_mock.post(f"{_BASE}/responses").mock(return_value=response_body) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=proxy_server.app), base_url="http://test") as caller: + response: Final = await caller.post( + "/backend-api/codex/responses", + json={"model": "subscription-model", "stream": stream, "input": "Hello"}, + headers={**_HEADERS, "x-pass-authorization": "Bearer test-hidden-token", "cookie": "test-hidden-cookie"}, + ) + assert response.status_code == 200 + logged: Final = await asyncio.wait_for(recorder.events.get(), timeout=5) + payload: Final = logged["standard_logging_object"] + assert isinstance(payload, dict) + assert payload["prompt_tokens"] == 12 + assert payload["completion_tokens"] == 3 + serialized: Final = json.dumps(logged, default=str) + for secret in (_TOKEN, MASTER_KEY, "test-hidden-token", "test-hidden-cookie"): + assert secret not in serialized diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 64ff8392c02..ed7817bd660 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -1,4 +1,5 @@ import asyncio +import base64 import copy import json import os @@ -9208,6 +9209,97 @@ def test_default_team_settings_bool_turn_off_message_logging_redacts(): ) +def _fake_chatgpt_oauth_token(account_id: str) -> str: + header = base64.urlsafe_b64encode(b'{"alg":"RS256"}').rstrip(b"=").decode() + payload = ( + base64.urlsafe_b64encode( + json.dumps({"https://api.openai.com/auth": {"chatgpt_account_id": account_id}, "exp": 4102444800}).encode() + ) + .rstrip(b"=") + .decode() + ) + return f"{header}.{payload}.fake-signature" + + +CHATGPT_OAUTH_TOKEN = f"Bearer {_fake_chatgpt_oauth_token('acct-alice')}" +CHATGPT_ACCOUNT_ID_HEADER_NAME = "ChatGPT-Account-Id" +PLATFORM_API_KEY_BEARER = "Bearer sk-proj-fake-platform-key-0123456789" +CHATGPT_LEAK_TARGET_PROVIDERS = ["openai", "azure", "anthropic", "bedrock", "vertex_ai"] + + +def _codex_client_headers( + authorization_header_name: str = "authorization", account_id: str | None = "acct-alice" +) -> dict: + return { + "content-type": "application/json", + "originator": "codex_cli_rs", + authorization_header_name: CHATGPT_OAUTH_TOKEN, + **({CHATGPT_ACCOUNT_ID_HEADER_NAME: account_id} if account_id is not None else {}), + } + + +def test_clean_headers_keeps_chatgpt_oauth_bearer_when_gateway_key_is_separate(): + result = clean_headers( + headers=Headers({**_codex_client_headers(), "x-litellm-api-key": "sk-proxy-virtual-key"}), + litellm_key_header_name="x-litellm-api-key", + forward_llm_provider_auth_headers=False, + authenticated_with_header="x-litellm-api-key", + ) + + assert result["authorization"] == CHATGPT_OAUTH_TOKEN + assert "x-litellm-api-key" not in result + + +def test_clean_headers_drops_bearer_that_authenticated_to_the_gateway(): + result = clean_headers( + headers=Headers({"authorization": CHATGPT_OAUTH_TOKEN, "content-type": "application/json"}), + litellm_key_header_name=None, + forward_llm_provider_auth_headers=False, + authenticated_with_header="authorization", + ) + + assert _authorization_values(result) == [] + + +@pytest.mark.parametrize("bearer", [PLATFORM_API_KEY_BEARER, "Bearer not.a-jwt", "Bearer a.bm90LWpzb24.c"]) +def test_clean_headers_drops_non_chatgpt_bearers(bearer): + result = clean_headers( + headers=Headers({"authorization": bearer, "x-litellm-api-key": "sk-proxy-virtual-key"}), + litellm_key_header_name="x-litellm-api-key", + forward_llm_provider_auth_headers=False, + authenticated_with_header="x-litellm-api-key", + ) + + assert _authorization_values(result) == [] + + +@pytest.mark.parametrize("authorization_header_name", AUTHORIZATION_HEADER_CASINGS) +def test_chatgpt_oauth_credential_and_account_reach_chatgpt_provider(authorization_header_name): + forwarded = _headers_forwarded_to(_codex_client_headers(authorization_header_name), "chatgpt") + + assert _authorization_values(forwarded) == [CHATGPT_OAUTH_TOKEN] + assert forwarded[CHATGPT_ACCOUNT_ID_HEADER_NAME] == "acct-alice" + + +@pytest.mark.parametrize("custom_llm_provider", CHATGPT_LEAK_TARGET_PROVIDERS) +def test_chatgpt_oauth_credential_is_never_forwarded_to_other_providers(custom_llm_provider): + forwarded = _headers_forwarded_to(_codex_client_headers(), custom_llm_provider) + + assert _authorization_values(forwarded) == [] + assert CHATGPT_OAUTH_TOKEN not in forwarded.values() + assert CHATGPT_ACCOUNT_ID_HEADER_NAME not in forwarded + + +def test_chatgpt_account_header_is_not_forwarded_without_chatgpt_oauth_bearer(): + headers = {**_codex_client_headers(), "authorization": PLATFORM_API_KEY_BEARER} + + assert _headers_forwarded_to(headers, "chatgpt") == {} + + +def test_anthropic_oauth_bearer_does_not_reach_chatgpt_provider(): + assert _authorization_values(_headers_forwarded_to(_client_headers(), "chatgpt")) == [] + + def test_add_user_api_key_auth_to_request_metadata_attributes_a_cli_session_to_its_alias(): data = {"model": "gpt-5.4-nano", "messages": [{"role": "user", "content": "hi"}], "litellm_metadata": {}} session = UserAPIKeyAuth( diff --git a/tests/unit/responses/test_responses_utils.py b/tests/unit/responses/test_responses_utils.py index 97a39f64001..0863abd1f1e 100644 --- a/tests/unit/responses/test_responses_utils.py +++ b/tests/unit/responses/test_responses_utils.py @@ -933,6 +933,42 @@ class TestResponsesInputToChatMessages: ) == [user_message] +class TestMergeProviderSpecificHeaders: + def test_scoped_headers_merge_only_for_matching_provider(self): + scoped = {"custom_llm_provider": "chatgpt", "extra_headers": {"Authorization": "Bearer client-token"}} + + merged = ResponsesAPIRequestUtils.merge_provider_specific_headers( + extra_headers={"originator": "codex_cli_rs"}, + provider_specific_header=scoped, + custom_llm_provider="chatgpt", + ) + untouched = ResponsesAPIRequestUtils.merge_provider_specific_headers( + extra_headers={"originator": "codex_cli_rs"}, + provider_specific_header=scoped, + custom_llm_provider="openai", + ) + + assert merged == {"originator": "codex_cli_rs", "Authorization": "Bearer client-token"} + assert untouched == {"originator": "codex_cli_rs"} + + def test_sequence_of_scoped_entries_each_keep_their_own_scope(self): + entries = [ + {"custom_llm_provider": "anthropic", "extra_headers": {"Authorization": "Bearer anthropic-token"}}, + {"custom_llm_provider": "chatgpt", "extra_headers": {"Authorization": "Bearer chatgpt-token"}}, + ] + + merged = ResponsesAPIRequestUtils.merge_provider_specific_headers( + extra_headers=None, provider_specific_header=entries, custom_llm_provider="chatgpt" + ) + + assert merged == {"Authorization": "Bearer chatgpt-token"} + + def test_no_scoped_headers_returns_extra_headers_unchanged(self): + assert ResponsesAPIRequestUtils.merge_provider_specific_headers( + extra_headers=None, provider_specific_header=None, custom_llm_provider="chatgpt" + ) is None + + def test_responses_object_bills_an_explicit_cache_hit_at_the_explicit_rate(): model = "dashscope/qwen-responses-explicit-cache-test" previous_row = litellm.model_cost.get(model) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 4814d36ee39..6aaa3379f47 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -283,6 +283,15 @@ def test_update_kwargs_does_not_mutate_defaults_and_merges_metadata(): assert kwargs["litellm_metadata"] == {"baz": 123} +def test_router_model_group_llm_provider_resolves_chatgpt_without_login(tmp_path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + params: Final = LiteLLM_Params(model="chatgpt/gpt-5.4") + + assert Router._model_group_llm_provider(params) == ("gpt-5.4", "chatgpt") + assert Router._model_group_llm_provider(LiteLLM_Params(model="no-such-provider-model-xyz")) == ("", "") + assert not any(tmp_path.iterdir()) + + def test_router_with_model_info_and_model_group(): """ Test edge case where user specifies model_group in model_info diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index b0086662f1c..740342f54d9 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -94,6 +94,7 @@ CONNECTION_NAMES: Final = ( "github_copilot_host", "configurable_clientside_auth_params", "use_xai_oauth", + "chatgpt_auth_file", "fireworks_forward_user_id", "aws_batch_role_arn", "aws_bedrock_control_plane_endpoint", diff --git a/tests/unit/types/test_router.py b/tests/unit/types/test_router.py index 2892a829eab..7855eba0042 100644 --- a/tests/unit/types/test_router.py +++ b/tests/unit/types/test_router.py @@ -16,8 +16,8 @@ from litellm.types.router import ( holds_secret_pointer, reject_server_owned_wif_params, server_owned_wif_fields_named, - server_owned_wif_fields_written, server_owned_wif_fields_present, + server_owned_wif_fields_written, ) from litellm.types.utils import ( CustomPricingLiteLLMParams, @@ -367,6 +367,7 @@ def test_server_owned_registry_includes_anthropic_openai_and_oauth_token_exchang + openai_wif_litellm_params + oauth_token_exchange_litellm_params + github_copilot_oauth_litellm_params + + ("chatgpt_auth_file",) ) assert set(openai_wif_litellm_params) == { "openai_identity_provider_id", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 262e8b6deae..d3c1e0422df 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1792,6 +1792,30 @@ export interface paths { patch: operations["azure_speech_proxy_route_azure_speech__endpoint__patch"]; trace?: never; }; + "/backend-api/codex/{endpoint}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Codex Passthrough Route + * @description Forward a client's ChatGPT subscription with separate LiteLLM authentication. + */ + get: operations["codex_passthrough_route_backend_api_codex__endpoint__get"]; + put?: never; + /** + * Codex Passthrough Route + * @description Forward a client's ChatGPT subscription with separate LiteLLM authentication. + */ + post: operations["codex_passthrough_route_backend_api_codex__endpoint__post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/batches": { parameters: { query?: never; @@ -35309,6 +35333,11 @@ export interface components { cache_read_input_token_cost_priority?: number | null; /** Cache Read Input Token Cost Ultrafast */ cache_read_input_token_cost_ultrafast?: number | null; + /** + * Chatgpt Auth File + * @description Path to a ChatGPT OAuth auth.json for this deployment. + */ + chatgpt_auth_file?: string | null; /** Citation Cost Per Token */ citation_cost_per_token?: number | null; /** Client Id */ @@ -50660,6 +50689,11 @@ export interface components { cache_read_input_token_cost_priority?: number | null; /** Cache Read Input Token Cost Ultrafast */ cache_read_input_token_cost_ultrafast?: number | null; + /** + * Chatgpt Auth File + * @description Path to a ChatGPT OAuth auth.json for this deployment. + */ + chatgpt_auth_file?: string | null; /** Citation Cost Per Token */ citation_cost_per_token?: number | null; /** Client Id */ @@ -53737,6 +53771,77 @@ export interface operations { }; }; }; + codex_passthrough_route_backend_api_codex__endpoint__get: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: "models" | "responses"; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + codex_passthrough_route_backend_api_codex__endpoint__post: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: "models" | "responses"; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + [key: string]: unknown; + }; + }; + }; + responses: { + /** @description Unmodified Codex response or server-sent events */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + [key: string]: unknown; + }; + "text/event-stream": string; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; list_batches_batches_get: { parameters: { query?: { From ffb79b10ee706ebd5a7f4a5784deb0ad127fc2a3 Mon Sep 17 00:00:00 2001 From: Zap777 <124182405+devanshc777@users.noreply.github.com> Date: Tue, 27 Oct 2026 04:30:02 +0530 Subject: [PATCH 079/155] fix(mcp): send tool-call stream headers once the call is authorized (#47897) OAuthScopeResponse holds a tools/call response's http.response.start so a managed OAuth scope challenge can replace it with a 403. Once the call was allowed, the held start still waited for the next body message, which for a slow tool is the 15s SSE keep-alive ping. Send it as soon as the call is allowed instead. Fixes #46659 --- .../mcp_server/scope_challenge.py | 51 +++++---- .../test_mcp_oauth_passthrough_tools.py | 108 ++++++++++++++++-- .../mcp_server/test_mcp_server_manager.py | 4 +- .../test_mcp_server_tool_calls_and_headers.py | 3 +- 4 files changed, 127 insertions(+), 39 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/scope_challenge.py b/litellm/proxy/_experimental/mcp_server/scope_challenge.py index c326418bee7..121234e305d 100644 --- a/litellm/proxy/_experimental/mcp_server/scope_challenge.py +++ b/litellm/proxy/_experimental/mcp_server/scope_challenge.py @@ -33,9 +33,8 @@ class OAuthScopeResponse: self._request_path = request_path self._ready = asyncio.Event() self._lock = asyncio.Lock() - self._start: Message | None = None self._challenge: MCPUpstreamAuthError | None = None - self._refused = False + self._closed = False @property def pending(self) -> bool: @@ -50,31 +49,33 @@ class OAuthScopeResponse: self._ready.set() async def __call__(self, message: Message) -> None: - if message["type"] == "http.response.start": - self._start = message - if message["status"] != 200: - self.allow() - return - await self._ready.wait() async with self._lock: - if self._refused: + if self._closed: return - if self._start is not None: - start: Final = self._start - self._start = None - if self._challenge is not None: - error: Final = self._challenge.to_http_exception(self._base_url, self._request_path) - response: Final = JSONResponse( - status_code=error.status_code, content={"detail": error.detail}, headers=error.headers - ) - self._refused = True - await self._send( - {"type": "http.response.start", "status": response.status_code, "headers": response.raw_headers} - ) - await self._send({"type": "http.response.body", "body": bytes(response.body)}) - return - await self._send(start) - await self._send(message) + try: + if message["type"] == "http.response.start": + if message["status"] != 200: + self.allow() + await self._ready.wait() + if self._challenge is not None: + error: Final = self._challenge.to_http_exception(self._base_url, self._request_path) + response: Final = JSONResponse( + status_code=error.status_code, content={"detail": error.detail}, headers=error.headers + ) + self._closed = True + await self._send( + { + "type": "http.response.start", + "status": response.status_code, + "headers": response.raw_headers, + } + ) + await self._send({"type": "http.response.body", "body": bytes(response.body)}) + return + await self._send(message) + except BaseException: + self._closed = True + raise def scope_response_from_request(request: object) -> OAuthScopeResponse | None: diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py index b0d53f9c93b..34b021df2b9 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py @@ -5,6 +5,7 @@ from litellm.proxy._experimental.mcp_server import operations as mcp_operations import asyncio import logging import sys +from typing import Final from unittest.mock import AsyncMock, MagicMock import httpx @@ -531,12 +532,12 @@ async def test_scope_challenge_replaces_pending_stream_headers_once(): send = AsyncMock() response = OAuthScopeResponse(send, "https://gateway.example", "/docs/mcp") - await response({"type": "http.response.start", "status": 200, "headers": []}) + headers: Final = asyncio.create_task(response({"type": "http.response.start", "status": 200, "headers": []})) body = asyncio.create_task(response({"type": "http.response.body", "body": b"event: message\n"})) await asyncio.sleep(0) send.assert_not_awaited() response.deny(MCPUpstreamAuthError(403, None, "docs", required_scope="tools.write")) - await asyncio.wait_for(body, 1) + await asyncio.wait_for(asyncio.gather(headers, body), 1) await response({"type": "http.response.body", "body": b"private tool error"}) assert send.await_count == 2 start, result = [call.args[0] for call in send.await_args_list] @@ -567,7 +568,7 @@ async def test_scope_authorization_releases_progress_without_waiting_for_tool_re context = ServerRequestContext(session=MagicMock(), lifespan_context={}, protocol_version="2025-11-25", method="tools/call", request=request) token = active_mcp_request_ctx_var.set(context) try: - await gate({"type": "http.response.start", "status": 200, "headers": []}) + headers: Final = asyncio.create_task(gate({"type": "http.response.start", "status": 200, "headers": []})) progress = asyncio.create_task(gate({"type": "http.response.body", "body": b"progress", "more_body": True})) await asyncio.sleep(0) for method, status in [("initialize", 200), ("tools/list", 200), ("tools/call", 403)]: @@ -582,7 +583,7 @@ async def test_scope_authorization_releases_progress_without_waiting_for_tool_re upstream = MCPClient(server_url="https://upstream.example/mcp") async with upstream._create_httpx_client_factory(transport=httpx2.MockTransport(lambda request: httpx2.Response(200, content=b"tool result")))() as client: async with client.stream("POST", "https://upstream.example/mcp", json={"method": "tools/call"}): - await asyncio.wait_for(progress, 1) + await asyncio.wait_for(asyncio.gather(headers, progress), 1) assert send.await_args_list[-1].args[0]["body"] == b"progress" gate.deny(MCPUpstreamAuthError(403, None, "docs", required_scope="tools.write")) await gate({"type": "http.response.body", "body": b"result"}) @@ -594,21 +595,105 @@ async def test_scope_authorization_releases_progress_without_waiting_for_tool_re @pytest.mark.asyncio @pytest.mark.parametrize("failure", [ValueError, asyncio.CancelledError]) -async def test_scope_middleware_releases_waiting_response_on_validation_or_cancellation(failure): +@pytest.mark.parametrize("fail_send", [False, True]) +async def test_scope_middleware_releases_waiting_response_on_validation_or_cancellation( + failure: type[BaseException], fail_send: bool, +) -> None: from mcp.server.context import ServerRequestContext from starlette.requests import Request from litellm.proxy._experimental.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse, finish_scope_response - send = AsyncMock() - gate = OAuthScopeResponse(send, "https://gateway.example", "/mcp/docs") - context = ServerRequestContext(session=MagicMock(), lifespan_context={}, protocol_version="2025-11-25", method="tools/call", request=Request({"type": "http", SCOPE_RESPONSE_KEY: gate})) + send: Final = AsyncMock(side_effect=BrokenPipeError("peer disconnected") if fail_send else None) + gate: Final = OAuthScopeResponse(send, "https://gateway.example", "/mcp/docs") + context: Final = ServerRequestContext(session=MagicMock(), lifespan_context={}, protocol_version="2025-11-25", method="tools/call", request=Request({"type": "http", SCOPE_RESPONSE_KEY: gate})) + headers: Final = asyncio.create_task(gate({"type": "http.response.start", "status": 200, "headers": []})) + await asyncio.sleep(0) with pytest.raises(failure): await finish_scope_response(context, AsyncMock(side_effect=failure)) - await gate({"type": "http.response.start", "status": 200, "headers": []}) + if fail_send: + with pytest.raises(BrokenPipeError, match="peer disconnected"): + await headers + else: + await headers await gate({"type": "http.response.body", "body": b"error"}) assert not gate.pending - assert send.await_args_list[-1].args[0]["body"] == b"error" + assert send.await_args_list[-1].args[0]["type"] == ("http.response.start" if fail_send else "http.response.body") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fail_send", [False, True]) +async def test_scope_transport_owns_header_send_cancellation_and_errors(fail_send: bool) -> None: + import httpx2 + from starlette.types import Message + + from litellm.proxy._experimental.mcp_server.legacy_callbacks import record_upstream_tool_authorization + from litellm.proxy._experimental.mcp_server.scope_challenge import OAuthScopeResponse + + send_started: Final = asyncio.Event() + send_stopped: Final = asyncio.Event() + writable: Final = asyncio.Event() + sent: Final[list[Message]] = [] + + async def send(message: Message) -> None: + sent.append(message) + assert message["type"] == "http.response.start" + send_started.set() + try: + if fail_send: + raise BrokenPipeError("peer disconnected") + await writable.wait() + finally: + send_stopped.set() + + gate: Final = OAuthScopeResponse(send, "https://gateway.example", "/mcp/docs") + headers: Final = asyncio.create_task(gate({"type": "http.response.start", "status": 200, "headers": []})) + body: Final = asyncio.create_task(gate({"type": "http.response.body", "body": b"result"})) + await asyncio.sleep(0) + assert not send_started.is_set() + response: Final = httpx2.Response(200, request=httpx2.Request( + "POST", "https://upstream.example/mcp", json={"method": "tools/call"}, + )) + try: + await record_upstream_tool_authorization(response, pending=gate) + await send_started.wait() + if fail_send: + with pytest.raises(BrokenPipeError, match="peer disconnected"): + await headers + else: + headers.cancel() + with pytest.raises(asyncio.CancelledError): + await headers + await body + assert len(sent) == 1 + assert send_stopped.is_set() + finally: + writable.set() + await send_stopped.wait() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("allowed_before_start", [False, True]) +async def test_scope_authorization_sends_stream_headers_before_the_first_body_chunk(allowed_before_start: bool) -> None: + from starlette.types import Message + + from litellm.proxy._experimental.mcp_server.scope_challenge import OAuthScopeResponse + + send: Final = AsyncMock() + gate: Final = OAuthScopeResponse(send, "https://gateway.example", "/mcp/docs") + start: Final[Message] = {"type": "http.response.start", "status": 200, "headers": [(b"content-type", b"text/event-stream")]} + if allowed_before_start: + gate.allow() + headers: Final = asyncio.create_task(gate(start)) + await asyncio.sleep(0) + if not allowed_before_start: + send.assert_not_awaited() + gate.allow() + await headers + send.assert_awaited_once_with(start) + await gate({"type": "http.response.body", "body": b"event: message\n", "more_body": True}) + assert send.await_args_list[-1].args[0]["body"] == b"event: message\n" + assert send.await_count == 2 @pytest.mark.asyncio @@ -697,13 +782,14 @@ async def test_scope_challenges_are_isolated_between_concurrent_requests(): context = ServerRequestContext(session=MagicMock(), lifespan_context={}, protocol_version="2025-11-25", method="tools/call", request=Request({"type": "http", SCOPE_RESPONSE_KEY: gate})) token = active_mcp_request_ctx_var.set(context) try: - await gate({"type": "http.response.start", "status": 200, "headers": []}) + headers: Final = asyncio.create_task(gate({"type": "http.response.start", "status": 200, "headers": []})) await asyncio.sleep(0) assert get_scope_response() is gate if denied: gate.deny(MCPUpstreamAuthError(403, None, "docs", required_scope="tools.write")) else: await record_upstream_tool_authorization(httpx2.Response(200, request=httpx2.Request("POST", "https://upstream.example/mcp", json={"method": "tools/call"})), pending=get_scope_response()) + await headers await gate({"type": "http.response.body", "body": b"done"}) return send.await_args_list[0].args[0]["status"] finally: diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 7315315a619..ecd08e17cda 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -7681,7 +7681,7 @@ class TestMCPServerManager: request=Request({"type": "http", SCOPE_RESPONSE_KEY: gate}), ) token = active_mcp_request_ctx_var.set(context) - await gate({"type": "http.response.start", "status": 200, "headers": []}) + headers: Final = asyncio.create_task(gate({"type": "http.response.start", "status": 200, "headers": []})) progress = asyncio.create_task(gate({"type": "http.response.body", "body": b"progress", "more_body": True})) async def upstream_call(*args, **kwargs): @@ -7701,7 +7701,7 @@ class TestMCPServerManager: assert not result.is_error finally: gate.allow() - await asyncio.wait_for(progress, 1) + await asyncio.wait_for(asyncio.gather(headers, progress), 1) active_mcp_request_ctx_var.reset(token) @pytest.mark.asyncio diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py index b8e28c3f9b8..a873e2ab1ae 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py @@ -280,7 +280,7 @@ async def test_mcp_server_tool_call_relays_upstream_auth_error_as_iserror( send = AsyncMock() response = OAuthScopeResponse(send, "https://gateway.example", "/mcp/pt") request = Request({"type": "http", "headers": [], SCOPE_RESPONSE_KEY: response}) - await response({"type": "http.response.start", "status": 200, "headers": []}) + headers: Final = asyncio.create_task(response({"type": "http.response.start", "status": 200, "headers": []})) try: from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError from litellm.proxy._experimental.mcp_server.server import ( @@ -328,6 +328,7 @@ async def test_mcp_server_tool_call_relays_upstream_auth_error_as_iserror( info_calls = [str(c.args[0]) for c in mock_logger.info.call_args_list if c.args] assert any("Upstream auth failure" in m for m in info_calls) response.allow() + await headers await response({"type": "http.response.body", "body": b"tool error"}) start = send.await_args_list[0].args[0] assert start["status"] == (403 if required_scope is not None else 200) From 593c640971728851d5dbf56941a26ef73e8095c6 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 26 Oct 2026 19:00:38 -0400 Subject: [PATCH 080/155] fix(models): declare Azure audio snapshot capabilities (#48475) --- .../model_prices_and_context_window_backup.json | 4 ++++ model_prices_and_context_window.json | 4 ++++ .../test_model_prices_and_context_window.py | 17 +++++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b7e0c881455..d9df9d97d7b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6206,6 +6206,8 @@ "text", "audio" ], + "supports_audio_input": true, + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6270,6 +6272,8 @@ "text", "audio" ], + "supports_audio_input": true, + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b7e0c881455..d9df9d97d7b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6206,6 +6206,8 @@ "text", "audio" ], + "supports_audio_input": true, + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6270,6 +6272,8 @@ "text", "audio" ], + "supports_audio_input": true, + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index c0abb18ca8c..45222b2c9ab 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -27,6 +27,23 @@ def prices() -> dict: return json.loads(PRICES_PATH.read_text()) +@pytest.mark.parametrize("model", ("azure/gpt-audio-1.5-2026-02-23", "azure/gpt-audio-mini-2025-10-06")) +def test_azure_audio_snapshot_capabilities_match_declared_modalities(model: str, local_model_cost_map: None) -> None: + catalog: Final = TypeAdapter(dict[str, JsonValue]).validate_json(PRICES_PATH.read_bytes()) + backup: Final = TypeAdapter(dict[str, JsonValue]).validate_json(BACKUP_PRICES_PATH.read_bytes()) + entry: Final = catalog[model] + assert isinstance(entry, dict) + assert entry == backup[model] + input_modalities: Final = entry["supported_modalities"] + output_modalities: Final = entry["supported_output_modalities"] + assert isinstance(input_modalities, list) + assert isinstance(output_modalities, list) + expected: Final = ("audio" in input_modalities, "audio" in output_modalities) + info: Final = litellm.get_model_info(model) + assert (info["supports_audio_input"], info["supports_audio_output"]) == expected + assert (litellm.supports_audio_input(model), litellm.supports_audio_output(model)) == expected + + class _CopilotEndpointRow(BaseModel): model_config = ConfigDict(frozen=True, extra="ignore") From db9eda19b4bb7a8c30cd93a037b45efd0b83b6d3 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 26 Oct 2026 16:04:33 -0700 Subject: [PATCH 081/155] fix(http_handler): name the timeout phase and limit that fired instead of "after None seconds" (#48250) * fix(http_handler): name the timeout phase and the limit that fired instead of 'after None seconds' Every HTTP verb on both handlers raised litellm.Timeout with the per-call timeout argument interpolated into the message, which is None whenever the caller relied on the client default, and always said "Connection" even for a read timeout. One helper now reads the limit httpx actually enforced from the failed request and names the phase that fired Fixes #14635 * fix(rust_bridge): name the timeout phase in the vault timeout message to match the httpx handler * fix(http_handler): drop the unreachable timeout fallbacks httpx already rules out * refactor(rust_bridge): drop a doc comment that restated the timeout helper --- .../src/exception_mapping_utils/mod.rs | 65 +++++------ .../src/exception_mapping_utils/original.rs | 14 +++ .../python-bridge/src/secrets/provider.rs | 19 +-- .../crates/python-bridge/src/secrets/vault.rs | 18 ++- litellm/llms/custom_httpx/http_handler.py | 108 ++++++++---------- .../llms/custom_httpx/test_http_handler.py | 83 ++++++++++++++ 6 files changed, 195 insertions(+), 112 deletions(-) diff --git a/litellm-rust/crates/core-utils/src/exception_mapping_utils/mod.rs b/litellm-rust/crates/core-utils/src/exception_mapping_utils/mod.rs index 162d325e4f4..8b63fadf5a6 100644 --- a/litellm-rust/crates/core-utils/src/exception_mapping_utils/mod.rs +++ b/litellm-rust/crates/core-utils/src/exception_mapping_utils/mod.rs @@ -51,7 +51,7 @@ mod rules; mod status; mod vertex_ai; -pub use original::{ExceptionFamily, OriginalException}; +pub use original::{ExceptionFamily, OriginalException, TimeoutPhase}; pub use public::{MappedFailure, PublicError, UpstreamResponse}; use rules::{Rule, contains_any, first_match}; @@ -67,7 +67,6 @@ const TIMEOUT_MARKERS: &[&str] = &[ pub struct ExceptionContext { pub model: String, pub custom_llm_provider: String, - pub asynchronous: bool, pub vertex_project: Option, pub vertex_location: Option, pub model_group: Option, @@ -105,11 +104,12 @@ pub fn exception_type( (None, message.clone(), None) } OriginalException::Timeout { + phase, timeout_seconds, elapsed_seconds, } => ( None, - timeout_message(context.asynchronous, *timeout_seconds, *elapsed_seconds), + timeout_message(*phase, *timeout_seconds, *elapsed_seconds), None, ), }; @@ -161,21 +161,16 @@ fn family_rules(family: ExceptionFamily) -> &'static [Rule] { } } -/// The text the Python HTTP handler's timeout carries: the sync and async handlers word it -/// differently. -fn timeout_message( - asynchronous: bool, +/// The text the Python HTTP handler's timeout carries: the phase that expired and the limit +/// httpx enforced for it, both rendered as Python would print them. +pub fn timeout_message( + phase: TimeoutPhase, timeout_seconds: Option, elapsed_seconds: Option, ) -> String { let timeout = python_float(timeout_seconds); - if asynchronous { - let elapsed = - python_float(elapsed_seconds.map(|seconds| (seconds * 1000.0).round() / 1000.0)); - format!("Connection timed out. Timeout passed={timeout}, time taken={elapsed} seconds") - } else { - format!("Connection timed out after {timeout} seconds.") - } + let elapsed = python_float(elapsed_seconds.map(|seconds| (seconds * 1000.0).round() / 1000.0)); + format!("{phase} timed out. Timeout passed={timeout}, time taken={elapsed} seconds") } fn python_float(value: Option) -> String { @@ -356,6 +351,7 @@ mod tests { #[test] fn a_handler_timeout_is_a_408_without_a_response() { let original = OriginalException::Timeout { + phase: TimeoutPhase::Read, timeout_seconds: Some(0.5), elapsed_seconds: Some(0.5031), }; @@ -363,7 +359,7 @@ mod tests { mapped("mistral", &original), MappedFailure { error: PublicError::Timeout { status: 408 }, - message: "MistralException - Connection timed out after 0.5 seconds.".into(), + message: "MistralException - Read timed out. Timeout passed=0.5, time taken=0.503 seconds".into(), upstream: None, debug_info: DEBUG.into(), } @@ -440,39 +436,44 @@ mod tests { } #[rstest::rstest] - #[case::sync( - false, + #[case::connect(TimeoutPhase::Connect, "Connection timed out.")] + #[case::read(TimeoutPhase::Read, "Read timed out.")] + #[case::write(TimeoutPhase::Write, "Write timed out.")] + #[case::pool(TimeoutPhase::Pool, "Connection pool timed out.")] + fn timeout_text_names_the_phase_that_expired( + #[case] phase: TimeoutPhase, + #[case] expected: &str, + ) { + assert_eq!( + timeout_message(phase, Some(0.5), Some(0.5031)), + format!("{expected} Timeout passed=0.5, time taken=0.503 seconds") + ); + } + + #[rstest::rstest] + #[case::rounds_the_elapsed_time( Some(0.5), Some(0.5031), - "Connection timed out after 0.5 seconds." - )] - #[case::async_rounds_the_elapsed_time( - true, - Some(0.5), - Some(0.5031), - "Connection timed out. Timeout passed=0.5, time taken=0.503 seconds" + "Timeout passed=0.5, time taken=0.503 seconds" )] #[case::whole_seconds_keep_a_decimal( - true, Some(600.0), Some(2.0), - "Connection timed out. Timeout passed=600.0, time taken=2.0 seconds" + "Timeout passed=600.0, time taken=2.0 seconds" )] #[case::unknown_values_render_as_none( - true, None, None, - "Connection timed out. Timeout passed=None, time taken=None seconds" + "Timeout passed=None, time taken=None seconds" )] - fn timeout_text_follows_the_delivery_mode( - #[case] asynchronous: bool, + fn timeout_text_renders_durations_as_python_floats( #[case] timeout_seconds: Option, #[case] elapsed_seconds: Option, #[case] expected: &str, ) { assert_eq!( - timeout_message(asynchronous, timeout_seconds, elapsed_seconds), - expected + timeout_message(TimeoutPhase::Read, timeout_seconds, elapsed_seconds), + format!("Read timed out. {expected}") ); } diff --git a/litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs b/litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs index 4de67e6e7b5..6517b093bbd 100644 --- a/litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs +++ b/litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs @@ -10,6 +10,7 @@ pub enum OriginalException { message: String, }, Timeout { + phase: TimeoutPhase, timeout_seconds: Option, elapsed_seconds: Option, }, @@ -20,6 +21,19 @@ pub enum OriginalException { }, } +/// The httpx timeout that expired, spelled the way the Python HTTP handler names it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display, strum::VariantArray)] +pub enum TimeoutPhase { + #[strum(serialize = "Connection")] + Connect, + #[strum(serialize = "Read")] + Read, + #[strum(serialize = "Write")] + Write, + #[strum(serialize = "Connection pool")] + Pool, +} + /// Which provider-specific text rules apply before the shared status table. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ExceptionFamily { diff --git a/litellm-rust/crates/python-bridge/src/secrets/provider.rs b/litellm-rust/crates/python-bridge/src/secrets/provider.rs index 568a0cd2228..5f4cd193fcf 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/provider.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/provider.rs @@ -88,20 +88,23 @@ fn aws_context( } fn read_timeout(value: Option<&Bound<'_, PyAny>>) -> PyResult> { - let Some(value) = value.filter(|value| !value.is_none()) else { - return Ok(None); - }; - let seconds = match value.extract::() { - Ok(value) => Some(value), - Err(_) => value.getattr("read")?.extract::>()?, - }; - seconds + read_timeout_seconds(value)? .map(|value| { Duration::try_from_secs_f64(value).map_err(|_| PyValueError::new_err("invalid timeout")) }) .transpose() } +pub(super) fn read_timeout_seconds(value: Option<&Bound<'_, PyAny>>) -> PyResult> { + let Some(value) = value.filter(|value| !value.is_none()) else { + return Ok(None); + }; + match value.extract::() { + Ok(value) => Ok(Some(value)), + Err(_) => value.getattr("read")?.extract::>(), + } +} + fn vault_context(params: Option<&Bound<'_, PyAny>>) -> PyResult { let params = params.and_then(|value| value.cast::().ok()); let nested = params diff --git a/litellm-rust/crates/python-bridge/src/secrets/vault.rs b/litellm-rust/crates/python-bridge/src/secrets/vault.rs index 46b29839fe4..79281662848 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/vault.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/vault.rs @@ -2,10 +2,12 @@ mod operation; pub(super) use operation::{Failure, FailureKind, FailureStage, delete, rotate, write}; +use litellm_core_utils::exception_mapping_utils::{TimeoutPhase, timeout_message}; use litellm_secrets::hashicorp::{Error, RawOperationError}; use pyo3::prelude::*; use super::mutation::{error_value, http_message, json_value}; +use super::provider::read_timeout_seconds; const HTTP_TRANSPORT_MODULE: &str = "litellm.rust_bridge.host.http_transport"; @@ -69,21 +71,17 @@ fn cause_message(py: Python<'_>, kind: FailureKind, context: &ErrorContext) -> P } FailureKind::Native(RawOperationError::Timeout { method, elapsed }) => { if method == "POST" { - let elapsed = py - .import("builtins")? - .call_method1("round", (elapsed.as_secs_f64(), 3))?; let kwargs = pyo3::types::PyDict::new(py); kwargs.set_item( "message", - format!( - "Connection timed out. Timeout passed={}, time taken={} seconds", - context.timeout.as_deref().unwrap_or("None"), - elapsed.str()? + timeout_message( + TimeoutPhase::Read, + context.timeout, + Some(elapsed.as_secs_f64()), ), )?; kwargs.set_item("model", "default-model-name")?; kwargs.set_item("llm_provider", "litellm-httpx-handler")?; - kwargs.set_item("headers", pyo3::types::PyDict::new(py))?; py.import("litellm.exceptions")? .getattr("Timeout")? .call((), Some(&kwargs))? @@ -159,7 +157,7 @@ fn response_text(py: Python<'_>, body: &[u8]) -> PyResult { #[derive(Default)] pub(super) struct ErrorContext { - timeout: Option, + timeout: Option, aiohttp: bool, } @@ -173,7 +171,7 @@ impl ErrorContext { return Ok(Self::default()); } Ok(Self { - timeout: timeout.map(|value| value.str()?.extract()).transpose()?, + timeout: read_timeout_seconds(timeout)?, aiohttp: py .import(HTTP_TRANSPORT_MODULE)? .getattr("uses_aiohttp_transport")? diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index cffeb44f12c..0b64885df45 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -20,6 +20,7 @@ from aiohttp import ClientSession, DummyCookieJar, TCPConnector from httpx import USE_CLIENT_DEFAULT, AsyncHTTPTransport, HTTPTransport from httpx._types import CertTypes, RequestFiles from httpx._utils import get_environment_proxies +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger @@ -566,6 +567,34 @@ async def _raise_masked_async_error(e: httpx.HTTPStatusError, stream: bool) -> N raise MaskedHTTPStatusError(e, message=_text, text=_text) from None +_TIMEOUT_PHASES: Final[Mapping[type[httpx.TimeoutException], tuple[str, str]]] = MappingProxyType( + { + httpx.ConnectTimeout: ("Connection", "connect"), + httpx.ReadTimeout: ("Read", "read"), + httpx.WriteTimeout: ("Write", "write"), + httpx.PoolTimeout: ("Connection pool", "pool"), + } +) +_UNKNOWN_TIMEOUT_PHASE: Final = ("Request", "read") +_TIMEOUT_EXTENSION: Final = TypeAdapter(dict[str, float | None]) + + +def _timeout_phase_seconds(error: httpx.TimeoutException, phase: str) -> float | None: + """The limit httpx enforced for ``phase`` on the request that raised ``error``, from the request itself.""" + return _TIMEOUT_EXTENSION.validate_python(error.request.extensions["timeout"]).get(phase) + + +def _raise_timeout(error: httpx.TimeoutException, started_at: float) -> NoReturn: + label, phase = _TIMEOUT_PHASES.get(type(error), _UNKNOWN_TIMEOUT_PHASE) + seconds: Final = _timeout_phase_seconds(error, phase) + time_taken: Final = round(time.time() - started_at, 3) + raise litellm.Timeout( + message=f"{label} timed out. Timeout passed={seconds}, time taken={time_taken} seconds", + model="default-model-name", + llm_provider="litellm-httpx-handler", + ) + + class HTTPResponseLimitError(ValueError): pass @@ -870,20 +899,7 @@ class AsyncHTTPHandler: finally: await new_client.aclose() except httpx.TimeoutException as e: - end_time: Final = time.time() - time_delta: Final = round(end_time - start_time, 3) - headers = {} - error_response: Final[httpx.Response | None] = getattr(e, "response", None) - if error_response is not None: - for key, value in error_response.headers.items(): - headers[f"response_headers-{key}"] = value - - raise litellm.Timeout( - message=f"Connection timed out. Timeout passed={timeout}, time taken={time_delta} seconds", - model="default-model-name", - llm_provider="litellm-httpx-handler", - headers=headers, - ) + _raise_timeout(e, start_time) except httpx.HTTPStatusError as e: await _raise_masked_async_error(e, stream) except Exception as e: @@ -902,6 +918,7 @@ class AsyncHTTPHandler: follow_redirects: bool | None = None, ): _follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT + start_time: Final = time.time() try: if timeout is None: timeout = self.timeout @@ -946,18 +963,7 @@ class AsyncHTTPHandler: finally: await new_client.aclose() except httpx.TimeoutException as e: - headers = {} - error_response: Final[httpx.Response | None] = getattr(e, "response", None) - if error_response is not None: - for key, value in error_response.headers.items(): - headers[f"response_headers-{key}"] = value - - raise litellm.Timeout( - message=f"Connection timed out after {timeout} seconds.", - model="default-model-name", - llm_provider="litellm-httpx-handler", - headers=headers, - ) + _raise_timeout(e, start_time) except httpx.HTTPStatusError as e: await _raise_masked_async_error(e, stream) except Exception as e: @@ -974,6 +980,7 @@ class AsyncHTTPHandler: stream: bool = False, content: _RequestContent | None = None, ): + start_time: Final = time.time() try: if timeout is None: timeout = self.timeout @@ -1012,18 +1019,7 @@ class AsyncHTTPHandler: finally: await new_client.aclose() except httpx.TimeoutException as e: - headers = {} - error_response: Final[httpx.Response | None] = getattr(e, "response", None) - if error_response is not None: - for key, value in error_response.headers.items(): - headers[f"response_headers-{key}"] = value - - raise litellm.Timeout( - message=f"Connection timed out after {timeout} seconds.", - model="default-model-name", - llm_provider="litellm-httpx-handler", - headers=headers, - ) + _raise_timeout(e, start_time) except httpx.HTTPStatusError as e: await _raise_masked_async_error(e, stream) except Exception as e: @@ -1514,6 +1510,7 @@ class HTTPHandler: content: _RequestContent | None = None, logging_obj: LiteLLMLoggingObject | None = None, ): + start_time: Final = time.time() try: # Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix) request_data, request_content = _prepare_request_data_and_content(data, content) @@ -1546,12 +1543,8 @@ class HTTPHandler: _anchor_handler_to(response, self) response.raise_for_status() return response - except httpx.TimeoutException: - raise litellm.Timeout( - message=f"Connection timed out after {timeout} seconds.", - model="default-model-name", - llm_provider="litellm-httpx-handler", - ) + except httpx.TimeoutException as e: + _raise_timeout(e, start_time) except httpx.HTTPStatusError as e: _raise_masked_sync_error(e, stream) except Exception as e: @@ -1568,6 +1561,7 @@ class HTTPHandler: timeout: float | httpx.Timeout | None = None, content: _RequestContent | None = None, ): + start_time: Final = time.time() try: # Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix) request_data, request_content = _prepare_request_data_and_content(data, content) @@ -1598,12 +1592,8 @@ class HTTPHandler: _anchor_handler_to(response, self) response.raise_for_status() return response - except httpx.TimeoutException: - raise litellm.Timeout( - message=f"Connection timed out after {timeout} seconds.", - model="default-model-name", - llm_provider="litellm-httpx-handler", - ) + except httpx.TimeoutException as e: + _raise_timeout(e, start_time) except httpx.HTTPStatusError as e: _raise_masked_sync_error(e, stream) except Exception as e: @@ -1620,6 +1610,7 @@ class HTTPHandler: timeout: float | httpx.Timeout | None = None, content: _RequestContent | None = None, ): + start_time: Final = time.time() try: # Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix) request_data, request_content = _prepare_request_data_and_content(data, content) @@ -1649,12 +1640,8 @@ class HTTPHandler: if stream: _anchor_handler_to(response, self) return response - except httpx.TimeoutException: - raise litellm.Timeout( - message=f"Connection timed out after {timeout} seconds.", - model="default-model-name", - llm_provider="litellm-httpx-handler", - ) + except httpx.TimeoutException as e: + _raise_timeout(e, start_time) except httpx.HTTPStatusError as e: _raise_masked_sync_error(e, stream) except Exception as e: @@ -1671,6 +1658,7 @@ class HTTPHandler: stream: bool = False, content: _RequestContent | None = None, ): + start_time: Final = time.time() try: # Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix) request_data, request_content = _prepare_request_data_and_content(data, content) @@ -1701,12 +1689,8 @@ class HTTPHandler: _anchor_handler_to(response, self) response.raise_for_status() return response - except httpx.TimeoutException: - raise litellm.Timeout( - message=f"Connection timed out after {timeout} seconds.", - model="default-model-name", - llm_provider="litellm-httpx-handler", - ) + except httpx.TimeoutException as e: + _raise_timeout(e, start_time) except httpx.HTTPStatusError as e: _raise_masked_sync_error(e, stream) except Exception as e: diff --git a/tests/unit/llms/custom_httpx/test_http_handler.py b/tests/unit/llms/custom_httpx/test_http_handler.py index bd4ed4c99cc..1607447f7bf 100644 --- a/tests/unit/llms/custom_httpx/test_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_http_handler.py @@ -2172,3 +2172,86 @@ async def test_async_ollama_ssl_verify_false_builds_a_client_that_skips_certific assert route.call_count == 1 assert text == "Sunny" assert _cached_clients_verify_modes(client_cache, AsyncHTTPHandler) == (ssl.CERT_NONE,) + + +_TIMEOUT_CLIENT_LIMITS: Final = httpx.Timeout(42.0, connect=7.0) +_TIMEOUT_PHASES: Final = ( + (httpx.ConnectTimeout, "Connection timed out", "Timeout passed=7.0"), + (httpx.ReadTimeout, "Read timed out", "Timeout passed=42.0"), + (httpx.WriteTimeout, "Write timed out", "Timeout passed=42.0"), + (httpx.PoolTimeout, "Connection pool timed out", "Timeout passed=42.0"), +) + + +def _timing_out(kind: type[httpx.TimeoutException]) -> httpx.MockTransport: + def _raise(request: httpx.Request) -> httpx.Response: + raise kind("simulated", request=request) + + return httpx.MockTransport(_raise) + + +@pytest.mark.parametrize("verb", ["post", "put", "patch", "delete"]) +@pytest.mark.parametrize(("kind", "label", "limit"), _TIMEOUT_PHASES) +def test_sync_timeout_without_a_per_call_timeout_names_the_phase_and_the_limit_that_fired( + verb: str, kind: type[httpx.TimeoutException], label: str, limit: str +) -> None: + """Issue #14635: the handler used to print the caller's absent argument as 'after None seconds'.""" + handler: Final = HTTPHandler() + handler.client.close() + handler.client = httpx.Client(transport=_timing_out(kind), timeout=_TIMEOUT_CLIENT_LIMITS) + try: + with pytest.raises(LitellmTimeout) as raised: + getattr(handler, verb)("https://example.com/v1/embeddings", json={"input": "hi"}) + finally: + handler.close() + + message: Final = str(raised.value) + assert label in message, message + assert limit in message, message + assert "None" not in message, message + + +@pytest.mark.parametrize("verb", ["post", "put", "patch"]) +@pytest.mark.parametrize(("kind", "label", "limit"), _TIMEOUT_PHASES) +async def test_async_timeout_without_a_per_call_timeout_names_the_phase_and_the_limit_that_fired( + verb: str, kind: type[httpx.TimeoutException], label: str, limit: str +) -> None: + handler: Final = AsyncHTTPHandler(timeout=_TIMEOUT_CLIENT_LIMITS) + await handler.client.aclose() + handler.client = httpx.AsyncClient(transport=_timing_out(kind), timeout=_TIMEOUT_CLIENT_LIMITS) + try: + with pytest.raises(LitellmTimeout) as raised: + await getattr(handler, verb)("https://example.com/v1/embeddings", json={"input": "hi"}) + finally: + await handler.close() + + message: Final = str(raised.value) + assert label in message, message + assert limit in message, message + assert "None" not in message, message + + +def test_sync_timeout_reports_the_per_call_timeout_as_a_float_when_one_is_given() -> None: + handler: Final = HTTPHandler() + handler.client.close() + handler.client = httpx.Client(transport=_timing_out(httpx.ReadTimeout), timeout=_TIMEOUT_CLIENT_LIMITS) + try: + with pytest.raises(LitellmTimeout) as raised: + handler.post("https://example.com/v1/embeddings", json={"input": "hi"}, timeout=30) + finally: + handler.close() + + assert "Read timed out. Timeout passed=30.0, time taken=" in str(raised.value), str(raised.value) + + +async def test_async_timeout_reports_the_per_call_timeout_as_a_float_when_one_is_given() -> None: + handler: Final = AsyncHTTPHandler(timeout=_TIMEOUT_CLIENT_LIMITS) + await handler.client.aclose() + handler.client = httpx.AsyncClient(transport=_timing_out(httpx.ReadTimeout), timeout=_TIMEOUT_CLIENT_LIMITS) + try: + with pytest.raises(LitellmTimeout) as raised: + await handler.post("https://example.com/v1/embeddings", json={"input": "hi"}, timeout=30) + finally: + await handler.close() + + assert "Read timed out. Timeout passed=30.0, time taken=" in str(raised.value), str(raised.value) From bc0f598db5866d1d3fdb5c99ce627fb5ea19a788 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 26 Oct 2026 19:11:38 -0400 Subject: [PATCH 082/155] fix(proxy): validate JSON object bodies before request handling (#48438) --- litellm/proxy/auth/auth_exception_handler.py | 12 +- .../proxy/common_utils/http_parsing_utils.py | 43 +++++-- .../proxy/auth/test_auth_exception_handler.py | 62 ++++++++++ .../test_user_api_key_auth_request_flow.py | 6 +- .../common_utils/test_http_parsing_utils.py | 112 +++++++++++++++++- tests/unit/proxy/lens/test_buffering.py | 17 ++- .../test_llm_pass_through_endpoints.py | 1 + 7 files changed, 226 insertions(+), 27 deletions(-) diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index f6637a1fdf3..2eb25cc6105 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -12,7 +12,7 @@ import litellm from litellm._logging import verbose_proxy_logger, verbose_proxy_stdout_logger from litellm.constants import EMPTY_MAPPING from litellm.integrations.otel.runtime import seed_request_identity -from litellm.litellm_core_utils.core_helpers import is_expected_client_error +from litellm.litellm_core_utils.core_helpers import as_str_mapping, is_expected_client_error from litellm.proxy._types import ( LitellmUserRoles, ModelAccessDeniedProxyException, @@ -139,7 +139,7 @@ class UserAPIKeyAuthExceptionHandler: async def handle_authentication_error( e: Exception, request: Request, - request_data: dict[str, object], + request_data: object, route: str, parent_otel_span: Span | None, api_key: str, @@ -190,6 +190,8 @@ class UserAPIKeyAuthExceptionHandler: request_route=route, ) else: + request_mapping: Final = as_str_mapping(request_data) + request_context: Final[dict[str, object]] = dict(request_mapping) if request_mapping is not None else {} # raise the exception to the caller requester_ip: Final = get_request_ip_address( request=request, @@ -229,7 +231,7 @@ class UserAPIKeyAuthExceptionHandler: # without this the failed trace would carry no team/key attributes. seed_request_identity( user_api_key_dict, - model=request_data.get("model"), + model=request_context.get("model"), ) # Budget checks live in tenant-scoped helpers (key / team / org / tag) @@ -242,14 +244,14 @@ class UserAPIKeyAuthExceptionHandler: resolve_llm_provider_for_rate_limit, ) - budget_model: Final = request_data.get("model") + budget_model: Final = request_context.get("model") _, e.llm_provider = resolve_llm_provider_for_rate_limit( budget_model if isinstance(budget_model, str) else None ) # Allow callbacks to transform the error response transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook( - request_data=_with_client_context(request_data, requester_ip, user_agent), + request_data=_with_client_context(request_context, requester_ip, user_agent), original_exception=e, user_api_key_dict=user_api_key_dict, error_type=ProxyErrorTypes.auth_error, diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index a38769ac41b..9f7fed40ab3 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -6,7 +6,7 @@ from typing import Annotated, Any, Final, Literal, Union, get_args, get_origin import orjson from fastapi import Request, UploadFile, status -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from starlette._utils import get_route_path from starlette.requests import HTTPConnection from typing_extensions import NotRequired, ReadOnly, Required, assert_never @@ -162,13 +162,11 @@ def coerce_numeric_form_fields( } -def _parse_binary_body(body: bytes) -> dict: +def _parse_binary_body(body: bytes) -> dict[str, object]: """JSON sent under a binary content type still parses; real binary (protobuf) carries no params -> {}.""" try: - parsed: Final = _DECODED_JSON.validate_python(orjson.loads(body)) - if isinstance(parsed, dict): - return parsed - except orjson.JSONDecodeError: + return dict(_JSON_REQUEST_BODY.validate_python(orjson.loads(body))) + except (orjson.JSONDecodeError, ValidationError): pass return {} @@ -299,8 +297,9 @@ async def read_request_body(request: Request | None) -> dict: body_str = re.sub(r"(? dict: safe_set_request_parsed_body(request=request, parsed_body=parsed_body) return parsed_body + except ValidationError as e: + from fastapi.routing import APIRoute + + from litellm.proxy.auth.auth_utils import ( + request_dispatched_to_pass_through_endpoint, + request_dispatched_to_provider_pass_through, + ) + from litellm.proxy.lens.buffering import LensRoute + + route: Final[object] = request.scope.get("route") if request is not None else None + if request is not None and ( + request_dispatched_to_pass_through_endpoint(request) + or request_dispatched_to_provider_pass_through(request) + or isinstance(route, LensRoute) + or ( + isinstance(route, APIRoute) + and route.body_field is not None + and not route.body_field.field_info.is_required() + and (await request.body()).strip() == b"null" + ) + ): + return {} + raise ProxyException( + message="Invalid JSON payload: expected an object", + type="invalid_request_error", + param="request_body", + code=status.HTTP_400_BAD_REQUEST, + ) from e except (json.JSONDecodeError, orjson.JSONDecodeError, ProxyException) as e: # Re-raise ProxyException as-is verbose_proxy_logger.error("Invalid JSON payload received: %s", e) diff --git a/tests/unit/proxy/auth/test_auth_exception_handler.py b/tests/unit/proxy/auth/test_auth_exception_handler.py index d83a8aa2fcd..e21491fd996 100644 --- a/tests/unit/proxy/auth/test_auth_exception_handler.py +++ b/tests/unit/proxy/auth/test_auth_exception_handler.py @@ -1,5 +1,7 @@ import asyncio import json +from collections.abc import Mapping +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -29,6 +31,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import INVALID_VIRTUAL_KEY_ERROR_MARKER from litellm.exceptions import BudgetExceededError +from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import ( ModelAccessDeniedProxyException, ProxyErrorTypes, @@ -812,6 +815,65 @@ def _http_request(client_host: str | None = "10.1.2.3", headers: dict[str, str] ) +class _AuthFailureRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.data: dict[str, object] | None = None + self.error: Exception | None = None + + async def async_post_call_failure_hook( + self, + request_data: dict[str, object], + original_exception: Exception, + user_api_key_dict: UserAPIKeyAuth, + traceback_str: str | None = None, + ) -> HTTPException | None: + self.data = dict(request_data) + self.error = original_exception + return None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_data", [[], None, "metadata", 1, True]) +@pytest.mark.parametrize("error_kind", ["auth", "budget", "database"]) +async def test_auth_error_preserves_classification_with_non_object_context( + monkeypatch: pytest.MonkeyPatch, request_data: object, error_kind: str +) -> None: + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(litellm, "budget_exceeded_status_code", 429) + errors: Final = { + "auth": HTTPException(status_code=401, detail="Invalid API key"), + "budget": BudgetExceededError(current_cost=2, max_budget=1), + "database": httpx.ConnectError("Database unavailable"), + } + error: Final = errors[error_kind] + expected: Final = _as_proxy_exception(error) + recorder: Final = _AuthFailureRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + monkeypatch.setattr(proxy_server, "general_settings", {"allow_requests_on_db_unavailable": False}) + + with pytest.raises(ProxyException) as exc_info: + await UserAPIKeyAuthExceptionHandler.handle_authentication_error( + error, + _http_request(headers={"user-agent": "shape-regression"}), + request_data, + "/key/info", + None, + "sk-test", + ) + + assert exc_info.value.code == expected.code + assert exc_info.value.type == expected.type + assert exc_info.value.message == expected.message + assert recorder.error is error + assert recorder.data is not None + metadata: Final = recorder.data["metadata"] + assert isinstance(metadata, Mapping) + assert metadata["requester_ip_address"] == "10.1.2.3" + assert metadata["user_agent"] == "shape-regression" + + @pytest.mark.asyncio @pytest.mark.parametrize( "auth_error, general_settings, request_kwargs, expected_ip", diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 108f7ce2983..2d9c781b0d1 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -7724,7 +7724,8 @@ async def test_user_api_key_auth_does_not_overwrite_end_user_id_set_by_builder() @pytest.mark.asyncio -async def test_user_api_key_auth_authenticates_before_raising_malformed_body_error(): +@pytest.mark.parametrize("body", [b'{}{"model": "gpt-4o"}', b"[]", b"null", b"[1e400]"]) +async def test_user_api_key_auth_authenticates_before_raising_malformed_body_error(body: bytes) -> None: """Regression (LIT-4780): a body that fails to parse must still be authenticated first, so the rejected request's trace carries the caller's key / team / user identity instead of an anonymous root span. The parse error is re-raised @@ -7742,10 +7743,11 @@ async def test_user_api_key_auth_authenticates_before_raising_malformed_body_err "headers": [(b"content-type", b"application/json")], "method": "POST", "path": "/chat/completions", + "query_string": b"probe=1", } ) request._url = URL(url="/chat/completions") - request._body = b'{}{"model": "gpt-4o"}' + request._body = body attrs = _proxy_attrs_for_centralized_checks(user_custom_auth=None) originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs} diff --git a/tests/unit/proxy/common_utils/test_http_parsing_utils.py b/tests/unit/proxy/common_utils/test_http_parsing_utils.py index cd5820fa2fd..354feb0bab1 100644 --- a/tests/unit/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/unit/proxy/common_utils/test_http_parsing_utils.py @@ -64,6 +64,100 @@ def _starlette_request( return Request(scope, receive) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body", + [b"[]", b'[["model", "test"]]', b"null", b'"s"', b"1", b"true", b"false"], +) +@pytest.mark.parametrize("path", ["/v1/messages", "/vertex_ai/v1/messages", "/lens/datasets"]) +async def test_read_request_body_rejects_non_object_json(body: bytes, path: str) -> None: + request: Final = _starlette_request(body, "application/json", path) + + for _ in range(2): + with pytest.raises(ProxyException) as exc_info: + await read_request_body(request) + assert exc_info.value.code == "400" + assert exc_info.value.type == "invalid_request_error" + assert exc_info.value.param == "request_body" + assert exc_info.value.message == "Invalid JSON payload: expected an object" + assert _safe_get_request_parsed_body(request) is None + assert await request.body() == body + + +@pytest.mark.asyncio +@pytest.mark.parametrize("body", [b"[1e400]", b"1e400", b"[NaN]"]) +async def test_read_request_body_rejects_non_objects_from_json_fallback(body: bytes) -> None: + with pytest.raises(ProxyException) as exc_info: + await read_request_body(_starlette_request(body, "application/json")) + assert exc_info.value.code == "400" + assert exc_info.value.message == "Invalid JSON payload: expected an object" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body, invalid_unicode", + [ + (b"[]", False), + (b"null", False), + (b'"value"', False), + (b"[1e400]", False), + (b'"\\ud800"', True), + (b'["\\ud800"]', True), + (b'{"value":"\\ud800"}', True), + ], +) +@pytest.mark.parametrize("dispatch", ["custom", "provider", "lens"]) +async def test_pass_through_body_validation_preserves_bytes(dispatch: str, body: bytes, invalid_unicode: bool) -> None: + from fastapi.routing import APIRoute + from litellm.proxy.lens.adapter import lens_request + from litellm.proxy.lens.buffering import LensRoute + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import vertex_proxy_route + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import create_pass_through_route + + path: Final = {"custom": "/v1/messages", "provider": "/vertex_ai/predict", "lens": "/lens/datasets"}[dispatch] + route: Final = { + "custom": APIRoute(path, create_pass_through_route(path, "https://upstream.test"), methods=["POST"]), + "provider": APIRoute("/vertex_ai/{endpoint:path}", vertex_proxy_route, methods=["POST"]), + "lens": LensRoute("/lens/{path:path}", lens_request, methods=["POST"]), + }[dispatch] + incoming: Final = _starlette_request(body, "application/json", path) + _, dispatched_scope = route.matches(incoming.scope) + request: Final = Request({**incoming.scope, **dispatched_scope}, incoming.receive) + + for _ in range(2): + if invalid_unicode: + with pytest.raises(ProxyException) as exc_info: + await read_request_body(request) + assert exc_info.value.code == "400" + else: + assert await read_request_body(request) == {} + assert await request.body() == body + + +@pytest.mark.asyncio +@pytest.mark.parametrize("optional", [False, True]) +@pytest.mark.parametrize("body", [b"null", b" \nnull\t", b"[]", b'"value"']) +async def test_null_body_respects_the_dispatched_body_requirement(optional: bool, body: bytes) -> None: + from fastapi.routing import APIRoute + from litellm.proxy.management_endpoints.key_management_endpoints import generate_key_fn + from litellm.proxy.spend_tracking.spend_management_endpoints import global_spend_end_users + + endpoint: Final = global_spend_end_users if optional else generate_key_fn + route: Final = APIRoute("/audit", endpoint, methods=["POST"]) + incoming: Final = _starlette_request(body, "application/json", "/audit") + _, dispatched_scope = route.matches(incoming.scope) + request: Final = Request({**incoming.scope, **dispatched_scope}, incoming.receive) + + if optional and body.strip() == b"null": + assert await read_request_body(request) == {} + assert await request.json() is None + else: + with pytest.raises(ProxyException) as exc_info: + await read_request_body(request) + assert exc_info.value.code == "400" + assert await request.body() == body + + @pytest.mark.asyncio async def test_read_request_body_marks_body_received_once_with_its_size(monkeypatch: pytest.MonkeyPatch): events: list[tuple[str, dict[str, str | int]]] = [] # mutable-ok: recorder for the injected phase_event double @@ -131,14 +225,20 @@ async def test_read_raw_json_body_is_none_for_form_bodies(): @pytest.mark.asyncio @pytest.mark.parametrize("content_type", ["application/x-protobuf", "application/protobuf; charset=binary"]) -async def test_protobuf_body_is_not_parsed_as_json(content_type): - # OTLP trace exports (POST /v1/traces) are binary protobuf; arbitrary bytes like these - # used to hit the JSON surrogate-repair path and fail auth with a 400. - body = b"\n\xa2\x01\n\x1c\n\x0cservice.name\x12\x0c\n\nswarm\xed\xa0\x80\xff" - request = _starlette_request(body, content_type) +@pytest.mark.parametrize( + "body", + [ + b"\n\xa2\x01\n\x1c\n\x0cservice.name\x12\x0c\n\nswarm\xed\xa0\x80\xff", + pytest.param(b"100000000", id="protobuf-also-json-number"), + ], +) +async def test_protobuf_body_is_not_parsed_as_json(content_type: str, body: bytes) -> None: + request: Final = _starlette_request(body, content_type) assert await read_request_body(request) == {} - assert await request.body() == body # body is still readable by the endpoint + assert await read_request_body(request) == {} + assert await read_raw_json_body(request) == body + assert await request.body() == body @pytest.mark.asyncio diff --git a/tests/unit/proxy/lens/test_buffering.py b/tests/unit/proxy/lens/test_buffering.py index b544b70d36b..2b868e72096 100644 --- a/tests/unit/proxy/lens/test_buffering.py +++ b/tests/unit/proxy/lens/test_buffering.py @@ -249,7 +249,10 @@ async def test_admission_releases_upload_on_disconnect_or_cancellation(cancel: b @pytest.mark.asyncio @pytest.mark.parametrize("reject_auth", [False, True]) -async def test_admitted_bytes_survive_auth_parsing_and_remain_reserved_until_sent(reject_auth: bool) -> None: +@pytest.mark.parametrize("body", [b" {} ", b"[]", b"null", b'"s"']) +async def test_admitted_bytes_survive_auth_parsing_and_remain_reserved_until_sent( + reject_auth: bool, body: bytes +) -> None: app: Final = FastAPI() router: Final = APIRouter(route_class=LensRoute) @@ -264,9 +267,11 @@ async def test_admitted_bytes_survive_auth_parsing_and_remain_reserved_until_sen if reject_auth: raise HTTPException(401, "Invalid key") - async with httpx.AsyncClient( - transport=httpx.MockTransport(lambda _: httpx.Response(200, content=b"result")) - ) as upstream: + def upstream_response(incoming: httpx.Request) -> httpx.Response: + assert incoming.content == body + return httpx.Response(200, content=b"result") + + async with httpx.AsyncClient(transport=httpx.MockTransport(upstream_response)) as upstream: @router.post("/lens/datasets") async def endpoint(request: Request, auth: Annotated[None, Depends(auth_reader)]) -> Response: @@ -284,14 +289,14 @@ async def test_admitted_bytes_survive_auth_parsing_and_remain_reserved_until_sen sending.set() await finish.wait() - incoming: Final = request(b" {} ") + incoming: Final = request(body) task: Final = asyncio.create_task(app(incoming.scope, incoming.receive, send)) try: await asyncio.wait_for(sending.wait(), 2) async with httpx.AsyncClient( transport=httpx.ASGITransport(app=app), base_url="http://gateway.test" ) as client: - blocked: Final = await client.post("/lens/datasets", content=b" {} " * 2) + blocked: Final = await client.post("/lens/datasets", content=b" " * (11 - len(body))) unrelated: Final = await client.post("/ordinary", content=b"ordinary body") assert blocked.status_code == 503 assert unrelated.content == b"ordinary body" diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 0186a33dbc4..60043daa402 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -4357,6 +4357,7 @@ class TestVertexRawPredictStreamingClassification: "type": "http", "method": "POST", "path": f"/vertex_ai/{endpoint}", + "path_params": {"endpoint": endpoint}, "headers": [ (b"content-type", b"application/json"), (b"x-litellm-api-key", b"test-key"), From a4ee9524441381c012c5f4b1bc2c3f7827a2a77b Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 26 Oct 2026 16:45:35 -0700 Subject: [PATCH 083/155] fix(proxy)!: gate key budget_duration and budget_id updates like max_budget (#47681) * fix(proxy): gate key budget_duration and budget_id updates like max_budget A key owner or team member could shorten budget_duration or swap budget_id on /key/update without the budget authorization max_budget gets, so the reset job cleared spend on a capped key or a bigger tier lifted its cap Fixes #46916 * test(proxy): cover tier keys and self-serve budget policies for key budget updates --------- Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- litellm/litellm_core_utils/duration_parser.py | 10 +- .../management_endpoints/key_budget_change.py | 44 +++- .../test_key_budget_owner_updates.py | 195 ++++++++++++++++++ .../test_key_budget_change.py | 109 ++++++++++ 4 files changed, 349 insertions(+), 9 deletions(-) create mode 100644 tests/integration/authorization/test_key_budget_owner_updates.py diff --git a/litellm/litellm_core_utils/duration_parser.py b/litellm/litellm_core_utils/duration_parser.py index d52716951df..95bc77e3eba 100644 --- a/litellm/litellm_core_utils/duration_parser.py +++ b/litellm/litellm_core_utils/duration_parser.py @@ -22,7 +22,7 @@ _BUDGET_DURATION_WORD_ALIASES: Final[dict[str, str]] = { } -def _normalize_duration(duration: str) -> str: +def normalize_duration(duration: str) -> str: return _BUDGET_DURATION_WORD_ALIASES.get(duration.strip().lower(), duration) @@ -61,7 +61,7 @@ def duration_in_seconds(duration: str) -> int: Returns time in seconds till when budget needs to be reset """ - value, unit = _extract_from_regex(duration=_normalize_duration(duration)) + value, unit = _extract_from_regex(duration=normalize_duration(duration)) if unit == "s": return value @@ -89,7 +89,7 @@ def duration_in_seconds_from(start: datetime, duration: str) -> int: month of a different length needs its own start to know when it ends. Every other unit is a fixed length and ignores `start`. """ - value, unit = _extract_from_regex(duration=_normalize_duration(duration)) + value, unit = _extract_from_regex(duration=normalize_duration(duration)) if unit == "mo": return _months_in_seconds(start=start, months=value) return duration_in_seconds(duration) @@ -129,7 +129,7 @@ def get_next_standardized_reset_time( current_time, _ = _setup_timezone(current_time, timezone_str) # Parse duration - value, unit = _parse_duration(_normalize_duration(duration)) + value, unit = _parse_duration(normalize_duration(duration)) if value is None: verbose_logger.warning( "Unrecognized budget_duration %r; falling back to a next-midnight reset. " @@ -171,7 +171,7 @@ def get_budget_window_start(duration: str, reset_at: datetime) -> datetime: """Start of the budget period that ends at `reset_at`, under the same rules `get_next_standardized_reset_time` used to pick it: `30d` and `Nmo` span calendar months and a duration it does not recognize resets at the next midnight.""" - value, unit = _parse_duration(_normalize_duration(duration)) + value, unit = _parse_duration(normalize_duration(duration)) if value is None: return reset_at - timedelta(days=1) match unit: diff --git a/litellm/proxy/management_endpoints/key_budget_change.py b/litellm/proxy/management_endpoints/key_budget_change.py index 3143ef53d76..197a13cdb13 100644 --- a/litellm/proxy/management_endpoints/key_budget_change.py +++ b/litellm/proxy/management_endpoints/key_budget_change.py @@ -9,6 +9,7 @@ from typing import Final, Literal, TypeAlias from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.duration_parser import normalize_duration from litellm.models.team import BudgetLimitEntry from litellm.models.verification_token import LiteLLM_VerificationToken from litellm.proxy._types import UpdateKeyRequest @@ -99,6 +100,30 @@ def _budget_limits_change(data: UpdateKeyRequest, existing: LiteLLM_Verification return "tightened" if kept_or_lowered else "loosened" +def _reset_schedule(budget_duration: str | None) -> str | None: + return None if budget_duration is None else normalize_duration(budget_duration) + + +def _changes_budget_duration(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> bool: + return "budget_duration" in data.model_fields_set and _reset_schedule(data.budget_duration) != _reset_schedule( + existing.budget_duration + ) + + +def _changes_budget_id(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> bool: + return "budget_id" in data.model_fields_set and data.budget_id != existing.budget_id + + +def _budget_duration_change(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> _FieldChange: + """ + Any new `budget_duration` restarts the reset schedule from now, so only clearing it tightens the + stored `max_budget`. A key with no `max_budget` has no cap for the schedule to reset. + """ + if existing.max_budget is None or not _changes_budget_duration(data, existing): + return "unchanged" + return "tightened" if data.budget_duration is None else "loosened" + + @dataclass(frozen=True, slots=True) class EffectiveKeyBudget: max_budget: float | None @@ -122,12 +147,23 @@ def effective_key_budget(data: UpdateKeyRequest, existing: LiteLLM_VerificationT def classify_key_budget_change(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> KeyBudgetChange: - overrides_linked_budget: Final = ( - existing.budget_id is not None and _max_budget_change(data, existing) != "unchanged" + overrides_linked_budget: Final = existing.budget_id is not None and ( + _max_budget_change(data, existing) != "unchanged" or _changes_budget_duration(data, existing) ) - if data.spend is not None or "soft_budget" in data.model_fields_set or overrides_linked_budget: + if ( + data.spend is not None + or "soft_budget" in data.model_fields_set + or overrides_linked_budget + or _changes_budget_id(data, existing) + ): return KeyBudgetAdminOnly() - changes: Final = frozenset((_max_budget_change(data, existing), _budget_limits_change(data, existing))) + changes: Final = frozenset( + ( + _max_budget_change(data, existing), + _budget_limits_change(data, existing), + _budget_duration_change(data, existing), + ) + ) if "loosened" in changes: return KeyBudgetLoosened() if "tightened" in changes: diff --git a/tests/integration/authorization/test_key_budget_owner_updates.py b/tests/integration/authorization/test_key_budget_owner_updates.py new file mode 100644 index 00000000000..8415c044b6f --- /dev/null +++ b/tests/integration/authorization/test_key_budget_owner_updates.py @@ -0,0 +1,195 @@ +from collections.abc import Iterator +from hashlib import sha256 +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import pytest +import yaml +from integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + eventually, + gateway_from_environment, + string_value, +) +from integration._support.process import owned_proxy +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.shares_worker_state + +CAP: Final = 0.1 +BUDGET_EXCEEDED: Final = 422 +FAST_RESET_TICK: Final = MappingProxyType( + {"PROXY_BUDGET_RESCHEDULER_MIN_TIME": "2", "PROXY_BUDGET_RESCHEDULER_MAX_TIME": "2"} +) + + +@pytest.fixture(scope="module") +def resetting(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + with ( + gateway_from_environment() as shared, + owned_proxy(shared, tmp_path_factory.mktemp("fast-reset"), FAST_RESET_TICK) as replica, + ): + yield replica + + +def _policy_config(directory: Path, policy: str) -> Path: + shared: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + general_settings: Final = { + **JSON_OBJECT.validate_python(shared["general_settings"]), + "self_serve_budget_policy": policy, + } + path: Final = directory / f"{policy}.yaml" + path.write_text(yaml.safe_dump({**shared, "general_settings": general_settings})) + return path + + +@pytest.fixture(scope="module", params=("lower_only", "ceiling")) +def self_serve( + request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[tuple[str, Gateway]]: + policy: Final = string_value(request.param) + directory: Final = tmp_path_factory.mktemp(policy) + with ( + gateway_from_environment() as shared, + owned_proxy(shared, directory, {}, config=_policy_config(directory, policy)) as replica, + ): + yield policy, replica + + +def _owner_credential(scenario: Scenario) -> str: + return scenario.key(user_id=scenario.user(user_role="internal_user")) + + +def _owned_key(gateway: Gateway, scenario: Scenario, owner: str, **fields: JsonValue) -> str: + response: Final = gateway.request("POST", "/key/generate", fields, key=owner) + assert response.status_code == 200, response.text + token: Final = string_value(JSON_OBJECT.validate_json(response.content)["key"]) + scenario.cleanups.callback(scenario.delete_key, token) + return token + + +def _key_row(gateway: Gateway, key: str) -> dict[str, JsonValue]: + rows: Final = gateway.rows( + "SELECT spend, max_budget, budget_duration, budget_id, budget_reset_at::text AS reset_at" + ' FROM "LiteLLM_VerificationToken" WHERE token = %s', + (sha256(key.encode()).hexdigest(),), + ) + assert len(rows) == 1, rows + return rows[0] + + +def _exhaust(gateway: Gateway, key: str) -> None: + gateway.post("/key/update", {"key": key, "spend": CAP}) + assert _key_row(gateway, key)["spend"] == CAP + + +def _chat_status(gateway: Gateway, model: str, key: str) -> int: + return gateway.request( + "POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": "budget"}]}, key=key + ).status_code + + +def test_key_owner_cannot_shorten_budget_duration_to_restart_an_exhausted_budget(resetting: Gateway) -> None: + with resetting.scenario() as scenario: + model: Final = scenario.model() + owner: Final = _owner_credential(scenario) + target: Final = _owned_key(resetting, scenario, owner, models=[model], max_budget=CAP, budget_duration="30d") + control: Final = _owned_key(resetting, scenario, owner, models=[model], max_budget=CAP, budget_duration="30d") + _exhaust(resetting, target) + _exhaust(resetting, control) + assert _chat_status(resetting, model, target) == BUDGET_EXCEEDED + before: Final = _key_row(resetting, target) + + denied: Final = resetting.request("POST", "/key/update", {"key": target, "budget_duration": "1s"}, key=owner) + + resetting.post("/key/update", {"key": control, "budget_duration": "1s"}) + eventually(lambda: _key_row(resetting, control)["spend"], lambda spend: spend == 0.0, seconds=60) + assert denied.status_code == 403, denied.text + assert _key_row(resetting, target) == before + assert _chat_status(resetting, model, target) == BUDGET_EXCEEDED + + +def test_key_owner_cannot_swap_or_reschedule_a_tier_key(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + low: Final = scenario.budget(max_budget=CAP) + high: Final = scenario.budget(max_budget=100.0) + owner: Final = _owner_credential(scenario) + target: Final = _owned_key(gateway, scenario, owner, models=[model], budget_id=low) + _exhaust(gateway, target) + assert _chat_status(gateway, model, target) == BUDGET_EXCEEDED + before: Final = _key_row(gateway, target) + + swapped: Final = gateway.request("POST", "/key/update", {"key": target, "budget_id": high}, key=owner) + rescheduled: Final = gateway.request("POST", "/key/update", {"key": target, "budget_duration": "1s"}, key=owner) + + assert swapped.status_code == 403, swapped.text + assert rescheduled.status_code == 403, rescheduled.text + assert _key_row(gateway, target) == before + assert _chat_status(gateway, model, target) == BUDGET_EXCEEDED + + +def test_team_member_cannot_shorten_budget_duration_on_a_team_key(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + team: Final = scenario.team(team_member_permissions=["/key/update", "/key/info"]) + member: Final = scenario.key(user_id=scenario.member(team), team_id=team) + target: Final = scenario.key(team_id=team, max_budget=CAP, budget_duration="30d") + before: Final = _key_row(gateway, target) + + denied: Final = gateway.request("POST", "/key/update", {"key": target, "budget_duration": "1s"}, key=member) + + assert denied.status_code == 403, denied.text + assert _key_row(gateway, target) == before + + +def test_key_owner_keeps_edits_that_cannot_restart_a_budget(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + owner: Final = _owner_credential(scenario) + capped: Final = _owned_key(gateway, scenario, owner, max_budget=CAP, budget_duration="monthly") + uncapped: Final = _owned_key(gateway, scenario, owner) + + renamed: Final = gateway.request("POST", "/key/update", {"key": capped, "key_alias": "renamed"}, key=owner) + resent: Final = gateway.request("POST", "/key/update", {"key": capped, "budget_duration": "30d"}, key=owner) + scheduled: Final = gateway.request("POST", "/key/update", {"key": uncapped, "budget_duration": "1d"}, key=owner) + + assert renamed.status_code == 200, renamed.text + assert resent.status_code == 200, resent.text + assert scheduled.status_code == 200, scheduled.text + assert _key_row(gateway, uncapped)["budget_duration"] == "1d" + + +def test_self_serve_policy_keeps_tier_keys_admin_only(self_serve: tuple[str, Gateway]) -> None: + _, proxy = self_serve + with proxy.scenario() as scenario: + low: Final = scenario.budget(max_budget=CAP) + high: Final = scenario.budget(max_budget=100.0) + owner: Final = _owner_credential(scenario) + target: Final = _owned_key(proxy, scenario, owner, budget_id=low) + before: Final = _key_row(proxy, target) + + swapped: Final = proxy.request("POST", "/key/update", {"key": target, "budget_id": high}, key=owner) + rescheduled: Final = proxy.request("POST", "/key/update", {"key": target, "budget_duration": "1d"}, key=owner) + + assert swapped.status_code == 403, swapped.text + assert rescheduled.status_code == 403, rescheduled.text + assert _key_row(proxy, target) == before + + +def test_self_serve_policy_gates_a_new_reset_period_like_a_raised_cap(self_serve: tuple[str, Gateway]) -> None: + policy, proxy = self_serve + with proxy.scenario() as scenario: + owner: Final = _owner_credential(scenario) + shortened: Final = _owned_key(proxy, scenario, owner, max_budget=CAP, budget_duration="30d") + cleared: Final = _owned_key(proxy, scenario, owner, max_budget=CAP, budget_duration="30d") + + shorten: Final = proxy.request("POST", "/key/update", {"key": shortened, "budget_duration": "1s"}, key=owner) + clear: Final = proxy.request("POST", "/key/update", {"key": cleared, "budget_duration": None}, key=owner) + + assert (shorten.status_code, _key_row(proxy, shortened)["budget_duration"]) == ( + (403, "30d") if policy == "lower_only" else (200, "1s") + ), shorten.text + assert clear.status_code == 200, clear.text + assert _key_row(proxy, cleared)["budget_duration"] is None diff --git a/tests/unit/proxy/management_endpoints/test_key_budget_change.py b/tests/unit/proxy/management_endpoints/test_key_budget_change.py index baf8d870117..484f8638513 100644 --- a/tests/unit/proxy/management_endpoints/test_key_budget_change.py +++ b/tests/unit/proxy/management_endpoints/test_key_budget_change.py @@ -45,6 +45,7 @@ def _key( max_budget: float | None = None, budget_limits: Sequence[Mapping[str, object]] | None = None, budget_id: str | None = None, + budget_duration: str | None = None, ) -> LiteLLM_VerificationToken: return LiteLLM_VerificationToken( token="hashed-owner-key", @@ -54,6 +55,7 @@ def _key( max_budget=max_budget, budget_limits=None if budget_limits is None else [dict(window) for window in budget_limits], budget_id=budget_id, + budget_duration=budget_duration, metadata={}, ) @@ -127,6 +129,56 @@ def _window(duration: str, cap: float) -> Mapping[str, object]: (UpdateKeyRequest(key="sk-1", spend=0.0), _key(), KeyBudgetAdminOnly()), (UpdateKeyRequest(key="sk-1", soft_budget=1.0), _key(), KeyBudgetAdminOnly()), (UpdateKeyRequest(key="sk-1", soft_budget=None, max_budget=5.0), _key(max_budget=10.0), KeyBudgetAdminOnly()), + ( + UpdateKeyRequest(key="sk-1", budget_duration="1s"), + _key(max_budget=10.0, budget_duration="30d"), + KeyBudgetLoosened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_duration="60d"), + _key(max_budget=10.0, budget_duration="30d"), + KeyBudgetLoosened(), + ), + (UpdateKeyRequest(key="sk-1", budget_duration="1d"), _key(max_budget=10.0), KeyBudgetLoosened()), + ( + UpdateKeyRequest(key="sk-1", budget_duration=None), + _key(max_budget=10.0, budget_duration="30d"), + KeyBudgetTightened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_duration="30d"), + _key(max_budget=10.0, budget_duration="30d"), + KeyBudgetUnchanged(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_duration="30d"), + _key(max_budget=10.0, budget_duration="Monthly"), + KeyBudgetUnchanged(), + ), + (UpdateKeyRequest(key="sk-1", budget_duration="1s"), _key(budget_duration="30d"), KeyBudgetUnchanged()), + (UpdateKeyRequest(key="sk-1", max_budget=5.0, budget_duration="1s"), _key(), KeyBudgetTightened()), + ( + UpdateKeyRequest(key="sk-1", budget_duration="1s"), + _key(budget_id="shared-tier-10", budget_duration="30d"), + KeyBudgetAdminOnly(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_duration=None), + _key(budget_id="shared-tier-10", budget_duration="30d"), + KeyBudgetAdminOnly(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_id="shared-tier-100"), + _key(budget_id="shared-tier-10"), + KeyBudgetAdminOnly(), + ), + (UpdateKeyRequest(key="sk-1", budget_id=None), _key(budget_id="shared-tier-10"), KeyBudgetAdminOnly()), + (UpdateKeyRequest(key="sk-1", budget_id="shared-tier-10"), _key(), KeyBudgetAdminOnly()), + ( + UpdateKeyRequest(key="sk-1", budget_id="shared-tier-10"), + _key(budget_id="shared-tier-10"), + KeyBudgetUnchanged(), + ), ], ) def test_classify_key_budget_change( @@ -465,3 +517,60 @@ async def test_owner_cannot_clear_max_budget_by_default(monkeypatch: pytest.Monk existing=_key(max_budget=10.0), ) assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("policy", "data", "existing"), + [ + (None, UpdateKeyRequest(key="sk-1", budget_duration="1s"), _key(max_budget=10.0, budget_duration="30d")), + ( + "lower_only", + UpdateKeyRequest(key="sk-1", budget_duration="1s"), + _key(max_budget=10.0, budget_duration="30d"), + ), + ("ceiling", UpdateKeyRequest(key="sk-1", budget_id="shared-tier-100"), _key(budget_id="shared-tier-10")), + ("ceiling", UpdateKeyRequest(key="sk-1", budget_duration="1s"), _key(budget_id="shared-tier-10")), + ], +) +async def test_owner_cannot_restart_or_relink_a_key_budget_without_admin( + monkeypatch: pytest.MonkeyPatch, policy: str | None, data: UpdateKeyRequest, existing: LiteLLM_VerificationToken +) -> None: + with pytest.raises(HTTPException) as exc: + await _update(monkeypatch, policy=policy, data=data, existing=existing) + assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("policy", "data", "existing"), + [ + (None, UpdateKeyRequest(key="sk-1", budget_duration="1d"), _key(budget_duration="30d")), + (None, UpdateKeyRequest(key="sk-1", budget_duration="24h"), _key(max_budget=10.0, budget_duration="daily")), + ( + "lower_only", + UpdateKeyRequest(key="sk-1", budget_duration=None), + _key(max_budget=10.0, budget_duration="30d"), + ), + ("ceiling", UpdateKeyRequest(key="sk-1", budget_duration="1s"), _key(max_budget=10.0, budget_duration="30d")), + ], +) +async def test_owner_changes_budget_duration_when_it_cannot_restart_a_cap_or_policy_allows( + monkeypatch: pytest.MonkeyPatch, policy: str | None, data: UpdateKeyRequest, existing: LiteLLM_VerificationToken +) -> None: + assert await _denial(monkeypatch, policy=policy, data=data, existing=existing) is None + + +@pytest.mark.asyncio +async def test_team_member_cannot_restart_a_team_key_budget_created_by_someone_else( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy=None, + data=UpdateKeyRequest(key="sk-1", budget_duration="1s"), + existing=_key(created_by=_TEAM_ADMIN, team_id=_TEAM, max_budget=10.0, budget_duration="30d"), + team=_team(["/key/update"]), + ) + assert exc.value.status_code == 403 From df06b2d4fbce65f67dbd714d41d27258b62c8123 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 26 Oct 2026 16:52:43 -0700 Subject: [PATCH 084/155] chore(lint): add google-convention docstring rules to strict ruff gate (#48245) * chore(lint): add google-convention docstring rules (D202, D403, D417) to strict ruff gate * fix(lint): mark docstring rules external in base ruff config so RUF100 keeps their noqa --- ruff-strict.toml | 5 ++++- ruff.toml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ruff-strict.toml b/ruff-strict.toml index 28102da3ccd..398cca15df2 100644 --- a/ruff-strict.toml +++ b/ruff-strict.toml @@ -2,7 +2,7 @@ extend = "ruff.toml" [lint] preview = true -select = ["ANN", "ASYNC230", "B004", "B006", "B008", "B009", "B010", "B018", "B019", "B021", "B026", "B033", "BLE", "C401", "C404", "C405", "C408", "C414", "C419", "C901", "D419", "DTZ001", "DTZ003", "DTZ005", "DTZ006", "DTZ007", "DTZ011", "EXE001", "EXE002", "F401", "FURB136", "FURB168", "FURB188", "I001", "LOG015", "N999", "PERF102", "PERF401", "PERF402", "PERF403", "PIE790", "PIE800", "PIE804", "PIE810", "PLC0206", "PLC0208", "PLC0414", "PLR0124", "PLR0206", "PLR0402", "PLR1704", "PLR1711", "PLR1714", "PLR1730", "PLR2044", "PLW0127", "PLW0133", "PLW0602", "PLW0603", "PLW1508", "PLW1510", "PYI030", "PYI036", "PYI041", "PYI064", "RET501", "RET504", "RUF010", "RUF012", "RUF015", "RUF019", "RUF022", "RUF023", "RUF046", "RUF051", "RUF059", "RUF100", "S110", "S112", "SIM101", "SIM102", "SIM103", "SIM113", "SIM114", "SIM115", "SIM117", "SIM118", "SIM201", "SIM210", "SIM211", "SIM222", "SIM401", "TC004", "TC005", "TID251", "TRY002", "TRY004", "TRY201", "TRY203", "TRY300", "UP006", "UP007", "UP008", "UP012", "UP018", "UP024", "UP028", "UP031", "UP032", "UP034", "UP035", "UP036", "UP037", "UP045"] +select = ["ANN", "ASYNC230", "B004", "B006", "B008", "B009", "B010", "B018", "B019", "B021", "B026", "B033", "BLE", "C401", "C404", "C405", "C408", "C414", "C419", "C901", "D202", "D403", "D417", "D419", "DTZ001", "DTZ003", "DTZ005", "DTZ006", "DTZ007", "DTZ011", "EXE001", "EXE002", "F401", "FURB136", "FURB168", "FURB188", "I001", "LOG015", "N999", "PERF102", "PERF401", "PERF402", "PERF403", "PIE790", "PIE800", "PIE804", "PIE810", "PLC0206", "PLC0208", "PLC0414", "PLR0124", "PLR0206", "PLR0402", "PLR1704", "PLR1711", "PLR1714", "PLR1730", "PLR2044", "PLW0127", "PLW0133", "PLW0602", "PLW0603", "PLW1508", "PLW1510", "PYI030", "PYI036", "PYI041", "PYI064", "RET501", "RET504", "RUF010", "RUF012", "RUF015", "RUF019", "RUF022", "RUF023", "RUF046", "RUF051", "RUF059", "RUF100", "S110", "S112", "SIM101", "SIM102", "SIM103", "SIM113", "SIM114", "SIM115", "SIM117", "SIM118", "SIM201", "SIM210", "SIM211", "SIM222", "SIM401", "TC004", "TC005", "TID251", "TRY002", "TRY004", "TRY201", "TRY203", "TRY300", "UP006", "UP007", "UP008", "UP012", "UP018", "UP024", "UP028", "UP031", "UP032", "UP034", "UP035", "UP036", "UP037", "UP045"] extend-select = [] # Overrides the inherited list: rules this gate enforces itself must NOT be external here, # so this config's RUF100 flags their stale `# noqa` directives. What remains external is @@ -39,6 +39,9 @@ external = [ [lint.mccabe] max-complexity = 15 +[lint.pydocstyle] +convention = "google" + [lint.pylint] max-args = 5 diff --git a/ruff.toml b/ruff.toml index 93396a723e7..715e5551fad 100644 --- a/ruff.toml +++ b/ruff.toml @@ -20,7 +20,7 @@ lint.external = [ # Family entries whose every strict rule graduated into extend-select above (FURB), and # standalone graduated codes (I001, RUF010, RUF022, RUF023, RUF051), are dropped so this # config's RUF100 polices their directives itself. - "ANN", "ASYNC230", "B", "C", "D419", "DTZ", "EXE", "LOG015", "N999", "PERF", + "ANN", "ASYNC230", "B", "C", "D202", "D403", "D417", "D419", "DTZ", "EXE", "LOG015", "N999", "PERF", "PIE", "PL", "PYI", "RET", "RUF012", "RUF015", "RUF019", "RUF046", "RUF059", "S110", "S112", "SIM", "TC", "TID251", "TRY", "UP", # Enforced by upstream litellm's ruff config, but not run in this repo's CI From 7fd8740ad5cc39c47ba433ef7444b52d301a3084 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Mon, 26 Oct 2026 16:54:38 -0700 Subject: [PATCH 085/155] fix(router): isolate forwarded Anthropic subscription cooldowns (#48485) * fix: isolate forwarded Anthropic subscription cooldowns * test: fix the clock for Anthropic cooldown regression --- litellm/router.py | 2 +- tests/unit/test_router/test_router.py | 81 +++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index c76e4ec843f..d5093384a17 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -563,7 +563,7 @@ _ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset( {"model", "api_base", "api_key", "api_version", "responses_api_version"} ) _ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params" -_FORWARDED_CREDENTIAL_ISOLATION_PROVIDERS: Final = frozenset({"chatgpt"}) +_FORWARDED_CREDENTIAL_ISOLATION_PROVIDERS: Final = frozenset({"chatgpt", "anthropic"}) _CLAUDE_CODE_SESSION_ID_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$") _CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS: Final = 3600 CLAUDE_CODE_SESSION_ROUTER_BINDING_TARGET: Final = "claude_code_session_router_binding" diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 6aaa3379f47..55628cb0c34 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -30475,3 +30475,84 @@ def test_request_carries_image_sees_chat_parts_and_responses_input_items(): assert litellm.Router._request_carries_image(None, {"input": _IMAGE_RESPONSES_INPUT}) is True assert litellm.Router._request_carries_image([{"role": "user", "content": "Say hi"}], {}) is False assert litellm.Router._request_carries_image(None, {"input": "Say hi"}) is False + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("isolate_litellm_state") +@pytest.mark.parametrize("surface", ["chat", "messages"]) +@pytest.mark.parametrize("credential_kind", ["oauth", "api_key"]) +async def test_anthropic_caller_rate_limit_does_not_block_other_credentials( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, surface: str, credential_kind: str +) -> None: + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.types.router import AllowedFailsPolicy + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr("time.time", lambda: 1767225600.0) + + def upstream(request: httpx.Request) -> httpx.Response: + credential: Final = request.headers.get("authorization", request.headers.get("x-api-key", "")) + if credential.endswith("limited") and route.call_count < 2: + return httpx.Response( + 429, + headers={"retry-after": "168307"}, + json={"type": "error", "error": {"type": "rate_limit_error", "message": "usage limit"}}, + ) + return httpx.Response(200, json={ + "id": "msg-healthy", "type": "message", "role": "assistant", "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "healthy account"}], "stop_reason": "end_turn", + "usage": {"input_tokens": 2, "output_tokens": 2}, + }) + + route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(side_effect=upstream) + router: Final = Router( + model_list=[{ + "model_name": "claude", "litellm_params": { + "model": "anthropic/claude-sonnet-5-5", "api_key": "fake-gateway-key", + }, "model_info": {"id": "shared-anthropic"}, + }], + num_retries=0, + allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1), + ) + client: Final = AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport()) + + async def call(account: str | None) -> object: + credentials: Final = ( + {"provider_specific_header": { + "custom_llm_provider": "anthropic", "extra_headers": { + "Authorization": f"Bearer sk-ant-oat-fake-{account}", + }, + }} if credential_kind == "oauth" else {"api_key": f"fake-{account}"} + ) if account is not None else {} + if surface == "messages": + max_tokens: Final = litellm.get_max_tokens("anthropic/claude-sonnet-5-5") + assert max_tokens is not None + return await router.aanthropic_messages( + model="claude", messages=[{"role": "user", "content": "hello"}], + client=client, max_tokens=max_tokens, **credentials, + ) + return await router.acompletion( + model="claude", messages=[{"role": "user", "content": "hello"}], + client=client, **credentials, + ) + + try: + for _ in range(2): + with pytest.raises(litellm.RateLimitError): + await call("limited") + await GLOBAL_LOGGING_WORKER.flush() + limited_calls: Final = len(route.calls) + assert limited_calls == 2 + await call("healthy") + assert len(route.calls) == limited_calls + 1 + assert route.calls[-1].request.headers.get( + "authorization", route.calls[-1].request.headers.get("x-api-key", "") + ) == ("Bearer sk-ant-oat-fake-healthy" if credential_kind == "oauth" else "fake-healthy") + await call(None) + assert len(route.calls) == limited_calls + 2 + assert route.calls[-1].request.headers["x-api-key"] == "fake-gateway-key" + await call("limited") + assert len(route.calls) == limited_calls + 3 + finally: + await GLOBAL_LOGGING_WORKER.flush() + await client.close() From db1c871c12c2add5242f300031012d324acd4e65 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 26 Oct 2026 17:03:05 -0700 Subject: [PATCH 086/155] refactor(cost-map): drop the legacy max_tokens field in favor of max_input_tokens and max_output_tokens (#48471) * refactor(cost-map): drop the legacy max_tokens field in favor of max_input_tokens and max_output_tokens Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: drop assertions that pinned the retired cost-map max_tokens field Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 3573 ++--------------- .../mcp_server/sampling_handler.py | 2 +- litellm/router.py | 1 - .../io_token_rate_limit_check.py | 2 +- litellm/utils.py | 15 +- model_prices_and_context_window.json | 3573 ++--------------- .../test_fallback_generalizations.py | 8 +- .../openai_like/test_openinfer_provider.py | 9 +- .../test_model_prices_and_context_window.py | 23 +- tests/unit/test_together_ai_model_metadata.py | 14 - tests/unit/test_utils.py | 41 +- 11 files changed, 543 insertions(+), 6718 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d9df9d97d7b..c913c879ccf 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9,9 +9,8 @@ "input_cost_per_audio_token": 0.0, "input_cost_per_token": 0.0, "litellm_provider": "one of https://docs.litellm.ai/docs/providers", - "max_input_tokens": "max input tokens, if the provider specifies it. if not default to max_tokens", - "max_output_tokens": "max output tokens, if the provider specifies it. if not default to max_tokens", - "max_tokens": "LEGACY parameter. set to max_output_tokens if provider specifies it. IF not set to max_input_tokens, if provider specifies it.", + "max_input_tokens": "max input tokens, if the provider specifies it", + "max_output_tokens": "max output tokens, if the provider specifies it", "mode": "one of: chat, embedding, completion, image_generation, audio_transcription, audio_speech, image_generation, moderation, rerank, search", "output_cost_per_reasoning_token": 0.0, "output_cost_per_token": 0.0, @@ -49,28 +48,24 @@ "1024-x-1024/50-steps/stability.stable-diffusion-xl-v1": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.04 }, "1024-x-1024/max-steps/stability.stable-diffusion-xl-v1": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.08 }, "512-x-512/50-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.018 }, "512-x-512/max-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.036 }, @@ -79,7 +74,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8191, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.25e-05, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -89,7 +83,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8191, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.88e-05, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -100,7 +93,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-large.md" @@ -111,7 +103,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-mini.md" @@ -121,7 +112,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 70000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -298,7 +288,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", @@ -311,7 +300,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md", @@ -324,7 +312,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", @@ -337,7 +324,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md", @@ -352,7 +338,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -368,7 +353,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.4e-07, "output_cost_per_token_batches": 1.2e-07, @@ -389,7 +373,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, @@ -414,7 +397,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -435,7 +417,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.311e-06, "supports_function_calling": true, @@ -455,7 +436,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.1e-05, "source": "https://aws.amazon.com/nova/pricing/", @@ -474,7 +454,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.597e-06, "supports_function_calling": true, @@ -494,7 +473,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.1e-05, "source": "https://aws.amazon.com/nova/pricing/", @@ -518,7 +496,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.75e-06, "output_cost_per_token_batches": 1.375e-06, @@ -544,7 +521,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -562,7 +538,6 @@ "amazon.nova-2-multimodal-embeddings-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 8172, - "max_tokens": 8172, "mode": "embedding", "input_cost_per_token": 1.35e-07, "input_cost_per_image": 6e-05, @@ -583,7 +558,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.4e-07, "output_cost_per_token_batches": 7e-08, @@ -602,7 +576,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.25e-05, "output_cost_per_token_batches": 6.25e-06, @@ -626,7 +599,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.2e-06, "output_cost_per_token_batches": 1.6e-06, @@ -646,7 +618,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2.75e-06, @@ -660,7 +631,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -669,7 +639,6 @@ "input_cost_per_token": 8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128, - "max_tokens": 128, "metadata": { "notes": "'supports_image_input' is a deprecated field. Use 'supports_embedding_image_input' instead." }, @@ -684,7 +653,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536 @@ -693,7 +661,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536 @@ -702,7 +669,6 @@ "input_cost_per_token": 2e-08, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -740,7 +706,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -755,7 +720,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -770,7 +734,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -784,7 +747,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 500, - "max_tokens": 500, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 512, @@ -798,7 +760,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 500, - "max_tokens": 500, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 512, @@ -812,7 +773,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 500, - "max_tokens": 500, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 512, @@ -860,7 +820,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 6e-07 }, @@ -869,7 +828,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 2e-07 }, @@ -878,7 +836,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06 }, @@ -889,7 +846,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, @@ -909,7 +865,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", @@ -941,7 +896,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -964,7 +918,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -982,7 +935,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -1001,7 +953,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -1021,7 +972,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -1039,7 +989,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -1054,7 +1003,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-06, "supports_tool_choice": true @@ -1067,7 +1015,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -1097,7 +1044,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -1125,7 +1071,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1164,7 +1109,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1204,7 +1148,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1244,7 +1187,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1284,7 +1226,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1324,7 +1265,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1364,7 +1304,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1405,7 +1344,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "thinking_always_on": true, "supports_function_calling": true, @@ -1427,7 +1365,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1470,7 +1407,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1513,7 +1449,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1556,7 +1491,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1596,7 +1530,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -1639,7 +1572,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -1678,7 +1610,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -1721,7 +1652,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -1760,7 +1690,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "search_context_cost_per_query": { @@ -1803,7 +1732,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "search_context_cost_per_query": { @@ -1842,7 +1770,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "search_context_cost_per_query": { @@ -1880,7 +1807,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "search_context_cost_per_query": { @@ -1922,7 +1848,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1965,7 +1890,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "search_context_cost_per_query": { @@ -2010,7 +1934,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -2053,7 +1976,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "search_context_cost_per_query": { @@ -2098,7 +2020,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2141,7 +2062,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -2186,7 +2106,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2229,7 +2148,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -2269,7 +2187,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2312,7 +2229,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -2357,7 +2273,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2395,7 +2310,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -2435,7 +2349,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -2480,7 +2393,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -2525,7 +2437,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2570,7 +2481,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2615,7 +2525,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2660,7 +2569,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2703,7 +2611,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2745,7 +2652,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -2790,7 +2696,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -2835,7 +2740,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -2880,7 +2784,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -2925,7 +2828,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -2970,7 +2872,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -3011,7 +2912,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3050,7 +2950,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3089,7 +2988,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3128,7 +3026,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3167,7 +3064,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3206,7 +3102,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3246,7 +3141,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3286,7 +3180,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3319,7 +3212,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05 }, @@ -3328,7 +3220,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -3338,7 +3229,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-07 }, @@ -3347,7 +3237,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -3356,7 +3245,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/codellama-CodeLlama-70b-Instruct-hf" @@ -3366,7 +3254,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/google-gemma-7b-it" @@ -3376,7 +3263,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07 }, @@ -3385,7 +3271,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -3394,7 +3279,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07 }, @@ -3403,7 +3287,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/meta-llama-Meta-Llama-3-70B-Instruct" @@ -3413,7 +3296,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/meta-llama-Meta-Llama-3-8B-Instruct" @@ -3423,7 +3305,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mistral-7B-Instruct-v0.1", @@ -3434,7 +3315,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mixtral-8x22B-Instruct-v0.1", @@ -3445,7 +3325,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mixtral-8x7B-Instruct-v0.1", @@ -3457,7 +3336,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.52e-07, "supports_function_calling": true, @@ -3473,7 +3351,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.48e-07, "supports_function_calling": true, @@ -3489,7 +3366,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.36e-06, "output_cost_per_token_flex": 1.68e-06, @@ -3508,7 +3384,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -3543,7 +3418,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3595,7 +3469,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3628,7 +3501,6 @@ "input_cost_per_token": 1.1e-07, "litellm_provider": "azure", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -3640,7 +3512,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 6e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -3670,7 +3541,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -3685,7 +3555,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -3709,7 +3578,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": true, @@ -3734,7 +3602,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3765,7 +3632,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3797,7 +3663,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3830,7 +3695,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3868,7 +3732,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3899,7 +3762,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3938,7 +3800,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3971,7 +3832,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -3996,7 +3856,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -4030,7 +3889,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": false, @@ -4052,7 +3910,6 @@ "litellm_provider": "azure", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -4087,7 +3944,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, @@ -4105,7 +3961,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -4154,7 +4009,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, @@ -4203,7 +4057,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -4252,7 +4105,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -4304,7 +4156,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -4345,7 +4196,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", @@ -4395,7 +4245,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 6e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", @@ -4439,7 +4288,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -4486,7 +4334,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -4533,7 +4380,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -4581,7 +4427,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -4629,7 +4474,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -4675,7 +4519,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -4717,7 +4560,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "output_cost_per_token_priority": 9e-06, @@ -4759,7 +4601,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "output_cost_per_token_priority": 9e-06, @@ -4801,7 +4642,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_priority": 2.5e-06, @@ -4843,7 +4683,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_priority": 2.5e-06, @@ -4883,7 +4722,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" @@ -4895,7 +4733,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Catalog-name twin of azure_ai/model_router: the flat $0.14 per M input tokens is the router's own fee, the routed model is priced on top of it" @@ -4908,7 +4745,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -4929,7 +4765,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -4949,7 +4784,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -4968,7 +4802,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -4987,7 +4820,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5006,7 +4838,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2.2e-05, "output_cost_per_token": 2.64e-06, @@ -5026,7 +4857,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 0.00022, "output_cost_per_token": 2.2e-05, @@ -5046,7 +4876,6 @@ "litellm_provider": "azure", "max_input_tokens": 16000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 8.8e-05, "output_cost_per_token": 2.2e-05, @@ -5076,7 +4905,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -5116,7 +4944,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, "output_cost_per_token_batches": 1.1e-06, @@ -5155,7 +4982,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "supported_endpoints": [ @@ -5196,7 +5022,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5228,7 +5053,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5261,7 +5085,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-07, "output_cost_per_token_batches": 2.2e-07, @@ -5298,7 +5121,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, "output_cost_per_token_batches": 3.3e-05, @@ -5317,7 +5139,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, "output_cost_per_token_batches": 3.3e-05, @@ -5335,7 +5156,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -5353,7 +5173,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -5371,7 +5190,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -5389,7 +5207,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -5409,7 +5226,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -5426,7 +5242,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -5443,7 +5258,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -5463,7 +5277,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -5484,7 +5297,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "supported_endpoints": [ @@ -5525,7 +5337,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5557,7 +5368,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5587,7 +5397,6 @@ "litellm_provider": "azure", "max_input_tokens": 4097, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -5597,7 +5406,7 @@ "input_cost_per_token": 1.65e-06, "litellm_provider": "azure_text", "max_input_tokens": 4097, - "max_tokens": 4097, + "max_output_tokens": 4097, "mode": "completion", "output_cost_per_token": 2.2e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -5607,7 +5416,6 @@ "litellm_provider": "azure", "max_input_tokens": 4097, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -5619,7 +5427,6 @@ "litellm_provider": "azure", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5631,7 +5438,6 @@ "litellm_provider": "azure", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5642,7 +5448,7 @@ "input_cost_per_token": 1.65e-06, "litellm_provider": "azure_text", "max_input_tokens": 4097, - "max_tokens": 4097, + "max_output_tokens": 4097, "mode": "completion", "output_cost_per_token": 2.2e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -5651,7 +5457,7 @@ "input_cost_per_token": 1.65e-06, "litellm_provider": "azure_text", "max_input_tokens": 4097, - "max_tokens": 4097, + "max_output_tokens": 4097, "mode": "completion", "output_cost_per_token": 2.2e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -5662,7 +5468,6 @@ "litellm_provider": "azure", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -5674,7 +5479,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_function_calling": true, @@ -5687,7 +5491,6 @@ "litellm_provider": "azure", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5699,7 +5502,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_function_calling": true, @@ -5712,7 +5514,6 @@ "litellm_provider": "azure", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.00012, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5724,7 +5525,6 @@ "litellm_provider": "azure", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.00012, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5735,7 +5535,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.3e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/", @@ -5748,7 +5547,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.3e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/", @@ -5762,7 +5560,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.3e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/", @@ -5779,7 +5576,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -5817,7 +5613,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -5855,7 +5650,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, @@ -5893,7 +5687,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, @@ -5929,7 +5722,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -5963,7 +5755,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -5997,7 +5788,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 0.00015, "output_cost_per_token_batches": 7.5e-05, @@ -6017,7 +5807,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -6034,7 +5823,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -6053,7 +5841,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -6073,7 +5860,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -6092,7 +5878,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -6125,7 +5910,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -6157,7 +5941,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -6190,7 +5973,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -6225,7 +6007,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6257,7 +6038,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6290,7 +6070,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -6322,7 +6101,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -6340,7 +6118,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -6358,7 +6135,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 6e-07, @@ -6392,7 +6168,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6414,7 +6189,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -6449,7 +6223,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -6483,7 +6256,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -6518,7 +6290,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -6552,7 +6323,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, @@ -6587,7 +6357,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6622,7 +6391,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6656,7 +6424,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6720,7 +6487,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 0.0002, "output_cost_per_token": 2e-05, @@ -6739,7 +6505,6 @@ "litellm_provider": "azure", "max_input_tokens": 16000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 2e-05, @@ -6793,7 +6558,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "audio_transcription", "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ @@ -6817,7 +6581,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, @@ -6860,7 +6623,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, @@ -6895,7 +6657,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, "output_cost_per_token_priority": 3.6e-06, @@ -6932,7 +6693,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -6971,7 +6731,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -7008,7 +6767,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7044,7 +6802,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -7083,7 +6840,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -7122,7 +6878,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -7158,7 +6913,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -7194,7 +6948,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, @@ -7228,7 +6981,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "supported_endpoints": [ @@ -7269,7 +7021,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7294,7 +7045,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7316,7 +7066,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7337,7 +7086,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7358,7 +7106,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7380,7 +7127,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.54e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7401,7 +7147,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.54e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7422,7 +7167,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7444,7 +7188,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.54e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7465,7 +7208,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.54e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7486,7 +7228,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.6e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7502,7 +7243,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.6e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7518,7 +7258,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7550,7 +7289,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7582,7 +7320,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7618,7 +7355,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_batches": 7e-06, @@ -7658,7 +7394,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_batches": 7e-06, @@ -7696,7 +7431,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7730,7 +7464,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, @@ -7762,7 +7495,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, "output_cost_per_token_batches": 8.4e-05, @@ -7801,7 +7533,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, "output_cost_per_token_batches": 8.4e-05, @@ -7850,7 +7581,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -7906,7 +7636,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -7956,7 +7685,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -8006,7 +7734,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -8062,7 +7789,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -8112,7 +7838,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -8163,7 +7888,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -8212,7 +7936,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -8266,7 +7989,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -8327,7 +8049,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -8391,7 +8112,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -8455,7 +8175,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -8519,7 +8238,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -8583,7 +8301,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, @@ -8647,7 +8364,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, @@ -8699,7 +8415,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -8748,7 +8463,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -8797,7 +8511,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, @@ -8846,7 +8559,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, @@ -8901,7 +8613,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -8954,7 +8665,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -9001,7 +8711,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -9050,7 +8759,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -9099,7 +8807,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, @@ -9148,7 +8855,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -9197,7 +8903,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -9242,7 +8947,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "reasoning_effort_levels": [ @@ -9279,7 +8983,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "reasoning_effort_levels": [ @@ -9325,7 +9028,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, @@ -9380,7 +9082,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, @@ -9436,7 +9137,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, @@ -9492,7 +9192,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, @@ -9542,7 +9241,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "output_cost_per_token_above_272k_tokens": 8.25e-05, @@ -9591,7 +9289,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "output_cost_per_token_above_272k_tokens": 8.25e-07, @@ -9646,7 +9343,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, @@ -9689,7 +9385,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.3e-05, "reasoning_effort_levels": [ @@ -9735,7 +9430,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, @@ -9790,7 +9484,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, @@ -9846,7 +9539,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, @@ -9902,7 +9594,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, @@ -9961,7 +9652,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -10020,7 +9710,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, "output_cost_per_token_batches": 1.65e-05, @@ -10075,7 +9764,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, "output_cost_per_token_batches": 1.65e-05, @@ -10127,7 +9815,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -10187,7 +9874,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_batches": 1.5e-05, @@ -10246,7 +9932,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -10299,7 +9984,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -10353,7 +10037,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -10406,7 +10089,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -10453,7 +10135,6 @@ "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -10497,7 +10178,6 @@ "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -10539,7 +10219,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "search_context_cost_per_query": { @@ -10590,7 +10269,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "search_context_cost_per_query": { @@ -10642,7 +10320,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "search_context_cost_per_query": { @@ -10690,7 +10367,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "search_context_cost_per_query": { @@ -11066,7 +10742,7 @@ "input_cost_per_token": 8e-06, "litellm_provider": "azure", "max_input_tokens": 32000, - "max_tokens": 32000, + "max_output_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_function_calling": true @@ -11075,7 +10751,7 @@ "input_cost_per_token": 8e-06, "litellm_provider": "azure", "max_input_tokens": 32000, - "max_tokens": 32000, + "max_output_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_function_calling": true @@ -11087,7 +10763,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_function_calling": true, @@ -11105,7 +10780,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_batches": 3e-05, @@ -11124,7 +10798,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -11142,7 +10815,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -11160,7 +10832,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "supported_endpoints": [ @@ -11191,7 +10862,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -11223,7 +10893,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 4e-05, "search_context_cost_per_query": { @@ -11261,7 +10930,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supports_prompt_caching": true, @@ -11278,7 +10946,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -11295,7 +10962,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, @@ -11326,7 +10992,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, @@ -11357,7 +11022,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supported_endpoints": [ @@ -11388,7 +11052,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -11433,7 +11096,6 @@ "input_cost_per_token": 1.43e-07, "litellm_provider": "azure", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -11443,7 +11105,6 @@ "input_cost_per_token": 2.2e-08, "litellm_provider": "azure", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -11453,7 +11114,6 @@ "input_cost_per_token": 1.1e-07, "litellm_provider": "azure", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -11504,7 +11164,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -11542,7 +11201,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.76e-06, "output_cost_per_token_batches": 8.8e-07, @@ -11578,7 +11236,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4.4e-07, "output_cost_per_token_batches": 2.2e-07, @@ -11612,7 +11269,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -11633,7 +11289,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -11653,7 +11308,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -11672,7 +11326,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -11691,7 +11344,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -11710,7 +11362,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2.2e-05, "output_cost_per_token": 2.64e-06, @@ -11730,7 +11381,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 0.00022, "output_cost_per_token": 2.2e-05, @@ -11750,7 +11400,6 @@ "litellm_provider": "azure", "max_input_tokens": 16000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 8.8e-05, "output_cost_per_token": 2.2e-05, @@ -11780,7 +11429,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -11820,7 +11468,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, "output_cost_per_token_batches": 1.1e-06, @@ -11858,7 +11505,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-07, "output_cost_per_token_batches": 2.2e-07, @@ -11896,7 +11542,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "supported_endpoints": [ @@ -11937,7 +11582,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -11969,7 +11613,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12002,7 +11645,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, "output_cost_per_token_batches": 3.3e-05, @@ -12021,7 +11663,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, "output_cost_per_token_batches": 3.3e-05, @@ -12039,7 +11680,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12057,7 +11697,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -12090,7 +11729,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -12123,7 +11761,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12141,7 +11778,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12159,7 +11795,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12180,7 +11815,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12204,7 +11838,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "azure_ai", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -12215,7 +11848,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "azure_ai", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -12227,7 +11859,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12245,7 +11876,6 @@ "azure_ai/FLUX-1.1-pro": { "litellm_provider": "azure_ai", "max_input_tokens": 5000, - "max_tokens": 5000, "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659", @@ -12256,7 +11886,6 @@ "azure_ai/FLUX.1-Kontext-pro": { "litellm_provider": "azure_ai", "max_input_tokens": 5000, - "max_tokens": 5000, "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/black-forest-labs/", @@ -12267,7 +11896,6 @@ "azure_ai/flux.2-pro": { "litellm_provider": "azure_ai", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -12279,7 +11907,6 @@ "input_cost_per_pixel": 5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "image_generation", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/black-forest-labs/", "supported_endpoints": [ @@ -12301,7 +11928,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.85e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12316,7 +11942,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.828e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12332,7 +11957,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.52e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12348,7 +11972,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.84e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12363,7 +11986,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.84e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12378,7 +12000,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 7.26e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12393,7 +12014,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.46e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12415,7 +12035,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.3e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12438,7 +12057,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12461,7 +12079,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12484,7 +12101,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ @@ -12513,7 +12129,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12528,7 +12143,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 512000, "max_output_tokens": 512000, - "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12571,7 +12185,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.64e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12637,7 +12250,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12660,7 +12272,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.1e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -12672,7 +12283,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12685,7 +12295,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 10000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://azure.microsoft.com/en-us/blog/introducing-the-llama-4-herd-in-azure-ai-foundry-and-azure-databricks/", @@ -12699,7 +12308,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 8192, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.7e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12711,7 +12319,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.54e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12723,7 +12330,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12736,7 +12342,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12749,7 +12354,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12762,7 +12366,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12775,7 +12378,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12788,7 +12390,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12801,7 +12402,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12814,7 +12414,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12827,7 +12426,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12840,7 +12438,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12854,7 +12451,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12867,7 +12463,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.2e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12881,7 +12476,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12893,7 +12487,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12935,7 +12528,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 8182, - "max_tokens": 8182, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12975,7 +12567,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12989,7 +12580,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" @@ -13001,7 +12591,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" @@ -13012,7 +12601,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-cohere-rerank-4-0-in-microsoft-foundry/4477076" @@ -13023,7 +12611,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-cohere-rerank-4-0-in-microsoft-foundry/4477076" @@ -13033,7 +12620,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13048,7 +12634,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13064,7 +12649,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.56e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13076,7 +12660,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.48e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13092,7 +12675,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 5.1e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13109,7 +12691,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13122,7 +12703,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 3072, @@ -13142,7 +12722,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13159,7 +12738,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -13180,7 +12758,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -13200,7 +12777,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13219,7 +12795,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13236,7 +12811,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, @@ -13250,7 +12824,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, @@ -13264,7 +12837,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13279,7 +12851,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.00971, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" @@ -13290,7 +12861,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 70000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13302,7 +12872,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13319,7 +12888,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13343,7 +12911,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-08, "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview", @@ -13356,7 +12923,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13369,7 +12935,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13381,7 +12946,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", @@ -13393,7 +12957,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13407,7 +12970,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "supports_assistant_prefill": true, @@ -13420,7 +12982,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13432,7 +12993,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13445,7 +13005,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -13459,7 +13018,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -13471,7 +13029,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_tool_choice": true }, @@ -13481,7 +13038,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_tool_choice": true }, @@ -13491,7 +13047,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_tool_choice": true }, @@ -13501,7 +13056,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_tool_choice": true }, @@ -13526,7 +13080,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -13536,7 +13089,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/ap-northeast-1/1-month-commitment/anthropic.claude-v2:1": { @@ -13545,7 +13097,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -13555,7 +13106,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -13565,7 +13115,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/ap-northeast-1/6-month-commitment/anthropic.claude-v2:1": { @@ -13574,7 +13123,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -13583,7 +13131,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 7.55e-06, "supports_tool_choice": true @@ -13593,7 +13140,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -13603,7 +13149,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -13614,7 +13159,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -13634,7 +13178,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13652,7 +13195,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -13672,7 +13214,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, "output_cost_per_token_flex": 1.52e-06, @@ -13686,7 +13227,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -13704,7 +13244,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13723,7 +13262,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, @@ -13737,7 +13275,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, @@ -13757,7 +13294,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.2e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -13768,7 +13304,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -13779,7 +13314,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -13799,7 +13333,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13817,7 +13350,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -13837,7 +13369,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.94e-06, "output_cost_per_token_flex": 1.47e-06, @@ -13851,7 +13382,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -13869,7 +13399,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13887,7 +13416,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -13906,7 +13434,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -13926,7 +13453,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13944,7 +13470,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -13963,7 +13488,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -13981,7 +13505,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13999,7 +13522,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.03e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14010,7 +13532,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.9e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14021,7 +13542,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -14041,7 +13561,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14059,7 +13578,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14078,7 +13596,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -14096,7 +13613,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14106,7 +13622,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-v2:1": { @@ -14115,7 +13630,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14125,7 +13639,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14135,7 +13648,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/eu-central-1/6-month-commitment/anthropic.claude-v2:1": { @@ -14144,7 +13656,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14153,7 +13664,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 8.38e-06, "supports_tool_choice": true @@ -14163,7 +13673,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05 }, @@ -14172,7 +13681,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -14184,7 +13692,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14202,7 +13709,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14221,7 +13727,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14239,7 +13744,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.78e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14250,7 +13754,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14262,7 +13765,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14280,7 +13782,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14299,7 +13800,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14317,7 +13817,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.55e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14328,7 +13827,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14340,7 +13838,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, "output_cost_per_token_flex": 9.3e-07, @@ -14358,7 +13855,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14377,7 +13873,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.01e-06, "output_cost_per_token_flex": 5.05e-07, @@ -14396,7 +13891,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.86e-06, "output_cost_per_token_flex": 9.3e-07, @@ -14414,7 +13908,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", @@ -14425,7 +13918,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.56e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", @@ -14437,7 +13929,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 9.1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", @@ -14450,7 +13941,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14468,7 +13958,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14487,7 +13976,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14504,7 +13992,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Anthropic via Invoke route does not currently support pdf input." }, @@ -14523,7 +14010,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.88e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14534,7 +14020,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.01e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14545,7 +14030,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -14565,7 +14049,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14583,7 +14066,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14603,7 +14085,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, "output_cost_per_token_flex": 1.52e-06, @@ -14617,7 +14098,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -14635,7 +14115,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14653,7 +14132,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14663,7 +14141,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-v2:1": { @@ -14672,7 +14149,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14682,7 +14158,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14692,7 +14167,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/us-east-1/6-month-commitment/anthropic.claude-v2:1": { @@ -14701,7 +14175,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14710,7 +14183,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-06, "supports_tool_choice": true @@ -14720,7 +14192,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -14730,7 +14201,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -14741,7 +14211,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14752,7 +14221,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14763,7 +14231,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", @@ -14774,7 +14241,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", @@ -14786,7 +14252,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", @@ -14798,7 +14263,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_flex": 9.25e-07, @@ -14818,7 +14282,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -14838,7 +14301,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -14855,7 +14317,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, @@ -14869,7 +14330,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, @@ -14887,7 +14347,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -14905,7 +14364,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_flex": 9.25e-07, @@ -14925,7 +14383,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -14943,7 +14400,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14963,7 +14419,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, @@ -14977,7 +14432,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, @@ -14995,7 +14449,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -15013,7 +14466,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.84e-06, "supports_function_calling": true, @@ -15027,7 +14479,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536 @@ -15036,7 +14487,6 @@ "input_cost_per_token": 2e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024 @@ -15046,7 +14496,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.7e-06 }, @@ -15055,7 +14504,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 4e-07 }, @@ -15064,7 +14512,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06 }, @@ -15077,7 +14524,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -15106,7 +14552,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -15127,7 +14572,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.5e-06, "supports_pdf_input": true @@ -15137,7 +14581,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 2.65e-06, "supports_pdf_input": true @@ -15145,7 +14588,6 @@ "bedrock/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 8172, - "max_tokens": 8172, "mode": "embedding", "input_cost_per_token": 1.62e-07, "input_cost_per_image": 7.2e-05, @@ -15164,7 +14606,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.88e-07, "supports_function_calling": true, @@ -15180,7 +14621,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.68e-07, "supports_function_calling": true, @@ -15194,7 +14634,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.84e-06, "supports_function_calling": true, @@ -15208,7 +14647,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536 @@ -15217,7 +14655,6 @@ "input_cost_per_token": 1.1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024 @@ -15227,7 +14664,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.6e-06 }, @@ -15236,7 +14672,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 4e-07 }, @@ -15245,7 +14680,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06 }, @@ -15258,7 +14692,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -15287,7 +14720,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -15308,7 +14740,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.5e-06, "supports_pdf_input": true @@ -15318,7 +14749,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 6e-07, "supports_pdf_input": true @@ -15329,7 +14759,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -15340,7 +14769,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -15351,7 +14779,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -15361,7 +14788,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/us-west-2/1-month-commitment/anthropic.claude-v2:1": { @@ -15370,7 +14796,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -15380,7 +14805,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -15390,7 +14814,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/us-west-2/6-month-commitment/anthropic.claude-v2:1": { @@ -15399,7 +14822,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -15408,7 +14830,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-06, "supports_tool_choice": true @@ -15418,7 +14839,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -15428,7 +14848,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -15439,7 +14858,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", @@ -15450,7 +14868,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", @@ -15462,7 +14879,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", @@ -15474,7 +14890,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_flex": 9.25e-07, @@ -15494,7 +14909,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -15514,7 +14928,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -15531,7 +14944,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, @@ -15545,7 +14957,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, @@ -15563,7 +14974,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -15582,7 +14992,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, @@ -15672,7 +15081,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -15683,7 +15091,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -15694,7 +15101,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": true, @@ -15705,7 +15111,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -15716,7 +15121,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -15727,7 +15131,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.5e-07, "source": "https://www.cerebras.ai/blog/openai-gpt-oss-120b-runs-fastest-on-cerebras", @@ -15742,7 +15145,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://inference-docs.cerebras.ai/support/pricing", @@ -15755,7 +15157,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.49e-06, "source": "https://api.cerebras.ai/public/v1/models/qwen-3.8-27b", @@ -15771,7 +15172,6 @@ "litellm_provider": "nlp_cloud", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 5e-07 }, @@ -15802,7 +15202,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -15829,7 +15228,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -15858,7 +15256,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -15892,7 +15289,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -15928,7 +15324,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -15972,7 +15367,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -16019,7 +15413,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -16063,7 +15456,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -16092,7 +15484,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16126,7 +15517,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16163,7 +15553,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16206,7 +15595,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16249,7 +15637,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16293,7 +15680,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16337,7 +15723,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -16382,7 +15767,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -16428,7 +15812,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_batches": 1e-05, @@ -16474,7 +15857,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16522,7 +15904,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16638,7 +16019,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 3072, "max_output_tokens": 3072, - "max_tokens": 3072, "mode": "chat", "output_cost_per_token": 1.923e-06, "rpm": 300 @@ -16648,7 +16028,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 2048, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.923e-06, "rpm": 300 @@ -16658,7 +16037,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.923e-06, "rpm": 300 @@ -16668,7 +16046,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.923e-06, "rpm": 300 @@ -16678,7 +16055,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-07, "rpm": 300, @@ -16690,7 +16066,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "rpm": 300 @@ -16700,7 +16075,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 80000, "max_output_tokens": 80000, - "max_tokens": 80000, "mode": "chat", "output_cost_per_token": 3.35e-07, "rpm": 300 @@ -16710,7 +16084,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-08, "rpm": 300 @@ -16720,7 +16093,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 15000, "max_output_tokens": 15000, - "max_tokens": 15000, "mode": "chat", "output_cost_per_token": 0.0, "rpm": 300 @@ -16731,7 +16103,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "rpm": 20, @@ -16743,7 +16114,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 80000, "max_output_tokens": 80000, - "max_tokens": 80000, "mode": "chat", "output_cost_per_token": 4.881e-06, "rpm": 300, @@ -16754,7 +16124,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.87e-07, "rpm": 300 @@ -16764,7 +16133,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 60000, "max_output_tokens": 60000, - "max_tokens": 60000, "mode": "chat", "output_cost_per_token": 2.01e-07, "rpm": 300 @@ -16775,7 +16143,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "rpm": 20, @@ -16787,7 +16154,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-07, "rpm": 300, @@ -16799,7 +16165,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "rpm": 300 @@ -16809,7 +16174,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 24000, "max_output_tokens": 24000, - "max_tokens": 24000, "mode": "chat", "output_cost_per_token": 2.253e-06, "rpm": 300, @@ -16820,7 +16184,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1.12e-07, "rpm": 300, @@ -16831,7 +16194,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1e-06, "rpm": 300 @@ -16842,7 +16204,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "rpm": 20, @@ -16854,7 +16215,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.5e-06, "rpm": 300, @@ -16866,7 +16226,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.55e-07, "rpm": 300 @@ -16876,7 +16235,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.35e-07, "rpm": 300, @@ -16888,7 +16246,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 3500, "max_output_tokens": 3500, - "max_tokens": 3500, "mode": "chat", "output_cost_per_token": 0.0, "rpm": 300 @@ -16898,7 +16255,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3e-07, "rpm": 300, @@ -16910,7 +16266,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.55e-07, "rpm": 300, @@ -16921,7 +16276,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.76e-07, "rpm": 300, @@ -16932,7 +16286,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "rpm": 300, @@ -16944,7 +16297,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 8.5e-07, "rpm": 300, @@ -16955,7 +16307,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 24000, "max_output_tokens": 24000, - "max_tokens": 24000, "mode": "chat", "output_cost_per_token": 1e-06, "rpm": 300, @@ -16966,7 +16317,6 @@ "litellm_provider": "codestral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/capabilities/code_generation/", @@ -16978,7 +16328,6 @@ "litellm_provider": "codestral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/capabilities/code_generation/", @@ -16990,7 +16339,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -17001,7 +16349,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -17011,7 +16358,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -17021,7 +16367,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -17031,7 +16376,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -17042,7 +16386,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -17053,7 +16396,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -17064,7 +16406,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "cohere", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -17076,7 +16417,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "cohere", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048, @@ -17087,7 +16427,6 @@ "input_cost_per_token": 8e-08, "litellm_provider": "cohere", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048, @@ -17109,7 +16448,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-cohere-rerank-3-5.md" @@ -17119,7 +16457,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -17131,7 +16468,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "source": "https://docs.cohere.com/docs/models" }, @@ -17140,7 +16476,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.cohere.com/docs/command-a-plus", @@ -17155,7 +16490,6 @@ "litellm_provider": "cohere", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 2e-06 }, @@ -17164,7 +16498,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -17175,7 +16508,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -17186,7 +16518,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.cohere.com/v2/docs/command-r7b", @@ -17199,7 +16530,7 @@ "input_cost_per_token": 1.12e-06, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17222,7 +16553,7 @@ "input_cost_per_token": 1.12e-06, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17245,7 +16576,7 @@ "input_cost_per_token": 1.12e-06, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17268,7 +16599,7 @@ "input_cost_per_token": 1e-08, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17291,7 +16622,7 @@ "input_cost_per_token": 1e-08, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17314,7 +16645,7 @@ "input_cost_per_token": 1e-08, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17337,7 +16668,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, "supported_endpoints": [ @@ -17367,7 +16697,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, "supported_endpoints": [ @@ -17398,7 +16727,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -17420,7 +16748,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -17442,7 +16769,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17458,7 +16784,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17474,7 +16799,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4.8e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17490,7 +16814,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 202745, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17506,7 +16829,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17522,7 +16844,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 229376, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17538,7 +16859,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17550,7 +16870,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17579,7 +16898,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17609,7 +16927,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 30720, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17622,7 +16939,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17635,7 +16951,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17648,7 +16963,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -17662,7 +16976,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -17675,7 +16988,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17706,7 +17018,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17737,7 +17048,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17769,7 +17079,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -17783,7 +17092,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17796,7 +17104,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -17810,7 +17117,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -17823,7 +17129,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17834,7 +17139,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17883,7 +17187,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17928,7 +17231,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17977,7 +17279,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18022,7 +17323,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18059,7 +17359,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18096,7 +17395,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18134,7 +17432,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18146,7 +17443,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18159,7 +17455,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18172,7 +17467,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18186,7 +17480,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18199,7 +17492,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.87e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18212,7 +17504,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 260096, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18250,7 +17541,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18284,7 +17574,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -18298,7 +17587,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18334,7 +17622,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -18352,7 +17639,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://docs.modelstudio.console.alibabacloud.com/en/model-studio/model-pricing", @@ -18371,7 +17657,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://docs.modelstudio.console.alibabacloud.com/en/model-studio/model-pricing", @@ -18390,7 +17675,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 98304, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -18711,7 +17995,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.qwencloud.com/models", @@ -18727,7 +18010,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.qwencloud.com/models", @@ -18743,7 +18025,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4.8e-06, "source": "https://www.qwencloud.com/models", @@ -18759,7 +18040,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 202745, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.qwencloud.com/models", @@ -18775,7 +18055,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.qwencloud.com/models", @@ -18791,7 +18070,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 229376, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.qwencloud.com/models", @@ -18807,7 +18085,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://www.qwencloud.com/models", @@ -18819,7 +18096,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -18848,7 +18124,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -18878,7 +18153,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 30720, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.4e-06, "source": "https://www.qwencloud.com/models", @@ -18891,7 +18165,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.qwencloud.com/models", @@ -18904,7 +18177,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.qwencloud.com/models", @@ -18917,7 +18189,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -18931,7 +18202,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -18944,7 +18214,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -18975,7 +18244,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19006,7 +18274,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19038,7 +18305,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -19052,7 +18318,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://www.qwencloud.com/models", @@ -19065,7 +18330,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -19079,7 +18343,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -19092,7 +18355,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19103,7 +18365,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19152,7 +18413,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19197,7 +18457,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19246,7 +18505,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19291,7 +18549,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19328,7 +18585,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19365,7 +18621,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19403,7 +18658,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.qwencloud.com/models", @@ -19415,7 +18669,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.qwencloud.com/models", @@ -19428,7 +18681,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://www.qwencloud.com/models", @@ -19441,7 +18693,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.qwencloud.com/models", @@ -19455,7 +18706,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://www.qwencloud.com/models", @@ -19468,7 +18718,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.87e-06, "source": "https://www.qwencloud.com/models", @@ -19481,7 +18730,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 260096, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19519,7 +18767,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19553,7 +18800,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://www.qwencloud.com/models", @@ -19567,7 +18813,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19603,7 +18848,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://www.qwencloud.com/models", @@ -19619,7 +18863,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 98304, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://www.qwencloud.com/models", @@ -19940,7 +19183,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -19956,7 +19198,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -19972,7 +19213,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4.8e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -19988,7 +19228,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 202745, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20004,7 +19243,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20020,7 +19258,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 229376, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20036,7 +19273,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20048,7 +19284,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20077,7 +19312,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20107,7 +19341,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 30720, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20120,7 +19353,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20133,7 +19365,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20146,7 +19377,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -20160,7 +19390,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -20173,7 +19402,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20204,7 +19432,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20235,7 +19462,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20267,7 +19493,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -20281,7 +19506,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20294,7 +19518,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -20308,7 +19531,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -20321,7 +19543,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20332,7 +19553,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20381,7 +19601,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20426,7 +19645,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20475,7 +19693,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20520,7 +19737,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20557,7 +19773,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20594,7 +19809,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20632,7 +19846,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20644,7 +19857,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20657,7 +19869,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20670,7 +19881,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20684,7 +19894,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20697,7 +19906,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.87e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20710,7 +19918,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 260096, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20748,7 +19955,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20782,7 +19988,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20796,7 +20001,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20832,7 +20036,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20850,7 +20053,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://docs.modelstudio.console.alibabacloud.com/en/model-studio/model-pricing", @@ -20869,7 +20071,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://docs.modelstudio.console.alibabacloud.com/en/model-studio/model-pricing", @@ -20888,7 +20089,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 98304, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -21210,7 +20410,6 @@ "input_dbu_cost_per_token": 1.429e-06, "litellm_provider": "databricks", "max_input_tokens": 512, - "max_tokens": 512, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21228,7 +20427,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21257,7 +20455,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21287,7 +20484,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21311,7 +20507,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21335,7 +20530,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21359,7 +20553,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21384,7 +20577,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21409,7 +20601,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21436,7 +20627,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21464,7 +20654,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21493,7 +20682,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21528,7 +20716,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21552,7 +20739,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21575,7 +20761,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21599,7 +20784,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21624,7 +20808,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields. Introductory launch rates of 28.571 input / 142.857 output / 35.714 cache write / 2.857 cache read DBU run through 2026-08-31; the standard rates are listed here because entries carry no expiry date." }, @@ -21653,7 +20836,6 @@ "token_count_framing": 97, "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference. Context/max output are the DeepSeek-published model limits (1M context, 384K max output)." }, @@ -21686,7 +20868,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference. Context/max output are the DeepSeek-published model limits (1M context, 384K max output)." }, @@ -21715,7 +20896,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21749,7 +20929,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21784,7 +20963,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21812,7 +20990,6 @@ "litellm_provider": "databricks", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "metadata": { "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." }, @@ -21833,7 +21010,6 @@ "litellm_provider": "databricks", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "metadata": { "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." }, @@ -21858,7 +21034,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21891,7 +21066,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21923,7 +21097,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21939,7 +21112,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." }, @@ -21964,7 +21136,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." }, @@ -21993,7 +21164,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22025,7 +21195,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22056,7 +21225,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22089,7 +21257,6 @@ "litellm_provider": "databricks", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22106,7 +21273,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference." }, @@ -22134,7 +21300,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22165,7 +21330,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22195,7 +21359,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22215,7 +21378,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22235,7 +21397,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22255,7 +21416,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22275,7 +21435,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22304,7 +21463,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22333,7 +21491,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22362,7 +21519,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields. Rates reflect OpenAI's promotional pricing in effect through November 21, 2026; afterwards input, cache and Batch rates are 25% higher and output rates 50% higher." }, @@ -22391,7 +21547,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22420,7 +21575,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22449,7 +21603,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22478,7 +21631,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22507,7 +21659,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22527,7 +21678,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22547,7 +21697,6 @@ "litellm_provider": "databricks", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22564,7 +21713,6 @@ "litellm_provider": "databricks", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22606,7 +21754,6 @@ "input_dbu_cost_per_token": 1.857e-06, "litellm_provider": "databricks", "max_input_tokens": 8192, - "max_tokens": 8192, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22653,7 +21800,6 @@ "token_count_framing": 100, "max_input_tokens": 1000000, "max_output_tokens": 1048576, - "max_tokens": 1048576, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference." }, @@ -22687,7 +21833,6 @@ "litellm_provider": "databricks", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)." }, @@ -22705,7 +21850,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22722,7 +21866,6 @@ "litellm_provider": "databricks", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22740,7 +21883,6 @@ "litellm_provider": "databricks", "max_input_tokens": 262144, "max_output_tokens": 25000, - "max_tokens": 25000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22790,7 +21932,6 @@ "input_dbu_cost_per_token": 2.86e-07, "litellm_provider": "databricks", "max_input_tokens": 32768, - "max_tokens": 32768, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22836,7 +21977,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -23431,7 +22571,6 @@ ] }, "deepinfra/Gryphe/MythoMax-L2-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 4e-07, @@ -23443,7 +22582,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/NousResearch/Hermes-3-Llama-3.1-405B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-06, @@ -23454,7 +22592,6 @@ "supports_function_calling": true }, "deepinfra/NousResearch/Hermes-3-Llama-3.1-70B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 7e-07, @@ -23465,7 +22602,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/QwQ-32B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.5e-07, @@ -23476,7 +22612,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen2.5-72B-Instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 3.6e-07, @@ -23488,7 +22623,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen2.5-7B-Instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 4e-08, @@ -23498,7 +22632,6 @@ "supports_tool_choice": false }, "deepinfra/Qwen/Qwen2.5-VL-32B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -23510,7 +22643,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen3-14B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1.2e-07, @@ -23522,7 +22654,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-235B-A22B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1.8e-07, @@ -23533,7 +22664,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen3-235B-A22B-Instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-08, @@ -23545,7 +22675,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-235B-A22B-Thinking-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 3e-07, @@ -23556,7 +22685,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen3-30B-A3B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1.2e-07, @@ -23568,7 +22696,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-32B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 8e-08, @@ -23580,7 +22707,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 4e-07, @@ -23591,7 +22717,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct-Turbo": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 3e-07, @@ -23605,7 +22730,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Next-80B-A3B-Instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-08, @@ -23617,7 +22741,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Next-80B-A3B-Thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.4e-07, @@ -23628,7 +22751,6 @@ "supports_function_calling": true }, "deepinfra/Sao10K/L3-8B-Lunaris-v1-Turbo": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 4e-08, @@ -23638,7 +22760,6 @@ "supports_tool_choice": false }, "deepinfra/Sao10K/L3.1-70B-Euryale-v2.2": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 8.5e-07, @@ -23649,7 +22770,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Sao10K/L3.3-70B-Euryale-v2.3": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6.5e-07, @@ -23659,7 +22779,6 @@ "supports_tool_choice": false }, "deepinfra/allenai/olmOCR-7B-0725-FP8": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2.7e-07, @@ -23669,7 +22788,6 @@ "supports_tool_choice": false }, "deepinfra/anthropic/claude-3-7-sonnet-latest": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 3.3e-06, @@ -23681,7 +22799,6 @@ "supports_function_calling": true }, "deepinfra/anthropic/claude-4-opus": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 1.65e-05, @@ -23692,7 +22809,6 @@ "supports_function_calling": true }, "deepinfra/anthropic/claude-4-sonnet": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 3.3e-06, @@ -23703,7 +22819,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 7e-07, @@ -23714,7 +22829,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1-0528": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 5e-07, @@ -23726,7 +22840,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1-0528-Turbo": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-06, @@ -23737,7 +22850,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -23747,7 +22859,6 @@ "supports_tool_choice": false }, "deepinfra/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2.7e-07, @@ -23758,7 +22869,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1-Turbo": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1e-06, @@ -23769,7 +22879,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-V3": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 3.2e-07, @@ -23781,7 +22890,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3-0324": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 2.4e-07, @@ -23795,7 +22903,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3.1": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 2.5e-07, @@ -23809,7 +22916,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3.1-Terminus": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 2.7e-07, @@ -23821,7 +22927,6 @@ "supports_function_calling": true }, "deepinfra/google/gemini-2.5-flash": { - "max_tokens": 1000000, "max_input_tokens": 1000000, "max_output_tokens": 1000000, "input_cost_per_token": 3e-07, @@ -23837,7 +22942,6 @@ "supports_audio_input": true }, "deepinfra/google/gemini-2.5-pro": { - "max_tokens": 1000000, "max_input_tokens": 1000000, "max_output_tokens": 1000000, "input_cost_per_token": 1.25e-06, @@ -23852,7 +22956,6 @@ "supports_audio_input": true }, "deepinfra/google/gemma-3-12b-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-08, @@ -23864,7 +22967,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-3-27b-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 8e-08, @@ -23876,7 +22978,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-3-4b-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-08, @@ -23888,7 +22989,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-3.2-11B-Vision-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4.9e-08, @@ -23898,7 +22998,6 @@ "supports_tool_choice": false }, "deepinfra/meta-llama/Llama-3.2-3B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-08, @@ -23909,7 +23008,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Llama-3.3-70B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2.3e-07, @@ -23920,7 +23018,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Llama-3.3-70B-Instruct-Turbo": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -23932,7 +23029,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 2e-07, @@ -23944,7 +23040,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-4-Scout-17B-16E-Instruct": { - "max_tokens": 327680, "max_input_tokens": 327680, "max_output_tokens": 327680, "input_cost_per_token": 1e-07, @@ -23956,7 +23051,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-Guard-3-8B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5.5e-08, @@ -23966,7 +23060,6 @@ "supports_tool_choice": false }, "deepinfra/meta-llama/Llama-Guard-4-12B": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 1.8e-07, @@ -23976,7 +23069,6 @@ "supports_tool_choice": false }, "deepinfra/meta-llama/Meta-Llama-3-8B-Instruct": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 3e-08, @@ -23987,7 +23079,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Meta-Llama-3.1-70B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4e-07, @@ -23998,7 +23089,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4e-07, @@ -24010,7 +23100,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, @@ -24021,7 +23110,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-08, @@ -24033,7 +23121,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/microsoft/WizardLM-2-8x22B": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 4.8e-07, @@ -24043,7 +23130,6 @@ "supports_tool_choice": false }, "deepinfra/microsoft/phi-4": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 7e-08, @@ -24054,7 +23140,6 @@ "supports_function_calling": true }, "deepinfra/mistralai/Mistral-Nemo-Instruct-2407": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.9e-08, @@ -24066,7 +23151,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/mistralai/Mistral-Small-24B-Instruct-2501": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-08, @@ -24077,7 +23161,6 @@ "supports_function_calling": true }, "deepinfra/mistralai/Mistral-Small-3.2-24B-Instruct-2506": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 7.5e-08, @@ -24088,7 +23171,6 @@ "supports_function_calling": true }, "deepinfra/mistralai/Mixtral-8x7B-Instruct-v0.1": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 4e-07, @@ -24099,7 +23181,6 @@ "supports_function_calling": true }, "deepinfra/moonshotai/Kimi-K2-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-07, @@ -24110,7 +23191,6 @@ "supports_function_calling": true }, "deepinfra/moonshotai/Kimi-K2-Instruct-0905": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 5e-07, @@ -24122,7 +23202,6 @@ "supports_function_calling": true }, "deepinfra/nvidia/Llama-3.1-Nemotron-70B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6e-07, @@ -24133,7 +23212,6 @@ "supports_function_calling": true }, "deepinfra/nvidia/Llama-3.3-Nemotron-Super-49B-v1.5": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -24157,7 +23235,6 @@ "supports_prompt_caching": true }, "deepinfra/nvidia/NVIDIA-Nemotron-Nano-9B-v2": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4e-08, @@ -24168,7 +23245,6 @@ "supports_function_calling": true }, "deepinfra/openai/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3.7e-08, @@ -24180,7 +23256,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/openai/gpt-oss-20b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, @@ -24192,7 +23267,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.5": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4e-07, @@ -24211,7 +23285,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -24234,7 +23307,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-07, "supports_assistant_prefill": true, @@ -24249,7 +23321,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.19e-06, "supports_assistant_prefill": true, @@ -24266,7 +23337,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -24291,7 +23361,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.1e-06, "supports_assistant_prefill": true, @@ -24306,7 +23375,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 4e-07, "supports_assistant_prefill": true, @@ -24321,7 +23389,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-1.html", @@ -24338,7 +23405,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 164000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_batches": 9.25e-07, @@ -24358,7 +23424,6 @@ "litellm_provider": "nlp_cloud", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "completion", "output_cost_per_token": 5e-07 }, @@ -24367,7 +23432,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 98304, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "supports_assistant_prefill": true, @@ -24381,7 +23445,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "supports_assistant_prefill": true, @@ -24395,7 +23458,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 229376, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "supports_assistant_prefill": true, @@ -24408,7 +23470,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - standard version with 2560 dimensions" }, @@ -24420,7 +23481,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - large version with 2048 dimensions" }, @@ -24432,7 +23492,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - text-240915 version with 4096 dimensions" }, @@ -24444,7 +23503,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - text-250515 version with 2048 dimensions" }, @@ -24456,7 +23514,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - text-240715 version with 2560 dimensions" }, @@ -24569,7 +23626,6 @@ "mode": "search" }, "sail/moonshotai/Kimi-K3": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 2.5e-06, @@ -24591,7 +23647,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/zai-org/GLM-5.3": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 9.8e-07, @@ -24613,7 +23668,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/zai-org/GLM-5.3-Flash": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1.1e-07, @@ -24635,7 +23689,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/deepseek-ai/DeepSeek-V4-Pro-0813": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 9.2e-07, @@ -24657,7 +23710,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 9e-08, @@ -24679,7 +23731,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/deepseek-ai/DeepSeek-V4.1-Flash": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 9e-08, @@ -24701,7 +23752,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/moonshotai/Kimi-K2.6": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1e-06, @@ -24724,7 +23774,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/google/gemma-4-31B-it": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 4e-07, @@ -24747,7 +23796,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/nvidia/Gemma-4-31B-IT-NVFP4": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.4e-07, @@ -24770,7 +23818,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/google/gemma-4-12B-it": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 3e-07, @@ -24792,7 +23839,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/openai/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6e-08, @@ -24808,7 +23854,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/Qwen/Qwen3.6-35B-A3B": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 5e-08, @@ -25047,7 +24092,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "cohere", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -25056,7 +24100,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "cohere", "max_input_tokens": 512, - "max_tokens": 512, "metadata": { "notes": "'supports_image_input' is a deprecated field. Use 'supports_embedding_image_input' instead." }, @@ -25069,7 +24112,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "cohere", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "supports_embedding_image_input": true @@ -25078,7 +24120,6 @@ "input_cost_per_token": 0.0001, "litellm_provider": "cohere", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "supports_embedding_image_input": true @@ -25089,7 +24130,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.12e-07, "supports_function_calling": true, @@ -25105,7 +24145,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.84e-07, "supports_function_calling": true, @@ -25121,7 +24160,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 4.2e-06, "output_cost_per_token_flex": 2.1e-06, @@ -25138,7 +24176,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, @@ -25161,7 +24198,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -25189,7 +24225,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -25205,7 +24240,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -25224,7 +24258,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -25244,7 +24277,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -25261,7 +24293,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -25292,7 +24323,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -25323,7 +24353,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -25363,7 +24392,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -25397,7 +24425,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.3e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", @@ -25410,7 +24437,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.9e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", @@ -25422,7 +24448,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-pixtral-large.md", @@ -27688,14 +26713,12 @@ "litellm_provider": "featherless_ai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat" }, "featherless_ai/featherless-ai/Qwerky-QwQ-32B": { "litellm_provider": "featherless_ai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat" }, "fireworks-ai-4.1b-to-16b": { @@ -27743,7 +26766,6 @@ "input_cost_per_token": 1.6e-08, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -27753,7 +26775,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://fireworks.ai/pricing", @@ -27766,7 +26787,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 20480, - "max_tokens": 20480, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://fireworks.ai/pricing", @@ -27778,7 +26798,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 160000, - "max_tokens": 160000, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://fireworks.ai/pricing", @@ -27790,7 +26809,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 20480, - "max_tokens": 20480, "mode": "chat", "output_cost_per_token": 2.19e-06, "source": "https://fireworks.ai/pricing", @@ -27802,7 +26820,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -27814,7 +26831,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/models/fireworks/deepseek-v3-0324", @@ -27826,7 +26842,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://fireworks.ai/pricing", @@ -27839,7 +26854,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://fireworks.ai/pricing", @@ -27852,7 +26866,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://fireworks.ai/models/fireworks/deepseek-v3p2", @@ -27867,7 +26880,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -27886,7 +26898,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.96e-06, "output_cost_per_token_priority": 4.95e-06, @@ -27906,7 +26917,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.96e-06, "output_cost_per_token_priority": 4.95e-06, @@ -27922,7 +26932,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -27935,7 +26944,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 96000, - "max_tokens": 96000, "mode": "chat", "output_cost_per_token": 2.19e-06, "source": "https://fireworks.ai/models/fireworks/glm-4p5", @@ -27949,7 +26957,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 96000, - "max_tokens": 96000, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://artificialanalysis.ai/models/glm-4-5-air", @@ -27964,7 +26971,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 202800, - "max_tokens": 202800, "mode": "chat", "source": "https://fireworks.ai/pricing", "supports_function_calling": true, @@ -27978,7 +26984,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 202800, - "max_tokens": 202800, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://fireworks.ai/models/fireworks/glm-4p7", @@ -27993,7 +26998,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28012,7 +27016,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_priority": 5.5e-06, @@ -28032,7 +27035,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 7.2e-07, @@ -28049,7 +27051,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28064,7 +27065,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://fireworks.ai/models/fireworks/kimi-k2-instruct", @@ -28077,7 +27077,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://app.fireworks.ai/models/fireworks/kimi-k2-instruct-0905", @@ -28090,7 +27089,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://fireworks.ai/pricing", @@ -28105,7 +27103,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -28122,7 +27119,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "output_cost_per_token_priority": 6e-06, @@ -28142,7 +27138,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "output_cost_per_token_priority": 6e-06, @@ -28158,7 +27153,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -28171,7 +27165,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://fireworks.ai/pricing", @@ -28184,7 +27177,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://fireworks.ai/pricing", @@ -28198,7 +27190,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://fireworks.ai/pricing", @@ -28211,7 +27202,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://fireworks.ai/pricing", @@ -28224,7 +27214,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -28237,7 +27226,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://fireworks.ai/pricing", @@ -28249,7 +27237,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://fireworks.ai/pricing", @@ -28262,7 +27249,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 196608, "max_output_tokens": 204800, - "max_tokens": 204800, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://fireworks.ai/models/fireworks/minimax-m2p1", @@ -28278,7 +27264,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 512000, "max_output_tokens": 512000, - "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.8e-06, @@ -28294,7 +27279,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://fireworks.ai/pricing", @@ -28307,7 +27291,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -28320,7 +27303,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -28333,7 +27315,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -28347,7 +27328,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28363,7 +27343,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 202800, - "max_tokens": 202800, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://fireworks.ai/models/fireworks/glm-4p7", @@ -28378,7 +27357,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28394,7 +27372,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28413,7 +27390,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_priority": 5.5e-06, @@ -28433,7 +27409,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 7.2e-07, @@ -28450,7 +27425,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28466,7 +27440,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -28483,7 +27456,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "output_cost_per_token_priority": 6e-06, @@ -28501,7 +27473,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28520,7 +27491,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "output_cost_per_token_priority": 6e-06, @@ -28538,7 +27508,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28554,7 +27523,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 196608, "max_output_tokens": 204800, - "max_tokens": 204800, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://fireworks.ai/models/fireworks/minimax-m2p1", @@ -28570,7 +27538,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 512000, "max_output_tokens": 512000, - "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.8e-06, @@ -28587,7 +27554,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -28601,7 +27567,6 @@ "input_cost_per_token": 8e-09, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -28610,7 +27575,6 @@ "input_cost_per_token": 8e-09, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -28619,7 +27583,6 @@ "input_cost_per_token": 8e-09, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -28628,7 +27591,6 @@ "input_cost_per_token": 1.6e-08, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -28637,7 +27599,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 3e-08, @@ -28665,7 +27626,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "input_cost_per_token": 1.26e-06, "output_cost_per_token": 3.96e-06, "cache_read_input_token_cost": 2.34e-07, @@ -28693,7 +27653,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 4e-07, "supports_prompt_caching": false, @@ -28716,7 +27675,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "input_cost_per_token": 1.4e-06, "output_cost_per_token": 4.4e-06, "cache_read_input_token_cost": 2.6e-07, @@ -28743,7 +27701,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "input_cost_per_token": 5e-07, "output_cost_per_token": 1.5e-06, "cache_read_input_token_cost": 2.5e-07, @@ -28767,7 +27724,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 196608, "max_output_tokens": 196608, - "max_tokens": 196608, "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 6e-08, @@ -28791,7 +27747,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 202752, "max_output_tokens": 202752, - "max_tokens": 202752, "input_cost_per_token": 1.4e-06, "output_cost_per_token": 4.4e-06, "cache_read_input_token_cost": 2.6e-07, @@ -28818,7 +27773,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 9e-07, @@ -28831,7 +27785,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, @@ -28844,7 +27797,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_batches": 3e-06, @@ -28858,7 +27810,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "supports_system_messages": true, @@ -28870,7 +27821,6 @@ "litellm_provider": "openai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "supports_system_messages": true, @@ -28882,7 +27832,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "supports_system_messages": true, @@ -28894,7 +27843,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "source": "OpenAI needs to add pricing for this ft model, will be updated when added by OpenAI. Defaulting to base model pricing", @@ -28911,7 +27859,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -28932,7 +27879,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -28952,7 +27898,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_batches": 6e-07, @@ -28974,7 +27919,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, @@ -28995,7 +27939,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.2e-06, "output_cost_per_token_batches": 1.6e-06, @@ -29016,7 +27959,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, "output_cost_per_token_batches": 4e-07, @@ -29037,7 +27979,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 1.6e-05, "output_cost_per_token_batches": 8e-06, @@ -29060,7 +28001,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -29117,7 +28057,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, @@ -29175,7 +28114,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -29229,7 +28167,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -29281,7 +28218,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, @@ -29327,7 +28263,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, @@ -29376,7 +28311,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -29416,7 +28350,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -29475,7 +28408,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -29535,7 +28467,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -29591,7 +28522,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -29629,7 +28559,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, @@ -29684,7 +28613,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, @@ -29731,7 +28659,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -29778,7 +28705,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -29824,7 +28750,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -29868,7 +28793,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -29920,7 +28844,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -29982,7 +28905,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_image": 0.00012, "output_cost_per_token": 1.2e-05, @@ -30035,7 +28957,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -30088,7 +29009,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -30149,7 +29069,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, @@ -30196,7 +29115,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -30259,7 +29177,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -30320,7 +29237,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -30384,7 +29300,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -30440,7 +29355,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 7.5e-06, "output_cost_per_token": 7.5e-06, @@ -30488,7 +29402,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -30531,7 +29444,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_image": 0.00012, "output_cost_per_token": 1.2e-05, @@ -30584,7 +29496,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -30641,7 +29552,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, @@ -30674,7 +29584,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_reasoning_token": 2.5e-06, @@ -30712,7 +29621,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, @@ -30759,7 +29667,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -30783,7 +29690,6 @@ "input_cost_per_token_batches": 1.2e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30800,7 +29706,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30818,7 +29723,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30837,7 +29741,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30856,7 +29759,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30870,7 +29772,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30882,7 +29783,6 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "gemini", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30901,7 +29801,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "gemini", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30922,7 +29821,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -30968,7 +29866,6 @@ "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "image_generation", "output_cost_per_image": 0.045, "output_cost_per_image_token": 6e-05, @@ -31020,7 +29917,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -31080,7 +29976,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "gemini", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -31097,7 +29992,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "rpm": 1000, @@ -31134,7 +30028,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "rpm": 1000, @@ -31180,7 +30073,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -31243,7 +30135,6 @@ "supports_reasoning": false, "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, @@ -31295,7 +30186,6 @@ "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -31344,7 +30234,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -31387,7 +30276,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.045, "output_cost_per_image_token": 6e-05, @@ -31429,7 +30317,6 @@ "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -31468,7 +30355,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, @@ -31513,7 +30399,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, @@ -31570,7 +30455,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -31629,7 +30513,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -31688,7 +30571,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, @@ -31718,7 +30600,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -31772,7 +30653,6 @@ "litellm_provider": "gemini", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -31815,7 +30695,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -31877,7 +30756,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -31940,7 +30818,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, @@ -32002,7 +30879,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -32067,7 +30943,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32129,7 +31004,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32192,7 +31066,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32250,7 +31123,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -32288,7 +31160,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -32352,7 +31223,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -32421,7 +31291,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, @@ -32466,7 +31335,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -32501,7 +31369,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -32564,7 +31431,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32625,7 +31491,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32689,7 +31554,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32745,7 +31609,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 7.5e-06, "output_cost_per_token": 7.5e-06, @@ -32792,7 +31655,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, @@ -32827,7 +31689,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "metadata": { "notes": "Rate limits not documented for gemini-exp-1114. Assuming same as gemini-1.5-pro.", "supports_tool_choice": true @@ -32850,7 +31711,6 @@ "litellm_provider": "gemini", "max_input_tokens": 2097152, "max_output_tokens": 8192, - "max_tokens": 8192, "metadata": { "notes": "Rate limits not documented for gemini-exp-1206. Assuming same as gemini-1.5-pro.", "supports_tool_choice": true @@ -32871,7 +31731,7 @@ "input_cost_per_token": 3.5e-07, "litellm_provider": "gemini", "max_output_tokens": 8192, - "max_tokens": 8192, + "max_input_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.05e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", @@ -32885,7 +31745,7 @@ "input_cost_per_token": 3.5e-07, "litellm_provider": "gemini", "max_output_tokens": 8192, - "max_tokens": 8192, + "max_input_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.05e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", @@ -32909,7 +31769,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_character": 0, "output_cost_per_character_above_128k_tokens": 0, @@ -32929,7 +31788,6 @@ "litellm_provider": "gemini", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, @@ -32945,7 +31803,6 @@ "litellm_provider": "gemini", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, @@ -32981,7 +31838,6 @@ "litellm_provider": "gemini", "max_input_tokens": 32767, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_character": 0, "output_cost_per_character_above_128k_tokens": 0, @@ -33000,7 +31856,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_image": 0.04, "output_cost_per_token": 0, @@ -33025,7 +31880,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -33049,7 +31903,6 @@ "deprecation_date": "2026-10-22", "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -33066,7 +31919,6 @@ "deprecation_date": "2026-10-22", "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, @@ -33082,7 +31934,6 @@ "deprecation_date": "2026-10-22", "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.05, "output_cost_per_second_1080p": 0.08, @@ -33097,7 +31948,6 @@ "gemini/veo-3.1-fast-generate-001": { "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -33113,7 +31963,6 @@ "gemini/veo-3.1-generate-001": { "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, @@ -33132,7 +31981,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33155,7 +32003,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33178,7 +32025,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 5e-06, "supported_endpoints": [ @@ -33199,7 +32045,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_100k_tokens": 2.5e-06, @@ -33224,7 +32069,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33247,7 +32091,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33270,7 +32113,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33292,7 +32134,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33315,7 +32156,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33337,7 +32177,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33360,7 +32199,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.75e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33379,7 +32217,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.75e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33396,7 +32233,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true }, @@ -33404,7 +32240,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true }, @@ -33412,7 +32247,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true }, @@ -33420,7 +32254,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true }, @@ -33428,7 +32261,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -33438,7 +32270,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "supports_function_calling": true, @@ -33451,7 +32282,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "supports_function_calling": true, @@ -33467,7 +32297,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -33477,7 +32306,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -33487,7 +32315,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -33496,7 +32323,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -33506,7 +32332,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -33515,7 +32340,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -33527,7 +32351,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://github.blog/changelog/2026-10-19-github-copilot-models-deprecated-on-october-19-2026", @@ -33542,7 +32365,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "supported_endpoints": [ @@ -33563,7 +32385,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -33585,7 +32406,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 4.5e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33607,7 +32427,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -33631,7 +32450,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_200k_tokens": 1.8e-06, @@ -33655,7 +32473,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -33680,7 +32497,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -33704,7 +32520,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -33728,7 +32543,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, @@ -33752,7 +32566,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -33776,7 +32589,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -33796,7 +32608,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 917504, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33814,7 +32625,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.2e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33829,19 +32639,16 @@ "github_copilot/text-embedding-3-small": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding" }, "github_copilot/text-embedding-3-small-inference": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding" }, "github_copilot/text-embedding-ada-002": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding" }, "chatgpt/gpt-6-sol": { @@ -33853,7 +32660,6 @@ ], "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_max_reasoning_effort": true, @@ -33874,7 +32680,6 @@ ], "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_max_reasoning_effort": true, @@ -33895,7 +32700,6 @@ ], "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_max_reasoning_effort": true, @@ -33916,7 +32720,6 @@ ], "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_max_reasoning_effort": true, @@ -33933,7 +32736,6 @@ "source": "https://platform.openai.com/docs/models/gpt-5.5", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -33953,7 +32755,6 @@ "source": "https://platform.openai.com/docs/models/gpt-5.6-luna", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -33973,7 +32774,6 @@ "source": "https://platform.openai.com/docs/models/gpt-5.6-sol", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -33993,7 +32793,6 @@ "source": "https://platform.openai.com/docs/models/gpt-5.6-terra", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34012,7 +32811,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34032,7 +32830,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34050,7 +32847,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34068,7 +32864,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34082,7 +32877,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34097,7 +32891,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34113,7 +32906,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34128,7 +32920,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34148,7 +32939,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34163,7 +32953,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34179,7 +32968,6 @@ "litellm_provider": "gigachat", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true, @@ -34190,7 +32978,6 @@ "litellm_provider": "gigachat", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true, @@ -34202,7 +32989,6 @@ "litellm_provider": "gigachat", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true, @@ -34213,7 +32999,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024 @@ -34222,7 +33007,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024 @@ -34231,7 +33015,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 4096, - "max_tokens": 4096, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2560 @@ -34240,7 +33023,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 4096, - "max_tokens": 4096, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048 @@ -34250,7 +33032,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_function_calling": true, @@ -34262,7 +33043,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -34273,7 +33053,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -34284,7 +33063,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -34295,7 +33073,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.4e-05, "supports_function_calling": true @@ -34305,7 +33082,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true @@ -34315,7 +33091,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true @@ -34325,7 +33100,6 @@ "litellm_provider": "gmi", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -34336,7 +33110,6 @@ "litellm_provider": "gmi", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -34347,7 +33120,6 @@ "litellm_provider": "gmi", "max_input_tokens": 163840, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-07, "supports_function_calling": true @@ -34357,7 +33129,6 @@ "litellm_provider": "gmi", "max_input_tokens": 163840, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 8.8e-07, "supports_function_calling": true @@ -34367,7 +33138,6 @@ "litellm_provider": "gmi", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -34379,7 +33149,6 @@ "litellm_provider": "gmi", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06 }, @@ -34388,7 +33157,6 @@ "litellm_provider": "gmi", "max_input_tokens": 196608, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06 }, @@ -34420,7 +33188,6 @@ "litellm_provider": "baseten", "max_input_tokens": 200000, "max_output_tokens": 200000, - "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -34465,7 +33232,6 @@ "litellm_provider": "baseten", "max_input_tokens": 128072, "max_output_tokens": 128072, - "max_tokens": 128072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inference.baseten.co/v1/models", @@ -34495,7 +33261,6 @@ "litellm_provider": "gmi", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-06, "supports_vision": true @@ -34505,7 +33270,6 @@ "litellm_provider": "gmi", "max_input_tokens": 202752, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2e-06 }, @@ -34518,7 +33282,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.9e-07, "output_cost_per_token_batches": 1.5e-07, @@ -34540,7 +33303,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 3.8e-07, "output_cost_per_token_batches": 1.9e-07, @@ -34562,7 +33324,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 8e-08, "output_cost_per_token_batches": 4e-08, @@ -34594,7 +33355,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -34634,7 +33394,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -34668,7 +33427,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -34700,7 +33458,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, @@ -34721,7 +33478,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://developers.openai.com/api/docs/pricing", @@ -34737,7 +33493,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, @@ -34755,7 +33510,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 2e-06, @@ -34772,7 +33526,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-06, "supports_prompt_caching": true, @@ -34785,7 +33538,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 2e-06, "source": "https://developers.openai.com/api/docs/pricing" @@ -34795,7 +33547,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 8192, "max_output_tokens": 4097, - "max_tokens": 4097, "mode": "completion", "output_cost_per_token": 2e-06 }, @@ -34805,7 +33556,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "supports_function_calling": true, @@ -34820,7 +33570,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_batches": 3e-05, @@ -34836,7 +33585,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_function_calling": true, @@ -34851,7 +33599,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_function_calling": true, @@ -34869,7 +33616,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_batches": 1.5e-05, @@ -34891,7 +33637,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -34935,7 +33680,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_priority": 1.4e-05, @@ -34978,7 +33722,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, @@ -35022,7 +33765,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_priority": 2.8e-06, @@ -35066,7 +33808,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -35104,7 +33845,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_priority": 8e-07, @@ -35141,7 +33881,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -35164,7 +33903,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -35188,7 +33926,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 1.7e-05, @@ -35213,7 +33950,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 1.7e-05, @@ -35235,7 +33971,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -35253,7 +33988,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -35271,7 +34005,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -35308,7 +34041,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -35343,7 +34075,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -35381,7 +34112,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -35419,7 +34149,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -35456,7 +34185,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -35497,7 +34225,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -35527,7 +34254,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 1e-06, @@ -35554,7 +34280,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 6e-07, @@ -35574,7 +34299,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -35592,7 +34316,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -35619,7 +34342,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -35700,7 +34422,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -35727,7 +34448,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -36281,7 +35001,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -36327,7 +35046,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5-codex", @@ -36362,7 +35080,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, @@ -36412,7 +35129,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex-mini", @@ -36443,7 +35159,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex-max", @@ -36475,7 +35190,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex", @@ -36507,7 +35221,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-chat-latest", @@ -36542,7 +35255,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, @@ -36594,7 +35306,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, @@ -36645,7 +35356,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.2-codex", @@ -36677,7 +35387,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.2-chat-latest", @@ -36712,7 +35421,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, @@ -36762,7 +35470,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, "output_cost_per_token_batches": 8.4e-05, @@ -36804,7 +35511,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, "output_cost_per_token_batches": 8.4e-05, @@ -36884,7 +35590,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -36971,7 +35676,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -37062,7 +35766,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_decisions": 0.0, @@ -37148,7 +35851,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -37230,7 +35932,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -37314,7 +36015,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -37397,7 +36097,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, @@ -37455,7 +36154,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-05, "output_cost_per_token_above_272k_tokens": 0.0001125, @@ -37494,7 +36192,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-05, "output_cost_per_token_above_272k_tokens": 0.0001125, @@ -37533,7 +36230,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 7.5e-05, "output_cost_per_token_above_272k_tokens": 0.0001125, @@ -37571,7 +36267,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -37610,7 +36305,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -37644,7 +36338,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "reasoning_effort_levels": [ @@ -37690,7 +36383,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -37753,7 +36445,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -37809,7 +36500,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -37861,7 +36551,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -37920,7 +36609,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -37978,7 +36666,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -38029,7 +36716,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -38082,7 +36768,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -38138,7 +36823,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "output_cost_per_token_flex": 2.25e-06, @@ -38193,7 +36877,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "output_cost_per_token_flex": 2.25e-06, @@ -38246,7 +36929,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, @@ -38298,7 +36980,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, @@ -38345,7 +37026,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 272000, - "max_tokens": 272000, "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, @@ -38388,7 +37068,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 272000, - "max_tokens": 272000, "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, @@ -38437,7 +37116,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -38483,7 +37161,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supported_endpoints": [ @@ -38519,7 +37196,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5-chat-latest", @@ -38552,7 +37228,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, @@ -38596,7 +37271,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.3-chat-latest", @@ -38633,7 +37307,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -38685,7 +37358,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -38735,7 +37407,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -38785,7 +37456,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -38871,7 +37541,6 @@ "litellm_provider": "openai", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -38906,7 +37575,6 @@ "litellm_provider": "openai", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -38941,7 +37609,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, @@ -38976,7 +37643,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, @@ -39013,7 +37679,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -39051,7 +37716,6 @@ "litellm_provider": "openai", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -39087,7 +37751,6 @@ "litellm_provider": "openai", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -39113,7 +37776,6 @@ }, "gradient_ai/alibaba-qwen3-32b": { "litellm_provider": "gradient_ai", - "max_tokens": 40960, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" @@ -39128,7 +37790,6 @@ "gradient_ai/anthropic-claude-3-opus": { "input_cost_per_token": 1.5e-05, "litellm_provider": "gradient_ai", - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 7.5e-05, "supported_endpoints": [ @@ -39144,7 +37805,6 @@ "gradient_ai/anthropic-claude-3.5-haiku": { "input_cost_per_token": 8e-07, "litellm_provider": "gradient_ai", - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 4e-06, "supported_endpoints": [ @@ -39160,7 +37820,6 @@ "gradient_ai/anthropic-claude-3.5-sonnet": { "input_cost_per_token": 3e-06, "litellm_provider": "gradient_ai", - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.5e-05, "supported_endpoints": [ @@ -39176,7 +37835,6 @@ "gradient_ai/anthropic-claude-3.7-sonnet": { "input_cost_per_token": 3e-06, "litellm_provider": "gradient_ai", - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.5e-05, "supported_endpoints": [ @@ -39192,7 +37850,6 @@ "gradient_ai/deepseek-r1-distill-llama-70b": { "input_cost_per_token": 9.9e-07, "litellm_provider": "gradient_ai", - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 9.9e-07, "supported_endpoints": [ @@ -39208,7 +37865,6 @@ "gradient_ai/llama3-8b-instruct": { "input_cost_per_token": 2e-07, "litellm_provider": "gradient_ai", - "max_tokens": 512, "mode": "chat", "output_cost_per_token": 2e-07, "supported_endpoints": [ @@ -39224,7 +37880,6 @@ "gradient_ai/llama3.3-70b-instruct": { "input_cost_per_token": 6.5e-07, "litellm_provider": "gradient_ai", - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 6.5e-07, "supported_endpoints": [ @@ -39240,7 +37895,6 @@ "gradient_ai/mistral-nemo-instruct-2407": { "input_cost_per_token": 3e-07, "litellm_provider": "gradient_ai", - "max_tokens": 512, "mode": "chat", "output_cost_per_token": 3e-07, "supported_endpoints": [ @@ -39255,7 +37909,6 @@ }, "gradient_ai/openai-gpt-4o": { "litellm_provider": "gradient_ai", - "max_tokens": 16384, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" @@ -39269,7 +37922,6 @@ }, "gradient_ai/openai-gpt-4o-mini": { "litellm_provider": "gradient_ai", - "max_tokens": 16384, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" @@ -39284,7 +37936,6 @@ "gradient_ai/openai-o3": { "input_cost_per_token": 2e-06, "litellm_provider": "gradient_ai", - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "supported_endpoints": [ @@ -39300,7 +37951,6 @@ "gradient_ai/openai-o3-mini": { "input_cost_per_token": 1.1e-06, "litellm_provider": "gradient_ai", - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supported_endpoints": [ @@ -39316,7 +37966,6 @@ "lemonade/Qwen3-Coder-30B-A3B-Instruct-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 32768, "max_input_tokens": 262144, "max_output_tokens": 32768, "mode": "chat", @@ -39328,7 +37977,6 @@ "lemonade/gpt-oss-20b-mxfp4-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 32768, "max_input_tokens": 131072, "max_output_tokens": 32768, "mode": "chat", @@ -39340,7 +37988,6 @@ "lemonade/gpt-oss-120b-mxfp-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 32768, "max_input_tokens": 131072, "max_output_tokens": 32768, "mode": "chat", @@ -39352,7 +37999,6 @@ "lemonade/Gemma-3-4b-it-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 8192, "max_input_tokens": 128000, "max_output_tokens": 8192, "mode": "chat", @@ -39364,7 +38010,6 @@ "lemonade/Qwen3-4B-Instruct-2507-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 32768, "max_input_tokens": 262144, "max_output_tokens": 32768, "mode": "chat", @@ -39378,7 +38023,6 @@ "litellm_provider": "amazon_nova", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.4e-07, "supports_function_calling": true, @@ -39390,7 +38034,6 @@ "litellm_provider": "amazon_nova", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.4e-07, "supports_function_calling": true, @@ -39404,7 +38047,6 @@ "litellm_provider": "amazon_nova", "max_input_tokens": 1000000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.25e-05, "supports_function_calling": true, @@ -39418,7 +38060,6 @@ "litellm_provider": "amazon_nova", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.2e-06, "supports_function_calling": true, @@ -39432,7 +38073,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://console.groq.com/docs/model/llama-guard-3-8b" @@ -39442,7 +38083,6 @@ "litellm_provider": "groq", "max_input_tokens": 512, "max_output_tokens": 512, - "max_tokens": 512, "mode": "chat", "output_cost_per_token": 3e-08, "source": "https://console.groq.com/docs/models" @@ -39452,7 +38092,6 @@ "litellm_provider": "groq", "max_input_tokens": 512, "max_output_tokens": 512, - "max_tokens": 512, "mode": "chat", "output_cost_per_token": 4e-08, "source": "https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m" @@ -39463,7 +38102,6 @@ "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 6e-07, "search_context_cost_per_query": { @@ -39484,7 +38122,6 @@ "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, "search_context_cost_per_query": { @@ -39505,7 +38142,6 @@ "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, "search_context_cost_per_query": { @@ -39524,7 +38160,6 @@ "litellm_provider": "groq", "max_input_tokens": 4000, "max_output_tokens": 50000, - "max_tokens": 50000, "mode": "audio_speech", "source": "https://console.groq.com/docs/model/canopylabs/orpheus-v1-english" }, @@ -39533,7 +38168,6 @@ "litellm_provider": "groq", "max_input_tokens": 4000, "max_output_tokens": 50000, - "max_tokens": 50000, "mode": "audio_speech", "source": "https://console.groq.com/docs/models" }, @@ -39551,7 +38185,6 @@ }, "heroku/claude-3-5-haiku": { "litellm_provider": "heroku", - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -39561,7 +38194,6 @@ }, "heroku/claude-3-5-sonnet-latest": { "litellm_provider": "heroku", - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -39571,7 +38203,6 @@ }, "heroku/claude-3-7-sonnet": { "litellm_provider": "heroku", - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -39581,7 +38212,6 @@ }, "heroku/claude-4-sonnet": { "litellm_provider": "heroku", - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -39627,7 +38257,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39640,7 +38269,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": true, @@ -39653,7 +38281,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39666,7 +38293,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39679,7 +38305,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-06, "supports_function_calling": true, @@ -39692,7 +38317,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "supports_function_calling": true, @@ -39705,7 +38329,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_function_calling": true, @@ -39718,7 +38341,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": true, @@ -39731,7 +38353,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "supports_function_calling": true, @@ -39744,7 +38365,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39757,7 +38377,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39770,7 +38389,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39783,7 +38401,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39796,7 +38413,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39809,7 +38425,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39822,7 +38437,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-06, "supports_function_calling": true, @@ -39835,7 +38449,6 @@ "litellm_provider": "ai21", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 3e-06 }, @@ -39844,7 +38457,6 @@ "litellm_provider": "ai21", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 1e-05 }, @@ -39853,7 +38465,6 @@ "litellm_provider": "ai21", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 1.5e-05 }, @@ -39862,7 +38473,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39872,7 +38482,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -39882,7 +38491,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -39892,7 +38500,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39902,7 +38509,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39912,7 +38518,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -39922,7 +38527,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -39932,7 +38536,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39942,7 +38545,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39952,7 +38554,6 @@ "litellm_provider": "jina_ai", "max_input_tokens": 1024, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://api.jina.ai/v1/models" @@ -39972,7 +38573,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -40009,7 +38609,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -40037,7 +38636,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 7e-06, "supports_function_calling": false, @@ -40050,7 +38648,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -40063,7 +38660,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -40077,7 +38673,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": true, @@ -40090,7 +38685,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.5e-06, "supports_function_calling": false, @@ -40103,7 +38697,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -40116,7 +38709,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -40130,7 +38722,6 @@ "litellm_provider": "inception", "max_input_tokens": 128000, "max_output_tokens": 50000, - "max_tokens": 50000, "mode": "chat", "output_cost_per_token": 7.5e-07, "supports_function_calling": true, @@ -40145,7 +38736,6 @@ "litellm_provider": "inception", "max_input_tokens": 260000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.5e-07, "source": "https://docs.inceptionlabs.ai/get-started/models", @@ -40160,7 +38750,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40181,7 +38771,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40202,7 +38792,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40223,7 +38813,7 @@ "input_cost_per_token": 1.72e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40244,7 +38834,7 @@ "input_cost_per_token": 1.72e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40265,7 +38855,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://inferenceapis.com/pricing", @@ -40286,7 +38876,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40307,7 +38897,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://inferenceapis.com/pricing", @@ -40328,7 +38918,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40349,7 +38939,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40370,7 +38960,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40391,7 +38981,7 @@ "input_cost_per_token": 1.72e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40412,7 +39002,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://inferenceapis.com/pricing", @@ -40433,7 +39023,7 @@ "input_cost_per_token": 1.82e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048575, - "max_tokens": 1048575, + "max_output_tokens": 1048575, "mode": "chat", "output_cost_per_token": 5.72e-06, "source": "https://inferenceapis.com/pricing", @@ -40454,7 +39044,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048575, - "max_tokens": 1048575, + "max_output_tokens": 1048575, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://inferenceapis.com/pricing", @@ -40474,7 +39064,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://inferenceapis.com/pricing", @@ -40493,7 +39083,7 @@ "input_cost_per_token": 4e-08, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.9e-07, "source": "https://inferenceapis.com/pricing", @@ -40513,7 +39103,7 @@ "input_cost_per_token": 8.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.42e-06, "source": "https://inferenceapis.com/pricing", @@ -40533,7 +39123,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40552,7 +39142,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40571,7 +39161,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40590,7 +39180,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40609,7 +39199,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40628,7 +39218,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40648,7 +39238,7 @@ "input_cost_per_token": 3.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 524288, - "max_tokens": 524288, + "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://inferenceapis.com/pricing", @@ -40669,7 +39259,7 @@ "input_cost_per_token": 3.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 524288, - "max_tokens": 524288, + "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://inferenceapis.com/pricing", @@ -40690,7 +39280,7 @@ "input_cost_per_token": 8.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.42e-06, "source": "https://inferenceapis.com/pricing", @@ -40710,7 +39300,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://inferenceapis.com/pricing", @@ -40729,7 +39319,7 @@ "input_cost_per_token": 4e-08, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.9e-07, "source": "https://inferenceapis.com/pricing", @@ -40748,7 +39338,7 @@ "input_cost_per_token": 2.6e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40768,7 +39358,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40788,7 +39378,7 @@ "input_cost_per_token": 2.6e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40807,7 +39397,7 @@ "input_cost_per_token": 2.6e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40827,7 +39417,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40848,7 +39438,7 @@ "input_cost_per_token": 1.82e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048575, - "max_tokens": 1048575, + "max_output_tokens": 1048575, "mode": "chat", "output_cost_per_token": 5.72e-06, "source": "https://inferenceapis.com/pricing", @@ -40869,7 +39459,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048575, - "max_tokens": 1048575, + "max_output_tokens": 1048575, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://inferenceapis.com/pricing", @@ -40891,7 +39481,6 @@ "litellm_provider": "text-completion-inception", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 7.5e-07 }, @@ -40900,7 +39489,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -40914,7 +39502,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -40928,7 +39515,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -40942,7 +39528,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -40955,7 +39540,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -40968,7 +39552,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -40981,7 +39564,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, @@ -40994,7 +39576,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": true, @@ -41007,7 +39588,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, @@ -41020,7 +39600,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -41033,7 +39612,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 16384, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -41046,7 +39624,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -41059,7 +39636,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -41072,7 +39648,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, @@ -41085,7 +39660,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -41098,7 +39672,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-08, "supports_function_calling": true, @@ -41112,7 +39685,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-08, "supports_function_calling": true, @@ -41125,7 +39697,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -41138,7 +39709,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -41151,7 +39721,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -41196,14 +39765,12 @@ "max-x-max/50-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.036 }, "max-x-max/max-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.072 }, @@ -41299,7 +39866,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_character": 1.5e-05, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", @@ -41310,7 +39876,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_character": 1e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", @@ -41321,7 +39886,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -41331,7 +39895,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.56e-06, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -41342,7 +39905,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -41356,7 +39918,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -41371,7 +39932,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.2e-07, "output_cost_per_token_batches": 1.1e-07, @@ -41385,7 +39945,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.6e-07, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -41399,7 +39958,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", @@ -41412,7 +39970,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", @@ -41425,7 +39982,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -41440,7 +39996,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -41454,7 +40009,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -41465,7 +40019,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -41477,7 +40030,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9.7e-07, "output_cost_per_token_batches": 4.85e-07, @@ -41500,7 +40052,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 10000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_batches": 3.3e-07, @@ -41522,7 +40073,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.25e-06, "search_context_cost_per_query": { @@ -41563,7 +40113,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.25e-06, "search_context_cost_per_query": { @@ -41604,7 +40153,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "search_context_cost_per_query": { @@ -41645,7 +40193,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.25e-06, "search_context_cost_per_query": { @@ -41686,7 +40233,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "search_context_cost_per_query": { @@ -41743,7 +40289,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-05, "search_context_cost_per_query": { @@ -41784,7 +40329,6 @@ "litellm_provider": "meta_llama", "max_input_tokens": 128000, "max_output_tokens": 4028, - "max_tokens": 4028, "mode": "chat", "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ @@ -41800,7 +40344,6 @@ "litellm_provider": "meta_llama", "max_input_tokens": 128000, "max_output_tokens": 4028, - "max_tokens": 4028, "mode": "chat", "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ @@ -41816,7 +40359,6 @@ "litellm_provider": "meta_llama", "max_input_tokens": 1000000, "max_output_tokens": 4028, - "max_tokens": 4028, "mode": "chat", "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ @@ -41833,7 +40375,6 @@ "litellm_provider": "meta_llama", "max_input_tokens": 10000000, "max_output_tokens": 4028, - "max_tokens": 4028, "mode": "chat", "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ @@ -41855,7 +40396,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_batches": 6e-07, @@ -41878,7 +40418,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 196000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_batches": 6e-07, @@ -41902,7 +40441,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 196000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 6e-07, "output_cost_per_token_priority": 2.1e-06, @@ -42100,7 +40638,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -42122,7 +40659,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 40000, - "max_tokens": 40000, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, @@ -42144,7 +40680,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_batches": 1e-07, @@ -42166,7 +40701,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-07, "output_cost_per_token_batches": 5e-08, @@ -42188,7 +40722,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.5e-07, "output_cost_per_token_batches": 7e-08, @@ -42208,7 +40741,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", @@ -42219,7 +40751,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", @@ -42230,7 +40761,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -42245,7 +40775,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, @@ -42265,7 +40794,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, @@ -42278,7 +40806,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", @@ -42293,7 +40820,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-08, "output_cost_per_token_batches": 2e-08, @@ -42313,7 +40839,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-07, "output_cost_per_token_batches": 1.5e-07, @@ -42330,7 +40855,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", @@ -42345,7 +40869,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", @@ -42360,7 +40883,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.5e-07, "source": "https://mistral.ai/technology/", @@ -42373,7 +40895,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://docs.mistral.ai/models/devstral-small-2-25-12", @@ -42388,7 +40909,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://mistral.ai/news/devstral-2-vibe-cli", @@ -42403,7 +40923,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://mistral.ai/news/devstral-2-vibe-cli", @@ -42417,7 +40936,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.mistral.ai/models/ministral-3-14b-25-12", @@ -42433,7 +40951,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.mistral.ai/models/ministral-3-14b-25-12", @@ -42449,7 +40966,6 @@ "litellm_provider": "mistral", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://docs.mistral.ai/models/ministral-3-3b-25-12", @@ -42465,7 +40981,6 @@ "litellm_provider": "mistral", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://docs.mistral.ai/models/ministral-3-3b-25-12", @@ -42480,7 +40995,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "source": "https://docs.mistral.ai/models/mistral-embed-23-12" }, @@ -42490,7 +41004,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -42671,7 +41184,6 @@ "litellm_provider": "mistral", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://docs.mistral.ai/models/voxtral-small-25-07", @@ -42686,7 +41198,6 @@ "litellm_provider": "mistral", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://docs.mistral.ai/models/voxtral-small-25-07", @@ -42702,7 +41213,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42728,7 +41238,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42750,7 +41259,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "reasoning_effort_levels": [ @@ -42773,7 +41281,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42795,7 +41302,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42817,7 +41323,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42896,7 +41401,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", @@ -42913,7 +41417,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03", @@ -42928,7 +41431,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding" }, "mistral/codestral-embed": { @@ -42936,7 +41438,6 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding" }, "mistral/codestral-embed-2505": { @@ -42944,7 +41445,6 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding" }, "mistral/mistral-large-4": { @@ -42952,7 +41452,7 @@ "input_cost_per_token": 6.8e-07, "litellm_provider": "mistral", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 2.09e-06, "reasoning_effort_levels": [ @@ -42973,7 +41473,7 @@ "input_cost_per_token": 6.8e-07, "litellm_provider": "mistral", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 2.09e-06, "reasoning_effort_levels": [ @@ -42995,7 +41495,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://docs.mistral.ai/models/mistral-large-3-25-12", @@ -43011,7 +41510,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://docs.mistral.ai/models/mistral-large-3-25-12", @@ -43027,7 +41525,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://docs.mistral.ai/models/mistral-large-3-25-12", @@ -43043,7 +41540,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -43064,7 +41560,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -43085,7 +41580,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -43106,7 +41600,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -43127,7 +41620,6 @@ "litellm_provider": "mistral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-07, "supports_assistant_prefill": true, @@ -43141,7 +41633,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6e-07, "reasoning_effort_levels": [ @@ -43162,7 +41653,6 @@ "litellm_provider": "mistral", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://mistral.ai/pricing", @@ -43178,7 +41668,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://mistral.ai/pricing", @@ -43194,7 +41683,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://mistral.ai/pricing", @@ -43210,7 +41698,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://mistral.ai/pricing", @@ -43226,7 +41713,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://mistral.ai/pricing", @@ -43242,7 +41728,6 @@ "litellm_provider": "mistral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_assistant_prefill": true, @@ -43255,7 +41740,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://mistral.ai/technology/", @@ -43269,7 +41753,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-06, "supports_assistant_prefill": true, @@ -43286,7 +41769,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, @@ -43308,7 +41790,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, @@ -43328,7 +41809,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://platform.kimi.ai/docs/pricing/chat-k27-code", @@ -43346,7 +41826,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", @@ -43363,7 +41842,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://platform.kimi.ai/docs/pricing/chat-k26", @@ -43380,7 +41858,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ @@ -43401,7 +41878,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43414,7 +41890,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43428,7 +41903,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43441,7 +41915,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43455,7 +41928,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43468,7 +41940,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43482,7 +41953,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43495,7 +41965,6 @@ "litellm_provider": "morph", "max_input_tokens": 16000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": false, @@ -43509,7 +41978,6 @@ "litellm_provider": "morph", "max_input_tokens": 16000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 1.9e-06, "supports_function_calling": false, @@ -43527,7 +41995,6 @@ "input_cost_per_video_per_second_above_8s_interval": 0.001, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -43551,7 +42018,6 @@ "input_cost_per_video_per_second_above_8s_interval": 0.001, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -43711,7 +42177,6 @@ ] }, "nebius/deepseek-ai/DeepSeek-R1": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 8e-07, @@ -43723,7 +42188,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-R1-0528": { - "max_tokens": 164000, "max_input_tokens": 164000, "max_output_tokens": 164000, "input_cost_per_token": 8e-07, @@ -43735,7 +42199,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2.5e-07, @@ -43746,7 +42209,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V3": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 5e-07, @@ -43757,7 +42219,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V3-0324": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 5e-07, @@ -43768,7 +42229,6 @@ "source": "https://nebius.com/prices" }, "nebius/google/gemma-3-27b-it": { - "max_tokens": 110000, "max_input_tokens": 110000, "max_output_tokens": 110000, "input_cost_per_token": 1e-07, @@ -43780,7 +42240,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/google%2Fgemma-3-27b-it" }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.3e-07, @@ -43791,7 +42250,6 @@ "source": "https://nebius.com/prices" }, "nebius/meta-llama/Llama-Guard-3-8B": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-08, @@ -43801,7 +42259,6 @@ "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-8B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-08, @@ -43812,7 +42269,6 @@ "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-70B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.3e-07, @@ -43823,7 +42279,6 @@ "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-405B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1e-06, @@ -43834,7 +42289,6 @@ "source": "https://nebius.com/prices" }, "nebius/mistralai/Mistral-Nemo-Instruct-2407": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 4e-08, @@ -43845,7 +42299,6 @@ "source": "https://nebius.com/prices" }, "nebius/NousResearch/Hermes-3-Llama-3.1-405B": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1e-06, @@ -43856,7 +42309,6 @@ "source": "https://nebius.com/prices" }, "nebius/nvidia/Llama-3.1-Nemotron-Ultra-253B-v1": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 6e-07, @@ -43867,7 +42319,6 @@ "source": "https://nebius.com/prices" }, "nebius/nvidia/Llama-3.3-Nemotron-Super-49B-v1": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -43878,7 +42329,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-235B-A22B": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2e-07, @@ -43889,7 +42339,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-32B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1e-07, @@ -43900,7 +42349,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-32B" }, "nebius/Qwen/Qwen3-30B-A3B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -43911,7 +42359,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-14B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 8e-08, @@ -43922,7 +42369,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-4B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 8e-08, @@ -43933,7 +42379,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/QwQ-32B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1.5e-07, @@ -43945,7 +42390,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-72B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.3e-07, @@ -43956,7 +42400,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-32B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 6e-08, @@ -43967,7 +42410,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-Coder-7B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-08, @@ -43978,7 +42420,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-VL-72B-Instruct": { - "max_tokens": 32000, "max_input_tokens": 32000, "max_output_tokens": 32000, "input_cost_per_token": 2.5e-07, @@ -43990,7 +42431,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/Qwen%2FQwen2.5-VL-72B-Instruct" }, "nebius/Qwen/Qwen2-VL-72B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.3e-07, @@ -44002,7 +42442,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2-VL-7B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-08, @@ -44013,7 +42452,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V4-Flash": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1.4e-07, @@ -44025,7 +42463,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash" }, "nebius/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_tokens": 1024000, "max_input_tokens": 1024000, "max_output_tokens": 1024000, "input_cost_per_token": 1.4e-07, @@ -44037,7 +42474,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash-0731" }, "nebius/deepseek-ai/DeepSeek-V4-Pro": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1.75e-06, @@ -44063,7 +42499,6 @@ "litellm_provider": "nebius", "max_input_tokens": 1048000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://tokenfactory.nebius.com/endpoints?modals=endpoint-details&model-id=deepseek-ai/DeepSeek-V4.1-Flash", @@ -44072,7 +42507,6 @@ "supports_vision": true }, "nebius/MiniMaxAI/MiniMax-M2.5": { - "max_tokens": 196608, "max_input_tokens": 196608, "max_output_tokens": 196608, "input_cost_per_token": 3e-07, @@ -44084,7 +42518,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M2.5" }, "nebius/MiniMaxAI/MiniMax-M3": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 3e-07, @@ -44097,7 +42530,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M3" }, "nebius/moonshotai/Kimi-K2.6": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9.5e-07, @@ -44110,7 +42542,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K2.6" }, "nebius/moonshotai/Kimi-K2.7-Code": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9.5e-07, @@ -44123,7 +42554,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/moonshotai%2FKimi-K2.7-Code" }, "nebius/moonshotai/Kimi-K3": { - "max_tokens": 1024000, "max_input_tokens": 1024000, "max_output_tokens": 1024000, "input_cost_per_token": 3e-06, @@ -44136,7 +42566,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K3" }, "nebius/NousResearch/Hermes-4-405B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-06, @@ -44148,7 +42577,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-405B" }, "nebius/NousResearch/Hermes-4-70B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.3e-07, @@ -44160,7 +42588,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-70B" }, "nebius/nvidia/Cosmos3-Super-Reasoner": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1e-07, @@ -44173,7 +42600,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/nvidia%2FCosmos3-Super-Reasoner" }, "nebius/nvidia/Llama-3_1-Nemotron-Ultra-253B-v1": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6e-07, @@ -44184,7 +42610,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FLlama-3_1-Nemotron-Ultra-253B-v1" }, "nebius/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 6e-08, @@ -44196,7 +42621,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNVIDIA-Nemotron-3-Nano-30B-A3B" }, "nebius/nvidia/Nemotron-3-Nano-Omni": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 6e-08, @@ -44209,7 +42633,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Nano-Omni" }, "nebius/nvidia/nemotron-3-super-120b-a12b": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 3e-07, @@ -44221,7 +42644,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2Fnemotron-3-super-120b-a12b" }, "nebius/nvidia/Nemotron-3-Ultra-550b-a55b": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1e-06, @@ -44233,7 +42655,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Ultra-550b-a55b" }, "nebius/nvidia/Nemotron-3_5-Lightning": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 6e-08, @@ -44245,7 +42666,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3_5-Lightning" }, "nebius/openai/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.5e-07, @@ -44257,7 +42677,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/openai%2Fgpt-oss-120b" }, "nebius/openbmb/MiniCPM-V-4_5": { - "max_tokens": 32000, "max_input_tokens": 32000, "max_output_tokens": 32000, "input_cost_per_token": 6.58e-07, @@ -44268,7 +42687,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/openbmb%2FMiniCPM-V-4_5" }, "nebius/Qwen/Qwen3-235B-A22B-Instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2e-07, @@ -44279,7 +42697,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-235B-A22B-Instruct-2507" }, "nebius/Qwen/Qwen3-30B-A3B-Instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1e-07, @@ -44290,7 +42707,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-30B-A3B-Instruct-2507" }, "nebius/Qwen/Qwen3-Next-80B-A3B-Thinking": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.5e-07, @@ -44302,7 +42718,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-Next-80B-A3B-Thinking" }, "nebius/Qwen/Qwen3.5-397B-A17B": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 6e-07, @@ -44324,7 +42739,6 @@ "supports_reasoning": true }, "nebius/zai-org/GLM-5.1": { - "max_tokens": 202752, "max_input_tokens": 202752, "max_output_tokens": 202752, "input_cost_per_token": 1.4e-06, @@ -44336,7 +42750,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.1" }, "nebius/zai-org/GLM-5.2": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1.4e-06, @@ -44351,7 +42764,7 @@ "input_cost_per_token": 1.4e-06, "litellm_provider": "nebius", "max_input_tokens": 1024000, - "max_tokens": 1024000, + "max_output_tokens": 1024000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3", @@ -44359,7 +42772,6 @@ "supports_reasoning": true }, "nebius/zai-org/GLM-5.3-Flash": { - "max_tokens": 1024000, "max_input_tokens": 1024000, "max_output_tokens": 1024000, "input_cost_per_token": 1.5e-07, @@ -44372,7 +42784,6 @@ "supports_vision": true }, "nebius/BAAI/bge-en-icl": { - "max_tokens": 32768, "max_input_tokens": 32768, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -44381,7 +42792,6 @@ "source": "https://nebius.com/prices" }, "nebius/BAAI/bge-multilingual-gemma2": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -44390,7 +42800,6 @@ "source": "https://nebius.com/prices" }, "nebius/intfloat/e5-mistral-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -44399,7 +42808,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-Embedding-8B": { - "max_tokens": 40960, "max_input_tokens": 40960, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -44416,7 +42824,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -44438,7 +42845,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.3e-07, "output_cost_per_token_batches": 1.2e-07, @@ -44459,7 +42865,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.4e-07, "output_cost_per_token_batches": 1.2e-07, @@ -44482,7 +42887,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 6.5e-07, "output_cost_per_token_batches": 3.25e-07, @@ -44505,7 +42909,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_batches": 3e-05, @@ -44528,7 +42931,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_batches": 3e-05, @@ -44550,7 +42952,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 0.0006, "output_cost_per_token_batches": 0.0003, @@ -44584,7 +42985,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 0.0006, "output_cost_per_token_batches": 0.0003, @@ -44622,7 +43022,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -44663,7 +43062,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 4e-05, "source": "https://developers.openai.com/api/docs/models/o3-deep-research", @@ -44696,7 +43094,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 4e-05, "source": "https://developers.openai.com/api/docs/models/o3-deep-research", @@ -44733,7 +43130,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -44776,7 +43172,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -44797,7 +43192,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -44816,7 +43210,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, @@ -44854,7 +43247,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, @@ -44897,7 +43289,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -44925,7 +43316,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-06, "source": "https://developers.openai.com/api/docs/models/o4-mini-deep-research", @@ -44958,7 +43348,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-06, "source": "https://developers.openai.com/api/docs/models/o4-mini-deep-research", @@ -44995,7 +43384,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -45022,7 +43410,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45035,7 +43422,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45048,7 +43434,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.068e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45061,7 +43446,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45075,7 +43459,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45088,7 +43471,6 @@ "litellm_provider": "oci", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45102,7 +43484,6 @@ "litellm_provider": "oci", "max_input_tokens": 10485760, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45115,7 +43496,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45128,7 +43508,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45141,7 +43520,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45154,7 +43532,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45167,7 +43544,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45180,7 +43556,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45193,7 +43568,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45206,7 +43580,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45219,7 +43592,6 @@ "litellm_provider": "oci", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45239,7 +43611,6 @@ "litellm_provider": "oci", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45258,7 +43629,6 @@ "litellm_provider": "oci", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45278,7 +43648,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45292,7 +43661,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45314,7 +43682,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45326,7 +43693,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45339,7 +43705,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 9e-08, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45351,7 +43716,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45363,7 +43727,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45375,7 +43738,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45388,7 +43750,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45400,7 +43761,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45412,7 +43772,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45424,7 +43783,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [], @@ -45437,7 +43795,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45452,7 +43809,6 @@ "litellm_provider": "oci", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -45468,7 +43824,6 @@ "litellm_provider": "oci", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -45485,7 +43840,6 @@ "litellm_provider": "oci", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -45501,7 +43855,6 @@ "litellm_provider": "oci", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -45515,7 +43868,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45527,7 +43879,6 @@ "litellm_provider": "oci", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45542,7 +43893,6 @@ "litellm_provider": "oci", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45557,7 +43907,6 @@ "litellm_provider": "oci", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45572,7 +43921,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45585,7 +43933,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45597,7 +43944,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -45607,7 +43953,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 384, @@ -45617,7 +43962,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -45627,7 +43971,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 384, @@ -45637,7 +43980,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -45648,7 +43990,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 384, @@ -45659,7 +44000,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 384, @@ -45670,7 +44010,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "oci", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -45682,7 +44021,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": false @@ -45692,7 +44030,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45701,7 +44038,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45710,7 +44046,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45720,7 +44055,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45730,7 +44064,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45740,7 +44073,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45750,7 +44082,6 @@ "litellm_provider": "ollama", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45760,7 +44091,6 @@ "litellm_provider": "ollama", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45770,7 +44100,6 @@ "litellm_provider": "ollama", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45780,7 +44109,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45790,7 +44118,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45799,7 +44126,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45808,7 +44134,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45817,7 +44142,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45826,7 +44150,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45835,7 +44158,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45844,7 +44166,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45854,7 +44175,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45863,7 +44183,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45872,7 +44191,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45882,7 +44200,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45892,7 +44209,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45902,7 +44218,6 @@ "litellm_provider": "ollama", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45912,7 +44227,6 @@ "litellm_provider": "ollama", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45922,7 +44236,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45932,7 +44245,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45941,7 +44253,6 @@ "litellm_provider": "ollama", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45951,7 +44262,6 @@ "litellm_provider": "ollama", "max_input_tokens": 2048, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45960,7 +44270,6 @@ "litellm_provider": "openai", "max_input_tokens": 32768, "max_output_tokens": 0, - "max_tokens": 0, "mode": "moderation", "output_cost_per_token": 0.0 }, @@ -45969,7 +44278,6 @@ "litellm_provider": "openai", "max_input_tokens": 32768, "max_output_tokens": 0, - "max_tokens": 0, "mode": "moderation", "output_cost_per_token": 0.0 }, @@ -45987,7 +44295,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -46013,7 +44320,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-07, "output_cost_per_token_batches": 1.5e-07, @@ -46033,7 +44339,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -46055,7 +44360,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_batches": 1e-07, @@ -46074,7 +44378,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46088,7 +44391,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46101,7 +44403,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46114,7 +44415,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46127,7 +44427,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 127000, "max_output_tokens": 127000, - "max_tokens": 127000, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46140,7 +44439,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 118000, "max_output_tokens": 118000, - "max_tokens": 118000, "mode": "chat", "output_cost_per_token": 1.3e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46153,7 +44451,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46167,7 +44464,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 6.3e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46180,7 +44476,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 8.7e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46193,7 +44488,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 9.1e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46207,7 +44501,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.3e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46221,7 +44514,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46235,7 +44527,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46249,7 +44540,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.9e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46263,7 +44553,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.9e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46296,7 +44585,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-06 }, @@ -46305,7 +44593,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.8e-06 }, @@ -46314,7 +44601,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-06 }, @@ -46323,7 +44609,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -46332,7 +44617,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07 }, @@ -46341,7 +44625,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46350,7 +44633,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46359,7 +44641,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-06 }, @@ -46369,7 +44650,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-06 }, @@ -46378,7 +44658,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46388,7 +44667,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46444,7 +44722,7 @@ "input_cost_per_token": 1e-06, "litellm_provider": "perplexity", "max_input_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-06, "search_context_cost_per_query": { @@ -46459,7 +44737,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "perplexity", "max_input_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 128000, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 8e-06, @@ -46476,7 +44754,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.8e-06 }, @@ -46486,7 +44763,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 12000, "max_output_tokens": 12000, - "max_tokens": 12000, "mode": "chat", "output_cost_per_token": 1.8e-06 }, @@ -46495,7 +44771,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 200000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -46509,7 +44784,7 @@ "input_cost_per_token": 1e-06, "litellm_provider": "perplexity", "max_input_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-06, "search_context_cost_per_query": { @@ -46524,7 +44799,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "perplexity", "max_input_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 8e-06, "search_context_cost_per_query": { @@ -46540,7 +44815,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46550,7 +44824,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 12000, "max_output_tokens": 12000, - "max_tokens": 12000, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46559,7 +44832,6 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46571,7 +44843,6 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46583,7 +44854,6 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46595,7 +44865,6 @@ "litellm_provider": "publicai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46607,7 +44876,6 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46619,7 +44887,6 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46859,7 +45126,6 @@ "input_cost_per_token": 4e-09, "litellm_provider": "perplexity", "max_input_tokens": 32768, - "max_tokens": 32768, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -46869,7 +45135,6 @@ "input_cost_per_token": 3e-08, "litellm_provider": "perplexity", "max_input_tokens": 32768, - "max_tokens": 32768, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2560, @@ -46880,7 +45145,6 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46892,7 +45156,6 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46905,7 +45168,6 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46919,7 +45181,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.8e-06, "supports_function_calling": true, @@ -46934,7 +45195,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 8.8e-07, "output_cost_per_token_flex": 4.4e-07, @@ -46954,7 +45214,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -46973,7 +45232,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -46990,7 +45248,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -47006,7 +45263,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.45e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47022,7 +45278,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.41e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47038,7 +45293,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.236e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47054,7 +45308,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.41e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47070,7 +45323,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47086,7 +45338,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.45e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47104,7 +45355,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.66e-06, "output_cost_per_token_batches": 1.33e-06, @@ -47125,7 +45375,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_batches": 6e-07, @@ -47180,7 +45429,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, "supports_tool_choice": true @@ -47190,7 +45438,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, "supports_tool_choice": true @@ -47200,7 +45447,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.75e-06, "supports_tool_choice": true @@ -47210,7 +45456,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.75e-06, "supports_tool_choice": true @@ -47220,7 +45465,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47230,7 +45474,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47240,7 +45483,6 @@ "litellm_provider": "replicate", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.75e-06, "supports_tool_choice": true @@ -47250,7 +45492,6 @@ "litellm_provider": "replicate", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.75e-06, "supports_tool_choice": true @@ -47260,7 +45501,6 @@ "litellm_provider": "replicate", "max_input_tokens": 8086, "max_output_tokens": 8086, - "max_tokens": 8086, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47270,7 +45510,6 @@ "litellm_provider": "replicate", "max_input_tokens": 8086, "max_output_tokens": 8086, - "max_tokens": 8086, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47280,7 +45519,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47290,7 +45528,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47300,7 +45537,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06, "supports_tool_choice": true @@ -47429,7 +45665,6 @@ "mode": "chat", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_system_messages": true }, @@ -47584,7 +45819,6 @@ "mode": "chat", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true @@ -47605,7 +45839,6 @@ "mode": "chat", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_reasoning": true, "supports_system_messages": true }, @@ -47615,7 +45848,6 @@ "litellm_provider": "cohere", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -47625,7 +45857,6 @@ "litellm_provider": "cohere", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -47635,7 +45866,6 @@ "litellm_provider": "cohere", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -47645,7 +45875,6 @@ "litellm_provider": "cohere", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://cohere.com/pricing" @@ -47656,7 +45885,6 @@ "litellm_provider": "cohere", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://cohere.com/pricing" @@ -47687,7 +45915,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -47696,7 +45923,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -47705,7 +45931,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -47714,7 +45939,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -47723,7 +45947,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -47732,7 +45955,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -47741,7 +45963,6 @@ "litellm_provider": "sambanova", "max_input_tokens": 196608, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://cloud.sambanova.ai/plans/pricing", @@ -47754,7 +45975,6 @@ "litellm_provider": "sambanova", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7e-06, "source": "https://cloud.sambanova.ai/plans/pricing" @@ -47764,7 +45984,6 @@ "litellm_provider": "sambanova", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount" }, @@ -47781,7 +46000,6 @@ "litellm_provider": "sambanova", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.sambanova.ai/plans/pricing", @@ -47790,7 +46008,6 @@ "supports_tool_choice": true }, "sambanova/DeepSeek-V3.1": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-06, @@ -47803,7 +46020,6 @@ "source": "https://cloud.sambanova.ai/plans/pricing" }, "sambanova/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2.2e-07, @@ -47816,7 +46032,6 @@ "source": "https://cloud.sambanova.ai/plans/pricing" }, "sambanova/DeepSeek-V3.2": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 3e-06, @@ -47828,7 +46043,6 @@ "source": "https://cloud.sambanova.ai/plans/pricing" }, "sambanova/gemma-4-31B-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3.8e-07, @@ -47844,7 +46058,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1000000, "max_output_tokens": 375000, - "max_tokens": 375000, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://scx.ai/models/deepseek-v4-1-flash", @@ -47859,7 +46072,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.08e-06, "source": "https://scx.ai/models/glm-5-3", @@ -47875,7 +46087,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.413e-05, "source": "https://scx.ai/models/kimi-k3", @@ -47891,7 +46102,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.98e-06, "source": "https://scx.ai/pricing", @@ -47908,7 +46118,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.99e-06, "source": "https://scx.ai/pricing", @@ -47923,7 +46132,6 @@ "litellm_provider": "snowflake", "max_input_tokens": 200000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, @@ -47939,7 +46147,6 @@ "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 1.35e-06, "output_cost_per_token": 5.4e-06, @@ -47950,56 +46157,48 @@ "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/jamba-1.5-large": { "litellm_provider": "snowflake", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/jamba-1.5-mini": { "litellm_provider": "snowflake", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/jamba-instruct": { "litellm_provider": "snowflake", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama2-70b-chat": { "litellm_provider": "snowflake", "max_input_tokens": 4096, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3-70b": { "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3-8b": { "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.1-405b": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 1.2e-06, "output_cost_per_token": 1.2e-06, @@ -48010,7 +46209,6 @@ "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 7.2e-07, "output_cost_per_token": 7.2e-07, @@ -48021,7 +46219,6 @@ "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 2.4e-07, "output_cost_per_token": 2.4e-07, @@ -48031,18 +46228,15 @@ "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.2-3b": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.3-70b": { - "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 7.2e-07, @@ -48056,21 +46250,18 @@ "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/mistral-large": { "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/mistral-large2": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, @@ -48082,39 +46273,33 @@ "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/reka-core": { "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/reka-flash": { "litellm_provider": "snowflake", "max_input_tokens": 100000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/snowflake-arctic": { "litellm_provider": "snowflake", "max_input_tokens": 4096, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/snowflake-llama-3.1-405b": { "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/snowflake-llama-3.3-70b": { - "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 7.2e-07, @@ -48311,21 +46496,18 @@ "stability.sd3-5-large-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.08 }, "stability.sd3-large-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.08 }, "stability.stable-image-core-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.04 }, @@ -48501,21 +46683,18 @@ "stability.stable-image-core-v1:1": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.04 }, "stability.stable-image-ultra-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.14 }, "stability.stable-image-ultra-v1:1": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.14 }, @@ -48549,7 +46728,6 @@ "litellm_provider": "text-completion-codestral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "completion", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/capabilities/code_generation/" @@ -48559,7 +46737,6 @@ "litellm_provider": "text-completion-codestral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "completion", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/capabilities/code_generation/" @@ -48570,7 +46747,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -48582,7 +46758,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -48593,7 +46768,6 @@ "input_cost_per_token_batches": 6.5e-08, "litellm_provider": "openai", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "output_cost_per_token_batches": 0.0, @@ -48605,7 +46779,6 @@ "input_cost_per_token_batches": 1e-08, "litellm_provider": "openai", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "output_cost_per_token_batches": 0.0, @@ -48616,7 +46789,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "openai", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -48627,7 +46799,6 @@ "input_cost_per_token_batches": 5e-08, "litellm_provider": "openai", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "output_cost_per_token_batches": 0.0 @@ -48637,7 +46808,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -48647,7 +46817,6 @@ "input_cost_per_token": 6.25e-09, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 3072, - "max_tokens": 3072, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -48659,7 +46828,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -48670,7 +46838,6 @@ "litellm_provider": "vertex_ai-text-models", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "completion", "output_cost_per_token": 2.8e-05, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -48680,7 +46847,6 @@ "litellm_provider": "vertex_ai-text-models", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "completion", "output_cost_per_token": 2.8e-05, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -48706,7 +46872,8 @@ "together-ai-8.1b-21b": { "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", - "max_tokens": 1000, + "max_output_tokens": 1000, + "max_input_tokens": 1000, "mode": "chat", "output_cost_per_token": 3e-07 }, @@ -48759,7 +46926,6 @@ "litellm_provider": "together_ai", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "metadata": { "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" }, @@ -48775,7 +46941,7 @@ "input_cost_per_token": 1.04e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.04e-06, "source": "https://api.together.ai/v1/models", @@ -48802,7 +46968,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://api.together.ai/v1/models", @@ -48823,7 +46989,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 202752, - "max_tokens": 202752, + "max_output_tokens": 202752, "metadata": { "successor": "together_ai/zai-org/GLM-5.2" }, @@ -48840,7 +47006,7 @@ "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 524288, - "max_tokens": 524288, + "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", @@ -48857,7 +47023,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://www.together.ai/pricing", @@ -48867,7 +47033,7 @@ "input_cost_per_token": 0.0, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.together.ai/docs/serverless-models" @@ -48876,7 +47042,7 @@ "input_cost_per_token": 1.7e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.5e-07, "source": "https://api.together.ai/v1/models", @@ -48891,7 +47057,7 @@ "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://api.together.ai/v1/models", @@ -48902,7 +47068,7 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "together_ai", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 4.5e-06, "source": "https://www.together.ai/pricing", @@ -48912,7 +47078,7 @@ "input_cost_per_token": 3.2e-07, "litellm_provider": "together_ai", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.28e-06, "source": "https://api.together.ai/v1/models" @@ -48922,7 +47088,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 1010000, - "max_tokens": 1010000, + "max_output_tokens": 1010000, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://api.together.ai/v1/models", @@ -48932,7 +47098,7 @@ "input_cost_per_token": 1e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://api.together.ai/v1/models" @@ -48943,7 +47109,7 @@ "input_cost_per_token": 1.4e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://api.together.ai/v1/models", @@ -48958,7 +47124,7 @@ "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", @@ -48972,7 +47138,7 @@ "input_cost_per_token": 1.32e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.96e-06, "source": "https://api.together.ai/v1/models", @@ -48987,7 +47153,7 @@ "input_cost_per_token": 3.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://api.together.ai/v1/models", @@ -48998,7 +47164,7 @@ "input_cost_per_token": 1.25e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.25e-06, "source": "https://docs.together.ai/docs/serverless-models", @@ -49011,7 +47177,7 @@ "input_cost_per_token": 3e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ @@ -49033,7 +47199,7 @@ "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", "max_input_tokens": 524288, - "max_tokens": 524288, + "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 4.05e-06, "source": "https://api.together.ai/v1/models", @@ -49049,7 +47215,6 @@ "litellm_provider": "together_ai", "max_input_tokens": 1048575, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://api.together.ai/v1/models", @@ -49066,7 +47231,6 @@ "litellm_provider": "together_ai", "max_input_tokens": 1048575, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://api.together.ai/v1/models", @@ -49083,7 +47247,6 @@ "litellm_provider": "together_ai", "max_input_tokens": 1048575, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://api.together.ai/v1/models", @@ -49168,7 +47331,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.4e-07, "output_cost_per_token_batches": 1.2e-07, @@ -49187,7 +47349,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.4e-07, "output_cost_per_token_batches": 7e-08, @@ -49208,7 +47369,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.2e-06, "output_cost_per_token_batches": 1.6e-06, @@ -49229,7 +47389,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, @@ -49249,7 +47408,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -49277,7 +47435,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -49295,7 +47452,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -49314,7 +47470,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -49332,7 +47487,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -49350,7 +47504,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -49389,7 +47542,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -49431,7 +47583,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -49463,7 +47614,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, @@ -49503,7 +47653,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 1024, @@ -49542,7 +47691,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, @@ -49580,7 +47728,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.4e-05, "prompt_cache_min_tokens": 512, @@ -49618,7 +47765,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_adaptive_thinking": true, @@ -49654,7 +47800,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.88e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -49672,7 +47817,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", @@ -49688,7 +47832,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", @@ -49703,7 +47846,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -49721,7 +47863,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", @@ -49741,7 +47882,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", @@ -49768,7 +47908,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "supports_function_calling": true, "supports_prompt_caching": false, @@ -49785,7 +47924,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "supports_assistant_prefill": true, @@ -49815,7 +47953,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -49843,7 +47980,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -49880,7 +48016,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -49917,7 +48052,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -49957,7 +48091,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -49988,7 +48121,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-r1.html", @@ -50004,7 +48136,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_batches": 9.25e-07, @@ -50025,7 +48156,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -50043,7 +48173,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -50057,7 +48186,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -50072,7 +48200,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.2e-07, "output_cost_per_token_batches": 1.1e-07, @@ -50086,7 +48213,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.6e-07, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -50100,7 +48226,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", @@ -50113,7 +48238,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", @@ -50126,7 +48250,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -50141,7 +48264,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -50156,7 +48278,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9.7e-07, "output_cost_per_token_batches": 4.85e-07, @@ -50179,7 +48300,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 10000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_batches": 3.3e-07, @@ -50200,7 +48320,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-pixtral-large.md", @@ -50212,7 +48331,6 @@ "litellm_provider": "v0", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -50226,7 +48344,6 @@ "litellm_provider": "v0", "max_input_tokens": 512000, "max_output_tokens": 512000, - "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -50240,7 +48357,6 @@ "litellm_provider": "v0", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -50254,7 +48370,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.4e-07 }, @@ -50263,7 +48378,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07 }, @@ -50272,7 +48386,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-07 }, @@ -50281,7 +48394,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -50292,7 +48404,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 262144, "max_output_tokens": 66536, - "max_tokens": 66536, "mode": "chat", "output_cost_per_token": 1.6e-06, "supports_function_calling": true, @@ -50303,7 +48414,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 300000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-07, "supports_vision": true, @@ -50315,7 +48425,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.4e-07, "supports_function_calling": true, @@ -50326,7 +48435,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 300000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.2e-06, "supports_vision": true, @@ -50338,7 +48446,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -50349,7 +48456,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, "supports_vision": true, @@ -50364,7 +48470,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_vision": true, @@ -50379,7 +48484,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_vision": true, @@ -50394,7 +48498,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_vision": true, @@ -50409,7 +48512,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_vision": true, @@ -50424,7 +48526,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_vision": true, @@ -50439,7 +48540,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -50452,7 +48552,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50470,7 +48569,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50488,7 +48586,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50507,7 +48604,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -50527,7 +48623,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_assistant_prefill": true, @@ -50547,7 +48642,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_assistant_prefill": true, @@ -50567,7 +48661,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": true, @@ -50590,7 +48683,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": false, @@ -50611,7 +48703,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50631,7 +48722,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50648,7 +48738,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -50660,7 +48749,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -50671,7 +48759,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -50682,7 +48769,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -50691,7 +48777,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.19e-06, "supports_tool_choice": true @@ -50701,7 +48786,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 9.9e-07, "supports_function_calling": true, @@ -50713,7 +48797,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9e-07, "supports_tool_choice": true @@ -50723,7 +48806,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1000000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.5e-06, "supports_vision": true, @@ -50742,7 +48824,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "supports_vision": true, @@ -50763,7 +48844,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -50772,7 +48852,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "supports_vision": true, @@ -50784,7 +48863,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -50793,7 +48871,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -50802,7 +48879,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -50811,7 +48887,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.9e-07, "supports_tool_choice": true @@ -50821,7 +48896,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 8e-08, "supports_tool_choice": true @@ -50831,7 +48905,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_tool_choice": true @@ -50841,7 +48914,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-08, "supports_function_calling": true, @@ -50852,7 +48924,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.6e-07, "supports_vision": true, @@ -50864,7 +48935,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-07 }, @@ -50873,7 +48943,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-07, "supports_function_calling": true, @@ -50884,7 +48953,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_vision": true, @@ -50896,7 +48964,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_function_calling": true, @@ -50907,7 +48974,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "supports_tool_choice": true @@ -50917,7 +48983,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-07, "supports_vision": true, @@ -50929,7 +48994,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 9e-07, "supports_function_calling": true, @@ -50940,7 +49004,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -50949,7 +49012,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.8e-07, "supports_function_calling": true, @@ -50961,7 +49023,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "supports_function_calling": true, @@ -50973,7 +49034,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true @@ -50983,7 +49043,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, @@ -50994,7 +49053,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1e-07, "supports_vision": true, @@ -51006,7 +49064,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -51015,7 +49072,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -51026,7 +49082,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.9e-07 }, @@ -51035,7 +49090,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -51047,7 +49101,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 65536, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": true @@ -51057,7 +49110,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.5e-07, "supports_vision": true, @@ -51070,7 +49122,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 6e-06, "supports_vision": true, @@ -51083,7 +49134,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.2e-06, "supports_function_calling": true, @@ -51094,7 +49144,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06 }, @@ -51103,7 +49152,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.9e-06 }, @@ -51112,7 +49160,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -51123,7 +49170,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06 }, @@ -51132,7 +49178,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_vision": true, @@ -51146,7 +49191,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "supports_vision": true, @@ -51161,7 +49205,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "supports_vision": true, @@ -51176,7 +49219,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "supports_vision": true, @@ -51191,7 +49233,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_vision": true, @@ -51206,7 +49247,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "supports_vision": true, @@ -51221,7 +49261,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_vision": true, @@ -51236,7 +49275,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_vision": true, @@ -51251,7 +49289,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supports_function_calling": true, @@ -51265,7 +49302,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supports_vision": true, @@ -51278,7 +49314,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -51287,7 +49322,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -51296,7 +49330,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -51305,7 +49338,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 127000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -51314,7 +49346,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.5e-05 }, @@ -51323,7 +49354,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 127000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 5e-06 }, @@ -51332,7 +49362,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 127000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 8e-06 }, @@ -51341,7 +49370,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_vision": true, @@ -51353,7 +49381,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_vision": true, @@ -51365,7 +49392,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -51376,7 +49402,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1e-05, "supports_vision": true, @@ -51388,7 +49413,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -51399,7 +49423,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_function_calling": true @@ -51409,7 +49432,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "supports_function_calling": true, @@ -51420,7 +49442,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-06, "supports_function_calling": true, @@ -51431,7 +49452,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -51442,7 +49462,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.2e-06, "supports_function_calling": true, @@ -51453,7 +49472,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 96000, - "max_tokens": 96000, "mode": "chat", "output_cost_per_token": 1.1e-06, "supports_function_calling": true, @@ -51465,7 +49483,6 @@ "input_cost_per_token": 4.5e-07, "max_input_tokens": 200000, "max_output_tokens": 200000, - "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 1.8e-06, "source": "https://vercel.com/ai-gateway/models/glm-4.6", @@ -51516,7 +49533,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -51530,7 +49546,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -51550,7 +49565,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -51579,7 +49593,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -51602,7 +49615,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -51618,7 +49630,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -51633,7 +49644,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, "supports_assistant_prefill": true, @@ -51647,7 +49657,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, "supports_assistant_prefill": true, @@ -51661,7 +49670,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_assistant_prefill": true, @@ -51675,7 +49683,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_assistant_prefill": true, @@ -51688,7 +49695,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -51701,7 +49707,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -51721,7 +49726,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51756,7 +49760,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51795,7 +49798,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51833,7 +49835,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51870,7 +49871,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51908,7 +49908,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51946,7 +49945,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -51984,7 +49982,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -52028,7 +50025,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -52066,7 +50062,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -52111,7 +50106,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52150,7 +50144,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52187,7 +50180,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52228,7 +50220,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52271,7 +50262,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52310,7 +50300,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52350,7 +50339,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -52380,7 +50368,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -52419,7 +50406,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -52458,7 +50444,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -52482,7 +50467,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -52494,7 +50478,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -52506,7 +50489,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -52518,7 +50500,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -52530,7 +50511,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -52541,7 +50521,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -52552,7 +50531,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -52566,7 +50544,6 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 163840, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.7e-06, "output_cost_per_token_batches": 8.5e-07, @@ -52588,7 +50565,6 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 163840, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.68e-06, "output_cost_per_token_batches": 8.4e-07, @@ -52609,7 +50585,6 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 65336, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, "output_cost_per_token_batches": 2.7e-06, @@ -52632,7 +50607,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, @@ -52691,7 +50665,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -52726,7 +50699,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -52759,7 +50731,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, @@ -52787,7 +50758,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, @@ -52817,7 +50787,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -52857,7 +50826,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -52916,7 +50884,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -52977,7 +50944,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -53034,7 +51000,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -53048,7 +51013,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -53059,7 +51023,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -53070,7 +51033,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -53081,7 +51043,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -53092,7 +51053,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -53122,7 +51082,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "audio_speech", "output_cost_per_image": 0.04, "output_cost_per_token": 0, @@ -53158,7 +51117,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "audio_speech", "output_cost_per_image": 0.08, "output_cost_per_token": 0, @@ -53195,7 +51153,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.6e-05, "source": "https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama-3.2-90b-vision-instruct-maas", @@ -53208,7 +51165,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama-3.2-90b-vision-instruct-maas", @@ -53221,7 +51177,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "metadata": { "notes": "VertexAI states that The Llama 3.1 API service for llama-3.1-70b-instruct-maas and llama-3.1-8b-instruct-maas are in public preview and at no cost." }, @@ -53237,7 +51192,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "metadata": { "notes": "VertexAI states that The Llama 3.2 API service is at no cost during public preview, and will be priced as per dollar-per-1M-tokens at GA." }, @@ -53254,7 +51208,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.15e-06, "output_cost_per_token_batches": 5.75e-07, @@ -53276,7 +51229,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.15e-06, "output_cost_per_token_batches": 5.75e-07, @@ -53298,7 +51250,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 10000000, "max_output_tokens": 10000000, - "max_tokens": 10000000, "mode": "chat", "output_cost_per_token": 7e-07, "output_cost_per_token_batches": 3.5e-07, @@ -53320,7 +51271,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 10000000, "max_output_tokens": 10000000, - "max_tokens": 10000000, "mode": "chat", "output_cost_per_token": 7e-07, "output_cost_per_token_batches": 3.5e-07, @@ -53341,7 +51291,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", @@ -53352,7 +51301,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", @@ -53363,7 +51311,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", @@ -53376,7 +51323,6 @@ "litellm_provider": "vertex_ai-minimax_models", "max_input_tokens": 196608, "max_output_tokens": 196608, - "max_tokens": 196608, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53390,7 +51336,6 @@ "litellm_provider": "vertex_ai-moonshot_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53405,7 +51350,6 @@ "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53423,7 +51367,6 @@ "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53440,7 +51383,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53452,7 +51394,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53464,7 +51405,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53476,7 +51416,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53488,7 +51427,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -53499,7 +51437,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -53510,7 +51447,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -53521,7 +51457,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -53532,7 +51467,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -53543,7 +51477,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-07, "supports_function_calling": true, @@ -53554,7 +51487,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -53567,7 +51499,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -53601,7 +51532,6 @@ "litellm_provider": "vertex_ai-openai_models", "max_input_tokens": 262144, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/maas/google/gemma-4-26b-a4b-it", @@ -53618,7 +51548,6 @@ "litellm_provider": "vertex_ai-openai_models", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.6e-07, "output_cost_per_token_batches": 1.8e-07, @@ -53631,7 +51560,6 @@ "litellm_provider": "vertex_ai-openai_models", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.5e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53646,7 +51574,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53663,7 +51590,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53683,7 +51609,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 2000000, "max_output_tokens": 2000000, - "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -53703,7 +51628,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 2000000, "max_output_tokens": 2000000, - "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -53724,7 +51648,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 200000, "max_output_tokens": 200000, - "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -53744,7 +51667,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 524288, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -53763,7 +51685,6 @@ "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 8.8e-07, "output_cost_per_token_batches": 4.4e-07, @@ -53783,7 +51704,6 @@ "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.8e-06, "output_cost_per_token_batches": 9e-07, @@ -53800,7 +51720,6 @@ "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53816,7 +51735,6 @@ "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53829,7 +51747,6 @@ "vertex_ai/veo-3.1-generate-preview": { "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, @@ -53844,7 +51761,6 @@ "vertex_ai/veo-3.1-fast-generate-preview": { "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -53861,7 +51777,6 @@ "deprecation_date": "2026-11-17", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, @@ -53877,7 +51792,6 @@ "deprecation_date": "2026-11-17", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -53893,7 +51807,6 @@ "vertex_ai/veo-3.1-lite-generate-001": { "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.05, "output_cost_per_second_1080p": 0.08, @@ -53911,7 +51824,6 @@ "litellm_provider": "voyage", "max_input_tokens": 16000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -53920,7 +51832,6 @@ "litellm_provider": "voyage", "max_input_tokens": 8000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -53929,7 +51840,6 @@ "litellm_provider": "voyage", "max_input_tokens": 8000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.voyageai.com/docs/pricing" @@ -53939,7 +51849,6 @@ "litellm_provider": "voyage", "max_input_tokens": 4000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.voyageai.com/docs/pricing" @@ -53949,7 +51858,6 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -53958,7 +51866,6 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -53967,7 +51874,6 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.voyageai.com/docs/pricing" @@ -53977,7 +51883,6 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.voyageai.com/docs/pricing" @@ -53986,7 +51891,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "voyage", "max_input_tokens": 4000, - "max_tokens": 4000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -53994,7 +51898,6 @@ "input_cost_per_token": 6e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54002,7 +51905,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -54012,7 +51914,6 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54020,7 +51921,6 @@ "input_cost_per_token": 2e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54028,7 +51928,6 @@ "input_cost_per_token": 6e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54036,7 +51935,6 @@ "input_cost_per_token": 2e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54044,7 +51942,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 16000, - "max_tokens": 16000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54052,7 +51949,6 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54060,7 +51956,6 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "voyage", "max_input_tokens": 120000, - "max_tokens": 120000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54068,7 +51963,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54076,7 +51970,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 16000, - "max_tokens": 16000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54084,7 +51977,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 16000, - "max_tokens": 16000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -54094,7 +51986,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 16000, - "max_tokens": 16000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54102,7 +51993,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "voyage", "max_input_tokens": 4096, - "max_tokens": 4096, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54110,7 +52000,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "voyage", "max_input_tokens": 4000, - "max_tokens": 4000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54118,13 +52007,11 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, "wandb/openai/gpt-oss-120b": { "supports_reasoning": true, - "max_tokens": 131072, "max_input_tokens": 131000, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, @@ -54135,7 +52022,6 @@ }, "wandb/openai/gpt-oss-20b": { "supports_reasoning": true, - "max_tokens": 131072, "max_input_tokens": 131000, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, @@ -54145,7 +52031,6 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/moonshotai/Kimi-K2.5": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "cache_read_input_token_cost": 1e-07, @@ -54160,7 +52045,6 @@ "supports_vision": true }, "wandb/meta-llama/Llama-3.1-8B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 131000, "max_output_tokens": 128000, "input_cost_per_token": 2.2e-07, @@ -54171,7 +52055,6 @@ }, "wandb/deepseek-ai/DeepSeek-V3.1": { "supports_reasoning": true, - "max_tokens": 128000, "max_input_tokens": 161000, "max_output_tokens": 128000, "input_cost_per_token": 5.5e-07, @@ -54181,7 +52064,6 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/meta-llama/Llama-3.3-70B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 7.1e-07, @@ -54195,7 +52077,6 @@ "litellm_provider": "watsonx", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 2e-07, "supports_audio_input": false, @@ -54213,7 +52094,6 @@ "litellm_provider": "watsonx", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_audio_input": false, @@ -54227,7 +52107,6 @@ "supports_vision": false }, "watsonx/bigscience/mt0-xxl-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.908e-06, @@ -54240,7 +52119,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/bigscience/mt0-xxl": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.908e-06, @@ -54253,7 +52131,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/core42/jais-13b-chat": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 0.0005, @@ -54265,7 +52142,6 @@ "supports_vision": false }, "watsonx/google/flan-t5-xl-3b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 6e-07, @@ -54277,7 +52153,6 @@ "supports_vision": false }, "watsonx/ibm/granite-13b-chat-v2": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 6e-07, @@ -54289,7 +52164,6 @@ "supports_vision": false }, "watsonx/ibm/granite-13b-instruct-v2": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 6e-07, @@ -54301,7 +52175,6 @@ "supports_vision": false }, "watsonx/ibm/granite-3-3-8b-instruct": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -54313,7 +52186,6 @@ "supports_vision": false }, "watsonx/ibm/granite-4-h-small": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6.36e-08, @@ -54326,7 +52198,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/ibm/granite-guardian-3-2-2b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -54338,7 +52209,6 @@ "supports_vision": false }, "watsonx/ibm/granite-guardian-3-3-8b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -54350,7 +52220,6 @@ "supports_vision": false }, "watsonx/ibm/granite-ttm-1024-96-r2": { - "max_tokens": 512, "max_input_tokens": 512, "max_output_tokens": 512, "input_cost_per_token": 3.8e-07, @@ -54362,7 +52231,6 @@ "supports_vision": false }, "watsonx/ibm/granite-ttm-1536-96-r2": { - "max_tokens": 512, "max_input_tokens": 512, "max_output_tokens": 512, "input_cost_per_token": 3.8e-07, @@ -54374,7 +52242,6 @@ "supports_vision": false }, "watsonx/ibm/granite-ttm-512-96-r2": { - "max_tokens": 512, "max_input_tokens": 512, "max_output_tokens": 512, "input_cost_per_token": 3.8e-07, @@ -54386,7 +52253,6 @@ "supports_vision": false }, "watsonx/ibm/granite-vision-3-2-2b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -54398,7 +52264,6 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-2-11b-vision-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 3.5e-07, @@ -54410,7 +52275,6 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-2-1b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1e-07, @@ -54422,7 +52286,6 @@ "supports_vision": false }, "watsonx/meta-llama/llama-3-2-3b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.5e-07, @@ -54434,7 +52297,6 @@ "supports_vision": false }, "watsonx/meta-llama/llama-3-2-90b-vision-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-06, @@ -54446,7 +52308,6 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-3-70b-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 7.526e-07, @@ -54459,7 +52320,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b": { - "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 3.71e-07, @@ -54472,7 +52332,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b-128e-instruct-fp8": { - "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 3.71e-07, @@ -54485,7 +52344,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-guard-3-11b-vision": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 3.5e-07, @@ -54497,7 +52355,6 @@ "supports_vision": true }, "watsonx/mistralai/mistral-medium-2505": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, @@ -54509,7 +52366,6 @@ "supports_vision": false }, "watsonx/mistralai/mistral-small-2503": { - "max_tokens": 32000, "max_input_tokens": 32000, "max_output_tokens": 32000, "input_cost_per_token": 1e-07, @@ -54521,7 +52377,6 @@ "supports_vision": false }, "watsonx/mistralai/mistral-small-3-1-24b-instruct-2503": { - "max_tokens": 16384, "max_input_tokens": 131072, "max_output_tokens": 16384, "input_cost_per_token": 1.06e-07, @@ -54534,7 +52389,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/mistralai/pixtral-12b-2409": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 3.5e-07, @@ -54546,7 +52400,6 @@ "supports_vision": true }, "watsonx/openai/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.59e-07, @@ -54559,7 +52412,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/sdaia/allam-1-13b-instruct": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1.8e-06, @@ -54596,7 +52448,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -54621,7 +52472,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -54645,7 +52495,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -54671,7 +52520,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -54699,7 +52547,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -54727,7 +52574,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -54751,7 +52597,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54773,7 +52618,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54795,7 +52639,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54817,7 +52660,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54839,7 +52681,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54861,7 +52702,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, "output_cost_per_token_batches": 1.1e-06, @@ -54884,7 +52724,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_batches": 1.6e-06, "output_cost_per_token_priority": 5.6e-06, @@ -54907,7 +52746,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -55505,7 +53343,6 @@ } }, "fireworks_ai/accounts/fireworks/models/qwen3-coder-480b-a35b-instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 4.5e-07, @@ -55515,7 +53352,6 @@ "supports_reasoning": true }, "fireworks_ai/accounts/fireworks/models/flux-kontext-pro": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 4e-08, @@ -55524,7 +53360,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/SSD-1B": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -55533,7 +53368,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/chronos-hermes-13b-v2": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55542,7 +53376,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-13b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55551,7 +53384,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-13b-instruct": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55560,7 +53392,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-13b-python": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55569,7 +53400,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-34b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -55578,7 +53408,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-34b-instruct": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -55587,7 +53416,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-34b-python": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -55596,7 +53424,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-70b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -55605,7 +53432,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-70b-instruct": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -55614,7 +53440,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-70b-python": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -55623,7 +53448,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-7b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55632,7 +53456,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-7b-instruct": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55641,7 +53464,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-7b-python": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55650,7 +53472,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-qwen-1p5-7b": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 2e-07, @@ -55659,7 +53480,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/codegemma-2b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -55668,7 +53488,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/codegemma-7b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -55677,7 +53496,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-671b-v2-p1": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 1.2e-06, @@ -55686,7 +53504,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-llama-3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -55695,7 +53512,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-llama-70b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55704,7 +53520,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-llama-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55713,7 +53528,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-qwen-14b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55722,7 +53536,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-qwen-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55731,7 +53544,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/flux-kontext-max": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 8e-08, @@ -55740,7 +53552,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/dbrx-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1.2e-06, @@ -55749,7 +53560,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-1b-base": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 1e-07, @@ -55758,7 +53568,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-33b-instruct": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -55767,7 +53576,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-7b-base": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55776,7 +53584,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-7b-base-v1p5": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55785,7 +53592,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-7b-instruct-v1p5": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55794,7 +53600,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-v2-lite-base": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 5e-07, @@ -55803,7 +53608,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-v2-lite-instruct": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 5e-07, @@ -55812,7 +53616,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-prover-v2": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 1.2e-06, @@ -55821,7 +53624,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-0528-distill-qwen3-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55830,7 +53632,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-llama-70b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55839,7 +53640,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-llama-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55848,7 +53648,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-qwen-14b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55857,7 +53656,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-qwen-1p5b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -55866,7 +53664,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-qwen-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55875,7 +53672,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-qwen-7b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55884,7 +53680,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-v2-lite-chat": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 5e-07, @@ -55893,7 +53688,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-v2p5": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1.2e-06, @@ -55902,7 +53696,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/devstral-small-2505": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55911,7 +53704,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/dobby-mini-unhinged-plus-llama-3-1-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55920,7 +53712,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/dobby-unhinged-llama-3-3-70b-new": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55929,7 +53720,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/dolphin-2-9-2-qwen2-72b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55938,7 +53728,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/dolphin-2p6-mixtral-8x7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -55947,7 +53736,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ernie-4p5-21b-a3b-pt": { - "max_tokens": 4096, "max_input_tokens": 131072, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -55956,7 +53744,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ernie-4p5-300b-a47b-pt": { - "max_tokens": 4096, "max_input_tokens": 131072, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -55965,7 +53752,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/fare-20b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55974,7 +53760,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/firefunction-v1": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -55983,7 +53768,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/firellava-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55992,7 +53776,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/firesearch-ocr-v6": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56001,7 +53784,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/fireworks-asr-large": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 0.0, @@ -56010,7 +53792,6 @@ "mode": "audio_transcription" }, "fireworks_ai/accounts/fireworks/models/fireworks-asr-v2": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 0.0, @@ -56019,7 +53800,6 @@ "mode": "audio_transcription" }, "fireworks_ai/accounts/fireworks/models/flux-1-dev": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56028,7 +53808,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/flux-1-dev-controlnet-union": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-09, @@ -56037,7 +53816,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/flux-1-dev-fp8": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 5e-10, @@ -56046,7 +53824,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/flux-1-schnell": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56055,7 +53832,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/flux-1-schnell-fp8": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 3.5e-10, @@ -56064,7 +53840,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/gemma-2b-it": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -56073,7 +53848,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gemma-3-27b-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56082,7 +53856,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gemma-7b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56091,7 +53864,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gemma-7b-it": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56100,7 +53872,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gemma2-9b-it": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56109,7 +53880,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/glm-4p5v": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.2e-06, @@ -56119,7 +53889,6 @@ "supports_reasoning": true }, "fireworks_ai/accounts/fireworks/models/gpt-oss-safeguard-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.2e-06, @@ -56128,7 +53897,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gpt-oss-safeguard-20b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-07, @@ -56137,7 +53905,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/hermes-2-pro-mistral-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56146,7 +53913,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/internvl3-38b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56155,7 +53921,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/internvl3-78b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56164,7 +53929,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/internvl3-8b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -56173,7 +53937,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/japanese-stable-diffusion-xl": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -56182,7 +53945,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/kat-coder": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-07, @@ -56191,7 +53953,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/kat-dev-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56200,7 +53961,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/kat-dev-72b-exp": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56209,7 +53969,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-guard-2-8b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56218,7 +53977,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-guard-3-1b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -56227,7 +53985,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-guard-3-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56236,7 +53993,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56245,7 +54001,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-13b-chat": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56254,7 +54009,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-70b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56263,7 +54017,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-70b-chat": { - "max_tokens": 2048, "max_input_tokens": 2048, "max_output_tokens": 2048, "input_cost_per_token": 9e-07, @@ -56272,7 +54025,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-7b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56281,7 +54033,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-7b-chat": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56290,7 +54041,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 9e-07, @@ -56299,7 +54049,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct-hf": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 9e-07, @@ -56308,7 +54057,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3-8b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56317,7 +54065,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3-8b-instruct-hf": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56326,7 +54073,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct-long": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56335,7 +54081,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56344,7 +54089,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct-1b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56353,7 +54097,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-nemotron-70b-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56362,7 +54105,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p2-1b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -56371,7 +54113,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p2-3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -56380,7 +54121,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56389,7 +54129,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llamaguard-7b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56398,7 +54137,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llava-yi-34b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -56407,7 +54145,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/minimax-m1-80k": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56416,7 +54153,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/minimax-m2": { - "max_tokens": 4096, "max_input_tokens": 196608, "max_output_tokens": 4096, "input_cost_per_token": 3e-07, @@ -56425,7 +54161,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ministral-3-14b-instruct-2512": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 2e-07, @@ -56434,7 +54169,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ministral-3-3b-instruct-2512": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 1e-07, @@ -56443,7 +54177,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ministral-3-8b-instruct-2512": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 2e-07, @@ -56452,7 +54185,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56461,7 +54193,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b-instruct-4k": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56470,7 +54201,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b-instruct-v0p2": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56479,7 +54209,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b-instruct-v3": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56488,7 +54217,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b-v0p2": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56497,7 +54225,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-large-3-fp8": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 1.2e-06, @@ -56506,7 +54233,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-nemo-base-2407": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -56515,7 +54241,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-nemo-instruct-2407": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -56524,7 +54249,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-small-24b-instruct-2501": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56533,7 +54257,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 1.2e-06, @@ -56542,7 +54265,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 1.2e-06, @@ -56551,7 +54273,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -56560,7 +54281,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -56569,7 +54289,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x7b-instruct-hf": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -56578,7 +54297,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mythomax-l2-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56587,7 +54305,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nemotron-nano-v2-12b-vl": { - "max_tokens": 4096, "max_input_tokens": 131072, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56596,7 +54313,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-capybara-7b-v1p9": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56605,7 +54321,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-2-mixtral-8x7b-dpo": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -56614,7 +54329,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-2-yi-34b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -56623,7 +54337,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-llama2-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56632,7 +54345,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-llama2-70b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -56641,7 +54353,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-llama2-7b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56650,7 +54361,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nvidia-nemotron-nano-12b-v2": { - "max_tokens": 131072, "max_input_tokens": 128000, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56659,7 +54369,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nvidia-nemotron-nano-9b-v2": { - "max_tokens": 131072, "max_input_tokens": 128000, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56668,7 +54377,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/openchat-3p5-0106-7b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56677,7 +54385,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/openhermes-2-mistral-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56686,7 +54393,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/openhermes-2p5-mistral-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56695,7 +54401,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/openorca-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56704,7 +54409,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phi-2-3b": { - "max_tokens": 2048, "max_input_tokens": 2048, "max_output_tokens": 2048, "input_cost_per_token": 1e-07, @@ -56713,7 +54417,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phi-3-mini-128k-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -56722,7 +54425,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phi-3-vision-128k-instruct": { - "max_tokens": 32064, "max_input_tokens": 32064, "max_output_tokens": 32064, "input_cost_per_token": 2e-07, @@ -56731,7 +54433,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phind-code-llama-34b-python-v1": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56740,7 +54441,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phind-code-llama-34b-v1": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56749,7 +54449,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phind-code-llama-34b-v2": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56758,7 +54457,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/playground-v2-1024px-aesthetic": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -56767,7 +54465,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/playground-v2-5-1024px-aesthetic": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -56776,7 +54473,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/pythia-12b": { - "max_tokens": 2048, "max_input_tokens": 2048, "max_output_tokens": 2048, "input_cost_per_token": 2e-07, @@ -56785,7 +54481,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen-qwq-32b-preview": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56794,7 +54489,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen-v2p5-14b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56803,7 +54497,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen-v2p5-7b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56812,7 +54505,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen1p5-72b-chat": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56821,7 +54513,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56830,7 +54521,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2-vl-2b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56839,7 +54529,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2-vl-72b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56848,7 +54537,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2-vl-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56857,7 +54545,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-0p5b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56866,7 +54553,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-14b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56875,7 +54561,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-1p5b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56884,7 +54569,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56893,7 +54577,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-32b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56902,7 +54585,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-72b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56911,7 +54593,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-72b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56920,7 +54601,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56929,7 +54609,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-0p5b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56938,7 +54617,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-0p5b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56947,7 +54625,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-14b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56956,7 +54633,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-14b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56965,7 +54641,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-1p5b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56974,7 +54649,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-1p5b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56983,7 +54657,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56992,7 +54665,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct-128k": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -57001,7 +54673,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct-32k-rope": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -57010,7 +54681,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct-64k": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 9e-07, @@ -57019,7 +54689,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-3b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -57028,7 +54697,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-3b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -57037,7 +54705,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57046,7 +54713,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57055,7 +54721,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-math-72b-instruct": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57064,7 +54729,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-vl-32b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 9e-07, @@ -57073,7 +54737,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-vl-3b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -57082,7 +54745,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-vl-72b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 9e-07, @@ -57091,7 +54753,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-vl-7b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -57100,7 +54761,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-0p6b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1e-07, @@ -57109,7 +54769,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-14b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 2e-07, @@ -57118,7 +54777,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-1p7b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -57127,7 +54785,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-1p7b-fp8-draft": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1e-07, @@ -57136,7 +54793,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-1p7b-fp8-draft-131072": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -57145,7 +54801,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-1p7b-fp8-draft-40960": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1e-07, @@ -57154,7 +54809,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-235b-a22b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2.2e-07, @@ -57163,7 +54817,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-235b-a22b-instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.2e-07, @@ -57172,7 +54825,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-235b-a22b-thinking-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.2e-07, @@ -57181,7 +54833,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-30b-a3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.5e-07, @@ -57190,7 +54841,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-30b-a3b-instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 5e-07, @@ -57199,7 +54849,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-30b-a3b-thinking-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-07, @@ -57208,7 +54857,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -57218,7 +54866,6 @@ "supports_reasoning": true }, "fireworks_ai/accounts/fireworks/models/qwen3-4b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 2e-07, @@ -57227,7 +54874,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-4b-instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2e-07, @@ -57236,7 +54882,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-8b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 2e-07, @@ -57246,7 +54891,6 @@ "supports_reasoning": true }, "fireworks_ai/accounts/fireworks/models/qwen3-coder-30b-a3b-instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -57255,7 +54899,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-coder-480b-instruct-bf16": { - "max_tokens": 4096, "max_input_tokens": 262144, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57264,7 +54907,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-embedding-0p6b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 0.0, @@ -57273,7 +54915,6 @@ "mode": "embedding" }, "fireworks_ai/accounts/fireworks/models/qwen3-embedding-4b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 0.0, @@ -57282,7 +54923,6 @@ "mode": "embedding" }, "fireworks_ai/accounts/fireworks/models/qwen3-next-80b-a3b-instruct": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57291,7 +54931,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-next-80b-a3b-thinking": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57300,7 +54939,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-reranker-0p6b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 0.0, @@ -57309,7 +54947,6 @@ "mode": "rerank" }, "fireworks_ai/accounts/fireworks/models/qwen3-reranker-4b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 0.0, @@ -57318,7 +54955,6 @@ "mode": "rerank" }, "fireworks_ai/accounts/fireworks/models/qwen3-reranker-8b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 2e-07, @@ -57328,7 +54964,6 @@ "source": "https://api.fireworks.ai/v1/serverless/models" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-235b-a22b-instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.2e-07, @@ -57337,7 +54972,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-235b-a22b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.2e-07, @@ -57346,7 +54980,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-30b-a3b-instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -57355,7 +54988,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-30b-a3b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -57364,7 +54996,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-32b-instruct": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57373,7 +55004,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-8b-instruct": { - "max_tokens": 4096, "max_input_tokens": 262144, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -57387,7 +55017,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -57398,7 +55027,6 @@ "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/qwq-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -57407,7 +55035,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/rolm-ocr": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -57416,7 +55043,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/snorkel-mistral-7b-pairrm-dpo": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57425,7 +55051,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/stable-diffusion-xl-1024-v1-0": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -57434,7 +55059,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/stablecode-3b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -57443,7 +55067,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder-16b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -57452,7 +55075,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder-7b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -57461,7 +55083,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder2-15b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -57470,7 +55091,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder2-3b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 1e-07, @@ -57479,7 +55099,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder2-7b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -57488,7 +55107,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/toppy-m-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57497,7 +55115,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/yi-34b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57506,7 +55123,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/yi-34b-200k-capybara": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 9e-07, @@ -57515,7 +55131,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/yi-34b-chat": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57524,7 +55139,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/yi-6b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -57533,7 +55147,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/zephyr-7b-beta": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57547,7 +55160,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -57564,7 +55176,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -57581,7 +55192,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -57596,7 +55206,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-06, "supports_function_calling": true, @@ -57608,7 +55217,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -57620,7 +55228,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.25e-06, "supports_function_calling": true @@ -57636,7 +55243,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -57647,7 +55253,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true @@ -57663,7 +55268,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 5e-07, "supports_function_calling": true, @@ -57675,7 +55279,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.5e-06, "supports_reasoning": true, @@ -57688,7 +55291,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.5e-07, "supports_function_calling": true, @@ -57699,7 +55301,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-07, "supports_vision": true, @@ -57717,7 +55318,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 9e-07, "supports_function_calling": true @@ -57729,7 +55329,6 @@ "output_cost_per_token": 4e-07, "max_input_tokens": 163840, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57746,7 +55345,6 @@ "output_cost_per_token": 1.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57762,7 +55360,6 @@ "output_cost_per_token": 2.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57779,7 +55376,6 @@ "output_cost_per_token": 3.3e-07, "max_input_tokens": 262144, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57796,7 +55392,6 @@ "output_cost_per_token": 1.38e-07, "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_vision": true, "supports_system_messages": true }, @@ -57807,7 +55402,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 262144, "max_output_tokens": 100352, - "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57824,7 +55418,6 @@ "output_cost_per_token": 1.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57841,7 +55434,6 @@ "output_cost_per_token": 2e-08, "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_vision": true, "supports_system_messages": true }, @@ -57852,7 +55444,6 @@ "output_cost_per_token": 4.1e-07, "max_input_tokens": 163840, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57867,7 +55458,6 @@ "output_cost_per_token": 3.95e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_vision": true, "supports_system_messages": true, "supports_reasoning": true, @@ -57881,7 +55471,6 @@ "output_cost_per_token": 9e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57899,7 +55488,6 @@ "output_cost_per_token": 2.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57916,7 +55504,6 @@ "output_cost_per_token": 1.2e-06, "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57932,7 +55519,6 @@ "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57947,7 +55533,6 @@ "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57962,7 +55547,6 @@ "output_cost_per_token": 3e-08, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_vision": true, "supports_system_messages": true, @@ -57975,7 +55559,6 @@ "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57992,7 +55575,6 @@ "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58007,7 +55589,6 @@ "output_cost_per_token": 8.45e-06, "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58021,7 +55602,6 @@ "output_cost_per_token": 6e-07, "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_tool_choice": true, "supports_vision": true, "supports_system_messages": true, @@ -58034,7 +55614,6 @@ "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58051,7 +55630,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 262144, "max_output_tokens": 100352, - "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58065,7 +55643,6 @@ "output_cost_per_token": 1.55e-06, "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58079,7 +55656,6 @@ "output_cost_per_token": 2.7e-07, "max_input_tokens": 160000, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58093,7 +55669,6 @@ "output_cost_per_token": 2.5e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58109,7 +55684,6 @@ "output_cost_per_token": 2.3e-06, "max_input_tokens": 131072, "max_output_tokens": 100352, - "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58123,7 +55697,6 @@ "output_cost_per_token": 1.12e-06, "max_input_tokens": 163840, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58139,7 +55712,6 @@ "output_cost_per_token": 2.2e-06, "max_input_tokens": 131072, "max_output_tokens": 98304, - "max_tokens": 98304, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58155,7 +55727,6 @@ "output_cost_per_token": 3e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58169,7 +55740,6 @@ "output_cost_per_token": 5e-08, "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_system_messages": true, "supports_response_schema": true }, @@ -58180,7 +55750,6 @@ "output_cost_per_token": 1e-07, "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_vision": true, "supports_system_messages": true, @@ -58193,7 +55762,6 @@ "output_cost_per_token": 1.8e-06, "max_input_tokens": 65536, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58211,7 +55779,6 @@ "output_cost_per_token": 1.5e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_tool_choice": true, "supports_vision": false, "supports_system_messages": true, @@ -58225,7 +55792,6 @@ "output_cost_per_token": 5.8e-07, "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58239,7 +55805,6 @@ "output_cost_per_token": 1.5e-07, "max_input_tokens": 32768, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, @@ -58252,7 +55817,6 @@ "output_cost_per_token": 4e-07, "max_input_tokens": 12288, "max_output_tokens": 12288, - "max_tokens": 12288, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58266,7 +55830,6 @@ "output_cost_per_token": 4e-07, "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58280,7 +55843,6 @@ "output_cost_per_token": 1.7e-07, "max_input_tokens": 60288, "max_output_tokens": 16000, - "max_tokens": 16000, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58292,7 +55854,6 @@ "output_cost_per_token": 2.2e-06, "max_input_tokens": 1000000, "max_output_tokens": 40000, - "max_tokens": 40000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58307,7 +55868,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 163840, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58324,7 +55884,6 @@ "output_cost_per_token": 3e-07, "max_input_tokens": 64000, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, @@ -58337,7 +55896,6 @@ "output_cost_per_token": 4e-08, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_system_messages": true, "supports_response_schema": true }, @@ -58348,7 +55906,6 @@ "output_cost_per_token": 6.2e-07, "max_input_tokens": 65535, "max_output_tokens": 8000, - "max_tokens": 8000, "supports_system_messages": true, "supports_response_schema": true }, @@ -58359,7 +55916,6 @@ "output_cost_per_token": 9e-08, "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_system_messages": true, "supports_reasoning": true }, @@ -58370,7 +55926,6 @@ "output_cost_per_token": 8e-07, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, @@ -58383,7 +55938,6 @@ "output_cost_per_token": 7.4e-07, "max_input_tokens": 8192, "max_output_tokens": 8000, - "max_tokens": 8000, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58395,7 +55949,6 @@ "output_cost_per_token": 8e-07, "max_input_tokens": 40960, "max_output_tokens": 20000, - "max_tokens": 20000, "supports_system_messages": true, "supports_reasoning": true, "supports_response_schema": true @@ -58407,7 +55960,6 @@ "output_cost_per_token": 8.5e-07, "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_vision": true, "supports_system_messages": true, "supports_response_schema": true @@ -58419,7 +55971,6 @@ "output_cost_per_token": 5.9e-07, "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_vision": true, "supports_system_messages": true }, @@ -58430,7 +55981,6 @@ "output_cost_per_token": 1.4e-07, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58442,7 +55992,6 @@ "output_cost_per_token": 8e-07, "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_vision": true, "supports_system_messages": true }, @@ -58453,7 +56002,6 @@ "output_cost_per_token": 1.48e-06, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58466,7 +56014,6 @@ "output_cost_per_token": 2.8e-07, "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_system_messages": true, "supports_reasoning": true }, @@ -58477,7 +56024,6 @@ "output_cost_per_token": 5e-08, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58489,7 +56035,6 @@ "output_cost_per_token": 7e-08, "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_system_messages": true }, "novita/baidu/ernie-4.5-vl-424b-a47b": { @@ -58499,7 +56044,6 @@ "output_cost_per_token": 1.25e-06, "max_input_tokens": 123000, "max_output_tokens": 16000, - "max_tokens": 16000, "supports_vision": true, "supports_system_messages": true, "supports_reasoning": true @@ -58511,7 +56055,6 @@ "output_cost_per_token": 1.1e-06, "max_input_tokens": 123000, "max_output_tokens": 12000, - "max_tokens": 12000, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58523,7 +56066,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 160000, "max_output_tokens": 160000, - "max_tokens": 160000, "supports_system_messages": true }, "novita/qwen/qwen3-32b-fp8": { @@ -58533,7 +56075,6 @@ "output_cost_per_token": 4.5e-07, "max_input_tokens": 40960, "max_output_tokens": 20000, - "max_tokens": 20000, "supports_system_messages": true, "supports_reasoning": true }, @@ -58544,7 +56085,6 @@ "output_cost_per_token": 4.5e-07, "max_input_tokens": 40960, "max_output_tokens": 20000, - "max_tokens": 20000, "supports_system_messages": true, "supports_reasoning": true, "supports_function_calling": true, @@ -58557,7 +56097,6 @@ "output_cost_per_token": 2e-07, "max_input_tokens": 98304, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_vision": true, "supports_system_messages": true }, @@ -58568,7 +56107,6 @@ "output_cost_per_token": 1.3e-06, "max_input_tokens": 64000, "max_output_tokens": 16000, - "max_tokens": 16000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58581,7 +56119,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 64000, "max_output_tokens": 16000, - "max_tokens": 16000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58596,7 +56133,6 @@ "output_cost_per_token": 5e-08, "max_input_tokens": 8192, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58609,7 +56145,6 @@ "output_cost_per_token": 9e-08, "max_input_tokens": 4096, "max_output_tokens": 3200, - "max_tokens": 3200, "supports_system_messages": true }, "novita/baidu/ernie-4.5-vl-28b-a3b-thinking": { @@ -58619,7 +56154,6 @@ "output_cost_per_token": 3.9e-07, "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58635,7 +56169,6 @@ "output_cost_per_token": 5e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58650,7 +56183,6 @@ "output_cost_per_token": 8.5e-07, "max_input_tokens": 131072, "max_output_tokens": 98304, - "max_tokens": 98304, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58666,7 +56198,6 @@ "output_cost_per_token": 7e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58681,7 +56212,6 @@ "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58696,7 +56226,6 @@ "output_cost_per_token": 9.7e-07, "max_input_tokens": 65536, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58713,7 +56242,6 @@ "output_cost_per_token": 9.7e-07, "max_input_tokens": 65536, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58730,7 +56258,6 @@ "output_cost_per_token": 7.5e-07, "max_input_tokens": 16384, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_system_messages": true }, "novita/baidu/ernie-4.5-vl-28b-a3b": { @@ -58740,7 +56267,6 @@ "output_cost_per_token": 5.6e-07, "max_input_tokens": 30000, "max_output_tokens": 8000, - "max_tokens": 8000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58755,7 +56281,6 @@ "output_cost_per_token": 2.8e-07, "max_input_tokens": 120000, "max_output_tokens": 8000, - "max_tokens": 8000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58768,7 +56293,6 @@ "output_cost_per_token": 1.38e-07, "max_input_tokens": 128000, "max_output_tokens": 20000, - "max_tokens": 20000, "supports_system_messages": true, "supports_reasoning": true }, @@ -58779,7 +56303,6 @@ "output_cost_per_token": 3e-08, "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_system_messages": true, "supports_reasoning": true, "supports_function_calling": true, @@ -58792,7 +56315,6 @@ "output_cost_per_token": 7e-08, "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58806,7 +56328,6 @@ "output_cost_per_token": 5e-08, "max_input_tokens": 32768, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58819,7 +56340,6 @@ "output_cost_per_token": 1.48e-06, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58831,8 +56351,7 @@ "input_cost_per_token": 7e-08, "output_cost_per_token": 0, "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768 + "max_output_tokens": 32768 }, "novita/qwen/qwen3-embedding-8b": { "litellm_provider": "novita", @@ -58840,8 +56359,7 @@ "input_cost_per_token": 7e-08, "output_cost_per_token": 0, "max_input_tokens": 32768, - "max_output_tokens": 4096, - "max_tokens": 4096 + "max_output_tokens": 4096 }, "novita/baai/bge-m3": { "litellm_provider": "novita", @@ -58849,8 +56367,7 @@ "input_cost_per_token": 1e-08, "output_cost_per_token": 1e-08, "max_input_tokens": 8192, - "max_output_tokens": 96000, - "max_tokens": 96000 + "max_output_tokens": 96000 }, "novita/qwen/qwen3-reranker-8b": { "litellm_provider": "novita", @@ -58858,8 +56375,7 @@ "input_cost_per_token": 5e-08, "output_cost_per_token": 5e-08, "max_input_tokens": 32768, - "max_output_tokens": 4096, - "max_tokens": 4096 + "max_output_tokens": 4096 }, "novita/baai/bge-reranker-v2-m3": { "litellm_provider": "novita", @@ -58867,11 +56383,9 @@ "input_cost_per_token": 1e-08, "output_cost_per_token": 1e-08, "max_input_tokens": 8000, - "max_output_tokens": 8000, - "max_tokens": 8000 + "max_output_tokens": 8000 }, "llamagate/llama-3.1-8b": { - "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 3e-08, @@ -58882,7 +56396,6 @@ "supports_response_schema": true }, "llamagate/llama-3.2-3b": { - "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 4e-08, @@ -58893,7 +56406,6 @@ "supports_response_schema": true }, "llamagate/mistral-7b-v0.3": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -58904,7 +56416,6 @@ "supports_response_schema": true }, "llamagate/qwen3-8b": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 4e-08, @@ -58915,7 +56426,6 @@ "supports_response_schema": true }, "llamagate/dolphin3-8b": { - "max_tokens": 8192, "max_input_tokens": 128000, "max_output_tokens": 8192, "input_cost_per_token": 8e-08, @@ -58926,7 +56436,6 @@ "supports_response_schema": true }, "llamagate/deepseek-r1-8b": { - "max_tokens": 16384, "max_input_tokens": 65536, "max_output_tokens": 16384, "input_cost_per_token": 1e-07, @@ -58938,7 +56447,6 @@ "supports_reasoning": true }, "llamagate/deepseek-r1-7b-qwen": { - "max_tokens": 16384, "max_input_tokens": 131072, "max_output_tokens": 16384, "input_cost_per_token": 8e-08, @@ -58950,7 +56458,6 @@ "supports_reasoning": true }, "llamagate/openthinker-7b": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 8e-08, @@ -58962,7 +56469,6 @@ "supports_reasoning": true }, "llamagate/qwen2.5-coder-7b": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 6e-08, @@ -58973,7 +56479,6 @@ "supports_response_schema": true }, "llamagate/deepseek-coder-6.7b": { - "max_tokens": 4096, "max_input_tokens": 16384, "max_output_tokens": 4096, "input_cost_per_token": 6e-08, @@ -58984,7 +56489,6 @@ "supports_response_schema": true }, "llamagate/codellama-7b": { - "max_tokens": 4096, "max_input_tokens": 16384, "max_output_tokens": 4096, "input_cost_per_token": 6e-08, @@ -58995,7 +56499,6 @@ "supports_response_schema": true }, "llamagate/qwen3-vl-8b": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 1.5e-07, @@ -59007,7 +56510,6 @@ "supports_vision": true }, "llamagate/llava-7b": { - "max_tokens": 2048, "max_input_tokens": 4096, "max_output_tokens": 2048, "input_cost_per_token": 1e-07, @@ -59018,7 +56520,6 @@ "supports_vision": true }, "llamagate/gemma3-4b": { - "max_tokens": 8192, "max_input_tokens": 128000, "max_output_tokens": 8192, "input_cost_per_token": 3e-08, @@ -59030,7 +56531,6 @@ "supports_vision": true }, "llamagate/nomic-embed-text": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 2e-08, "output_cost_per_token": 0, @@ -59038,7 +56538,6 @@ "mode": "embedding" }, "llamagate/qwen3-embedding-8b": { - "max_tokens": 40960, "max_input_tokens": 40960, "input_cost_per_token": 2e-08, "output_cost_per_token": 0, @@ -59046,7 +56545,6 @@ "mode": "embedding" }, "libertai/hermes-3-8b-tee": { - "max_tokens": 16000, "max_input_tokens": 16000, "max_output_tokens": 16000, "input_cost_per_token": 1.5e-07, @@ -59060,7 +56558,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/gemma-4-31b-it": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59074,7 +56571,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/gemma-4-31b-it-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59089,7 +56585,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.6-27b": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59103,7 +56598,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.6-27b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59118,7 +56612,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.6-35b-a3b": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59132,7 +56625,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.6-35b-a3b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59147,7 +56639,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.5-122b-a10b": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.5e-07, @@ -59161,7 +56652,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.5-122b-a10b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.5e-07, @@ -59176,7 +56666,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/deepseek-v4-flash": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 2.5e-07, @@ -59190,7 +56679,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/deepseek-v4-flash-thinking": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 2.5e-07, @@ -59205,7 +56693,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/bge-m3": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -59221,7 +56708,6 @@ "litellm_provider": "sarvam", "max_input_tokens": 8192, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 0, "supports_reasoning": true @@ -59297,7 +56783,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -59325,7 +56810,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -59357,7 +56841,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -59467,7 +56950,6 @@ "litellm_provider": "gemini", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_token": 0.0, @@ -59488,7 +56970,6 @@ "litellm_provider": "gemini", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_token": 0.0, @@ -59511,7 +56992,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59537,7 +57017,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59564,7 +57043,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59599,7 +57077,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -59634,7 +57111,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -59668,7 +57144,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -59701,7 +57176,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59729,7 +57203,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59758,7 +57231,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59796,7 +57268,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -59833,7 +57304,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, @@ -59860,7 +57330,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 9e-06, "output_cost_per_token": 9e-06, @@ -59885,7 +57354,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 6e-06, "output_cost_per_token": 6e-06, @@ -59905,7 +57373,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, @@ -59925,7 +57392,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -59984,7 +57450,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -60044,7 +57509,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -60106,7 +57570,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -60167,7 +57630,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -60218,7 +57680,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -60257,7 +57718,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -60299,7 +57759,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1.05e-06, @@ -60325,7 +57784,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token_batches": 1.5e-07, "output_cost_per_token_priority": 5.25e-07, @@ -60351,7 +57809,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1.05e-06, @@ -60375,7 +57832,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 1e-07, "output_cost_per_token_priority": 3.5e-07, @@ -60405,7 +57861,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -60449,7 +57904,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -60484,7 +57938,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-cyber.html", "use_openai_responses_path": true, @@ -60520,7 +57973,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -60560,7 +58012,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -60601,7 +58052,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html", "supported_modalities": [ @@ -60638,7 +58088,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html", "supported_modalities": [ @@ -60675,7 +58124,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html", "supported_modalities": [ @@ -60712,7 +58160,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html", "supported_modalities": [ @@ -60749,7 +58196,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html", "supported_modalities": [ @@ -60783,7 +58229,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html", "supported_modalities": [ @@ -60815,7 +58260,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html", "supported_modalities": [ @@ -60847,7 +58291,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html", "supported_modalities": [ @@ -60879,7 +58322,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html", "supported_modalities": [ @@ -60914,7 +58356,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html", "supported_modalities": [ @@ -60955,7 +58396,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token_above_272k_tokens_ultrafast": 0.000495, "output_cost_per_token_ultrafast": 0.00033, @@ -60996,7 +58436,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -61035,7 +58474,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -61082,7 +58520,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_ultrafast": 0.000495, "output_cost_per_token_ultrafast": 0.00033, @@ -61123,7 +58560,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61162,7 +58598,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61207,7 +58642,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_ultrafast": 0.00045, "output_cost_per_token_ultrafast": 0.0003, @@ -61247,7 +58681,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61282,7 +58715,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61320,7 +58752,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61355,7 +58786,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61395,7 +58825,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -61435,7 +58864,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -61470,7 +58898,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token_batches": 2e-07, "output_cost_per_token_priority": 7e-07, @@ -61496,7 +58923,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token_batches": 2e-07, "output_cost_per_token_priority": 7e-07, @@ -61522,7 +58948,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_batches": 4e-08, "output_cost_per_token_priority": 1.4e-07, @@ -61552,7 +58977,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1048576, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_flex": 1.25e-06, @@ -61580,7 +59004,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token_flex": 3.3e-06, "output_cost_per_token_priority": 1.155e-05, @@ -61605,7 +59028,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -61636,7 +59058,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -61674,7 +59095,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -61722,7 +59142,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token_flex": 3.3e-06, "output_cost_per_token_priority": 1.155e-05, @@ -61750,7 +59169,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token_flex": 3e-06, "output_cost_per_token_priority": 1.05e-05, @@ -61765,7 +59183,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, @@ -61805,7 +59222,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4.3125e-06, "source": "https://www.volcengine.com/docs/82379/1544106", @@ -61821,7 +59237,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 1024000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4.3125e-06, "source": "https://www.volcengine.com/docs/82379/1544106", @@ -61839,7 +59254,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 1024000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3.88126e-07, "source": "https://www.volcengine.com/docs/82379/1544106", @@ -61856,7 +59270,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.15625e-06, "source": "https://www.volcengine.com/docs/82379/1544106", @@ -61870,7 +59283,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, @@ -61908,7 +59320,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, @@ -61946,7 +59357,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, @@ -62041,7 +59451,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, @@ -62059,7 +59468,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, @@ -62080,7 +59488,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -62110,7 +59517,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -62132,7 +59538,6 @@ } }, "snowflake/claude-sonnet-4-5": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 3e-06, @@ -62150,7 +59555,6 @@ "snowflake/claude-sonnet-4-6": { "supports_adaptive_thinking": true, "supports_legacy_thinking": true, - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 3e-06, @@ -62166,7 +59570,6 @@ "prompt_cache_min_tokens": 1024 }, "snowflake/claude-4-sonnet": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 3e-06, @@ -62182,7 +59585,6 @@ "prompt_cache_min_tokens": 1024 }, "snowflake/claude-4-opus": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 5e-06, @@ -62199,7 +59601,6 @@ "prompt_cache_min_tokens": 1024 }, "snowflake/claude-haiku-4-5": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 1e-06, @@ -62215,7 +59616,6 @@ "prompt_cache_min_tokens": 4096 }, "snowflake/claude-3-7-sonnet": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 3e-06, @@ -62231,7 +59631,6 @@ "supports_response_schema": true }, "snowflake/openai-gpt-4.1": { - "max_tokens": 16384, "max_input_tokens": 300000, "max_output_tokens": 16384, "input_cost_per_token": 2e-06, @@ -62246,7 +59645,6 @@ "supports_response_schema": true }, "snowflake/openai-gpt-5": { - "max_tokens": 16384, "max_input_tokens": 300000, "max_output_tokens": 16384, "input_cost_per_token": 1.25e-06, @@ -62262,7 +59660,6 @@ "supports_response_schema": true }, "snowflake/openai-gpt-5-mini": { - "max_tokens": 16384, "max_input_tokens": 1000000, "max_output_tokens": 16384, "input_cost_per_token": 3e-07, @@ -62274,7 +59671,6 @@ "supports_response_schema": true }, "snowflake/openai-gpt-5-nano": { - "max_tokens": 16384, "max_input_tokens": 5000000, "max_output_tokens": 16384, "input_cost_per_token": 1.5e-07, @@ -62286,7 +59682,6 @@ "supports_response_schema": true }, "snowflake/llama4-maverick": { - "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.4e-07, @@ -62297,7 +59692,6 @@ "supports_system_messages": true }, "snowflake/snowflake-arctic-embed-l-v2.0": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 7e-08, "output_cost_per_token": 0.0, @@ -62305,7 +59699,6 @@ "mode": "embedding" }, "snowflake/snowflake-arctic-embed-m-v2.0": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 7e-08, "output_cost_per_token": 0.0, @@ -62315,7 +59708,7 @@ "soniox/stt-async-v4": { "litellm_provider": "soniox", "max_output_tokens": 8000, - "max_tokens": 8000, + "max_input_tokens": 8000, "input_cost_per_second": 0.0, "output_cost_per_second": 2.77778e-05, "mode": "audio_transcription", @@ -62328,7 +59721,7 @@ "soniox/stt-async-v5": { "litellm_provider": "soniox", "max_output_tokens": 8000, - "max_tokens": 8000, + "max_input_tokens": 8000, "input_cost_per_second": 0.0, "output_cost_per_second": 2.77778e-05, "mode": "audio_transcription", @@ -62505,7 +59898,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62559,7 +59951,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62613,7 +60004,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62667,7 +60057,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 2.2e-08, @@ -62721,7 +60110,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62775,7 +60163,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62829,7 +60216,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62883,7 +60269,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 2.2e-08, @@ -62937,7 +60322,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.48e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -62963,7 +60347,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -62989,7 +60372,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 2.2e-08, @@ -63044,7 +60426,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -63099,7 +60480,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63124,7 +60504,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63149,7 +60528,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63173,7 +60551,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63197,7 +60574,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63221,7 +60597,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63245,7 +60620,6 @@ "litellm_provider": "tencent", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63269,7 +60643,6 @@ "litellm_provider": "tencent", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63336,7 +60709,6 @@ "source": "https://docs.devin.ai/desktop/models" }, "pinstripes/ps/glm-4.5-air": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.25e-07, @@ -63349,7 +60721,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3.6-35b-a3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.4e-07, @@ -63362,7 +60733,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3-30b-a3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-08, @@ -63375,7 +60745,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3-coder-30b-a3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-07, @@ -63388,7 +60757,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/deepseek-v4-flash": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 1e-07, @@ -63401,7 +60769,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/minimax-m2.7": { - "max_tokens": 1000192, "max_input_tokens": 1000192, "max_output_tokens": 1000192, "input_cost_per_token": 2.55e-07, @@ -63418,7 +60785,6 @@ "litellm_provider": "darkbloom", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.65e-07, "source": "https://www.darkbloom.dev/", @@ -63435,7 +60801,6 @@ "litellm_provider": "darkbloom", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7e-08, "source": "https://www.darkbloom.dev/", @@ -63455,7 +60820,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -63482,7 +60846,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -63511,7 +60874,6 @@ "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://api.x.ai/v1/language-models", @@ -63584,7 +60946,6 @@ "litellm_provider": "openai", "max_input_tokens": 16000, "max_output_tokens": 2000, - "max_tokens": 2000, "mode": "realtime", "source": "https://developers.openai.com/api/docs/pricing", "supported_modalities": [ @@ -63609,7 +60970,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -63653,7 +61013,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -63696,7 +61055,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "prompt_cache_min_tokens": 512, @@ -63732,7 +61090,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 5e-06, "search_context_cost_per_query": { @@ -63767,7 +61124,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6e-07, "reasoning_effort_levels": [ @@ -63788,7 +61144,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/models/model-cards/leanstral-1-5", @@ -63878,7 +61233,6 @@ "mode": "chat", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, @@ -63977,7 +61331,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 2.1e-05, "output_cost_per_token": 2.1e-05, @@ -64006,7 +61359,6 @@ "litellm_provider": "gemini", "max_input_tokens": 98304, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "audio_transcription", "output_cost_per_token": 1.2e-05, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -64031,7 +61383,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "audio_transcription", "output_cost_per_token": 2.1e-05, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -64112,7 +61463,6 @@ "input_cost_per_token": 8e-09, "litellm_provider": "perplexity", "max_input_tokens": 32768, - "max_tokens": 32768, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64122,7 +61472,6 @@ "input_cost_per_token": 5e-08, "litellm_provider": "perplexity", "max_input_tokens": 32768, - "max_tokens": 32768, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2560, @@ -64132,7 +61481,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64142,7 +61490,6 @@ "input_cost_per_token": 6e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64152,7 +61499,6 @@ "input_cost_per_token": 2e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64162,7 +61508,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64172,7 +61517,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 120000, - "max_tokens": 120000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64182,7 +61526,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64198,7 +61541,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_priority": 8.25e-07, @@ -64217,7 +61559,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.5e-06, @@ -64237,7 +61578,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.8e-06, "output_cost_per_token_priority": 2.25e-06, @@ -64257,7 +61597,7 @@ "input_cost_per_token_priority": 2.75e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_priority": 8.25e-07, @@ -64275,7 +61615,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_priority": 1.875e-05, @@ -64300,7 +61639,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_priority": 8.25e-07, @@ -64319,7 +61657,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.5e-06, @@ -64339,7 +61676,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.8e-06, "output_cost_per_token_priority": 2.25e-06, @@ -64359,7 +61695,7 @@ "input_cost_per_token_priority": 2.75e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_priority": 8.25e-07, @@ -64376,7 +61712,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -64393,7 +61728,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -64411,7 +61745,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_priority": 1.875e-05, @@ -64433,7 +61766,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, "reasoning_effort_levels": [ @@ -64456,7 +61788,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, "output_cost_per_token_priority": 2.8125e-05, @@ -64498,7 +61829,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_priority": 2.25e-06, @@ -64518,7 +61848,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_priority": 2.5e-07, @@ -64538,7 +61867,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.4e-06, "output_cost_per_token_priority": 3e-06, @@ -64558,7 +61886,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_priority": 2.25e-06, @@ -64593,7 +61920,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_priority": 2.5e-07, @@ -64613,7 +61939,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.4e-06, "output_cost_per_token_priority": 3e-06, @@ -64648,7 +61973,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -64665,7 +61989,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -64681,7 +62004,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, "reasoning_effort_levels": [ @@ -64704,7 +62026,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, "output_cost_per_token_priority": 2.8125e-05, @@ -64726,7 +62047,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -64743,7 +62063,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 3.96e-06, "source": "https://novita.ai/pricing", @@ -64760,7 +62079,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://novita.ai/pricing", @@ -64777,7 +62095,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 5.8e-07, "source": "https://novita.ai/pricing", @@ -64794,7 +62111,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://novita.ai/pricing", @@ -64811,7 +62127,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://novita.ai/pricing", @@ -64828,7 +62143,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -64845,7 +62159,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://novita.ai/pricing", @@ -64862,7 +62175,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.5e-06, "source": "https://novita.ai/pricing", @@ -64878,7 +62190,6 @@ "litellm_provider": "novita", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://novita.ai/pricing", @@ -64895,7 +62206,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://novita.ai/pricing", @@ -64912,7 +62222,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 3.2000000000000003e-06, "source": "https://novita.ai/pricing", @@ -64929,7 +62238,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.8e-07, "source": "https://novita.ai/pricing", @@ -64945,7 +62253,6 @@ "litellm_provider": "novita", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://novita.ai/pricing", @@ -64962,7 +62269,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.8e-07, "source": "https://novita.ai/pricing", @@ -64978,7 +62284,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.6e-06, "source": "https://novita.ai/pricing", @@ -64994,7 +62299,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.15e-06, "source": "https://novita.ai/pricing", @@ -65010,7 +62314,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.0000000000000002e-07, "source": "https://novita.ai/pricing", @@ -65026,7 +62329,6 @@ "litellm_provider": "novita", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.13e-06, "source": "https://novita.ai/pricing", @@ -65042,7 +62344,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.3600000000000004e-07, "source": "https://novita.ai/pricing", @@ -65059,7 +62360,6 @@ "litellm_provider": "novita", "max_input_tokens": 1000000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.75e-06, "source": "https://novita.ai/pricing", @@ -65076,7 +62376,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.044e-06, "source": "https://novita.ai/pricing", @@ -65092,7 +62391,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.6000000000000003e-06, "source": "https://novita.ai/pricing", @@ -65108,7 +62406,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.4e-06, "source": "https://novita.ai/pricing", @@ -65125,7 +62422,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://novita.ai/pricing", @@ -65142,7 +62438,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65159,7 +62454,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65175,7 +62469,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.0000000000000003e-07, "source": "https://novita.ai/pricing", @@ -65190,7 +62483,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.0000000000000003e-07, "source": "https://novita.ai/pricing", @@ -65206,7 +62498,6 @@ "litellm_provider": "novita", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65223,7 +62514,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://novita.ai/pricing", @@ -65240,7 +62530,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131100, - "max_tokens": 131100, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://novita.ai/pricing", @@ -65256,7 +62545,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://novita.ai/pricing", @@ -65271,7 +62559,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.2000000000000003e-06, "source": "https://novita.ai/pricing", @@ -65286,7 +62573,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://novita.ai/pricing", @@ -65301,7 +62587,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.6000000000000003e-06, "source": "https://novita.ai/pricing", @@ -65317,7 +62602,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131100, - "max_tokens": 131100, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://novita.ai/pricing", @@ -65334,7 +62618,6 @@ "litellm_provider": "novita", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.2000000000000003e-06, "source": "https://novita.ai/pricing", @@ -65350,7 +62633,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://novita.ai/pricing", @@ -65364,7 +62646,6 @@ "litellm_provider": "novita", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-08, "source": "https://novita.ai/pricing", @@ -65376,7 +62657,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://novita.ai/pricing", @@ -65393,7 +62673,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65410,7 +62689,6 @@ "litellm_provider": "novita", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.0000000000000003e-07, "source": "https://novita.ai/pricing", @@ -65426,7 +62704,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.4850000000000002e-06, "source": "https://novita.ai/pricing", @@ -65441,7 +62718,6 @@ "litellm_provider": "novita", "max_input_tokens": 64000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 8.900000000000001e-07, "source": "https://api.novita.ai/v3/openai/models", @@ -65454,7 +62730,6 @@ "litellm_provider": "novita", "max_input_tokens": 64000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65468,7 +62743,6 @@ "litellm_provider": "novita", "max_input_tokens": 64000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 8.900000000000001e-07, "source": "https://api.novita.ai/v3/openai/models", @@ -65481,7 +62755,6 @@ "litellm_provider": "novita", "max_input_tokens": 64000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65495,7 +62768,6 @@ "litellm_provider": "novita", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.66e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65509,7 +62781,6 @@ "litellm_provider": "novita", "max_input_tokens": 131000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2e-08, "source": "https://api.novita.ai/v3/openai/models", @@ -65518,7 +62789,7 @@ }, "wandb/deepseek-ai/DeepSeek-V4-Flash": { "supports_reasoning": true, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1049000, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, @@ -65532,7 +62803,7 @@ }, "wandb/deepseek-ai/DeepSeek-V4-Flash-0731": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 1.3e-07, "output_cost_per_token": 2.8e-07, @@ -65545,7 +62816,7 @@ }, "wandb/deepseek-ai/DeepSeek-V4-Pro": { "supports_reasoning": true, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1049000, "input_cost_per_token": 1.15e-06, "output_cost_per_token": 2.55e-06, @@ -65559,7 +62830,7 @@ }, "wandb/google/gemma-4-31B-it": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 1e-07, "output_cost_per_token": 3.4e-07, @@ -65570,7 +62841,7 @@ }, "wandb/ibm-granite/granite-4.1-8b": { "deprecation_date": "2026-10-05", - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131000, "input_cost_per_token": 5e-08, "output_cost_per_token": 1e-07, @@ -65581,7 +62852,7 @@ }, "wandb/JetBrains/Mellum2-12B-A2.5B-Instruct": { "deprecation_date": "2026-10-05", - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131000, "input_cost_per_token": 5e-08, "output_cost_per_token": 1e-07, @@ -65592,7 +62863,7 @@ }, "wandb/meta-llama/Llama-3.1-70B-Instruct": { "deprecation_date": "2026-10-05", - "max_tokens": 128000, + "max_output_tokens": 128000, "max_input_tokens": 131000, "input_cost_per_token": 8e-07, "output_cost_per_token": 8e-07, @@ -65603,7 +62874,7 @@ }, "wandb/MiniMaxAI/MiniMax-M3": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 2.3e-07, "output_cost_per_token": 9.6e-07, @@ -65616,7 +62887,7 @@ }, "wandb/moonshotai/Kimi-K2.7-Code": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 7.1e-07, "output_cost_per_token": 3.5e-06, @@ -65629,7 +62900,7 @@ }, "wandb/moonshotai/Kimi-K2.6": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 6.5e-07, "output_cost_per_token": 3.41e-06, @@ -65642,7 +62913,7 @@ }, "wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 7e-08, "output_cost_per_token": 2e-07, @@ -65655,7 +62926,7 @@ }, "wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 5e-07, "output_cost_per_token": 2.15e-06, @@ -65668,7 +62939,7 @@ }, "wandb/OpenPipe/Qwen3-14B-Instruct": { "deprecation_date": "2026-10-05", - "max_tokens": 32768, + "max_output_tokens": 32768, "max_input_tokens": 32800, "input_cost_per_token": 5e-08, "output_cost_per_token": 2.2e-07, @@ -65679,7 +62950,7 @@ }, "wandb/Qwen/Qwen3.8-27B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 4e-07, "output_cost_per_token": 3e-06, @@ -65692,7 +62963,7 @@ }, "wandb/Qwen/Qwen3.6-35B-A3B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.25e-06, @@ -65703,7 +62974,7 @@ }, "wandb/Qwen/Qwen3.6-27B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 6e-07, "output_cost_per_token": 3.6e-06, @@ -65718,7 +62989,7 @@ "wandb/Qwen/Qwen3.5-35B-A3B": { "deprecation_date": "2026-10-05", "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.25e-06, @@ -65729,7 +63000,7 @@ }, "wandb/Qwen/Qwen3-30B-A3B-Instruct-2507": { "deprecation_date": "2026-10-05", - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 1e-07, "output_cost_per_token": 3e-07, @@ -65762,7 +63033,7 @@ }, "wandb/zai-org/GLM-5.2": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 1049000, "input_cost_per_token": 7.6e-07, "output_cost_per_token": 2.42e-06, @@ -65774,7 +63045,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "deepinfra/openai/gpt-oss-120b-Turbo": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07, @@ -65788,7 +63059,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/MiniMaxAI/MiniMax-M2.7": { - "max_tokens": 196608, + "max_output_tokens": 196608, "max_input_tokens": 196608, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1e-06, @@ -65804,7 +63075,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.8-27B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4e-07, "output_cost_per_token": 3e-06, @@ -65820,7 +63091,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-31B-it-Ultra": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 2.7e-07, "output_cost_per_token": 7.6e-07, @@ -65834,7 +63105,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/moonshotai/Kimi-K2.5": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4.5e-07, "output_cost_per_token": 2.25e-06, @@ -65850,7 +63121,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.7-Flash": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 6e-08, "output_cost_per_token": 4e-07, @@ -65866,7 +63137,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.6": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 5e-07, "output_cost_per_token": 2e-06, @@ -65885,7 +63156,7 @@ "input_cost_per_token": 5e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-05, "prompt_cache_min_tokens": 1024, @@ -65899,7 +63170,7 @@ "supports_vision": true }, "deepinfra/anthropic/claude-sonnet-4-6": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, @@ -65915,7 +63186,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemini-3.5-flash": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 1.5e-06, "output_cost_per_token": 9e-06, @@ -65930,7 +63201,7 @@ "supports_audio_input": true }, "deepinfra/XiaomiMiMo/MiMo-V2.5": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4e-07, "output_cost_per_token": 2e-06, @@ -65946,7 +63217,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Max": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 1.2e-06, "output_cost_per_token": 6e-06, @@ -65961,7 +63232,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-31B-it-turbo": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 9e-08, "output_cost_per_token": 3.4e-07, @@ -65976,7 +63247,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/thinkingmachines/Inkling-Small": { - "max_tokens": 524288, + "max_output_tokens": 524288, "max_input_tokens": 524288, "input_cost_per_token": 4.5e-07, "output_cost_per_token": 1.2e-06, @@ -65992,7 +63263,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-models/Muse-Glimmer-30B": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, @@ -66008,7 +63279,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Max-Thinking": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 1.2e-06, "output_cost_per_token": 6e-06, @@ -66023,7 +63294,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-VL-235B-A22B-Instruct": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2e-07, "output_cost_per_token": 8.8e-07, @@ -66038,7 +63309,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-VL-30B-A3B-Instruct": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07, @@ -66051,7 +63322,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.5-27B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.6e-07, "output_cost_per_token": 2.6e-06, @@ -66065,7 +63336,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.6-35B-A3B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1e-07, "output_cost_per_token": 9.5e-07, @@ -66079,7 +63350,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/nvidia/Nemotron-Content-Safety-3.5": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 2e-07, "output_cost_per_token": 2e-07, @@ -66092,7 +63363,7 @@ "input_cost_per_token": 5e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-05, "prompt_cache_min_tokens": 512, @@ -66106,7 +63377,7 @@ "supports_vision": true }, "deepinfra/thinkingmachines/Inkling": { - "max_tokens": 524288, + "max_output_tokens": 524288, "max_input_tokens": 524288, "input_cost_per_token": 9.5e-07, "output_cost_per_token": 4.05e-06, @@ -66122,7 +63393,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/moonshotai/Kimi-K2.6": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7.5e-07, "output_cost_per_token": 3.5e-06, @@ -66138,7 +63409,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4-Pro-0813": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1.3e-06, "output_cost_per_token": 2.6e-06, @@ -66153,7 +63424,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.7-Max": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 2.5e-06, "output_cost_per_token": 7.5e-06, @@ -66168,7 +63439,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/ByteDance/Seed-2.0-mini": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 1e-07, "output_cost_per_token": 4e-07, @@ -66184,7 +63455,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.8-2.4T-A95B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, @@ -66200,7 +63471,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/MiniMaxAI/MiniMax-M3": { - "max_tokens": 524288, + "max_output_tokens": 524288, "max_input_tokens": 524288, "input_cost_per_token": 2.8e-07, "output_cost_per_token": 1.1e-06, @@ -66216,7 +63487,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemini-3.1-flash-lite": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.5e-06, @@ -66231,7 +63502,7 @@ "supports_audio_input": true }, "deepinfra/google/gemini-3.7-flash": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 7.5e-07, "output_cost_per_token": 3.75e-06, @@ -66247,7 +63518,7 @@ "supports_audio_input": true }, "deepinfra/inclusionAI/Ling-3.0-flash": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 6e-08, "output_cost_per_token": 1.8e-07, @@ -66263,7 +63534,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/stepfun-ai/Step-3.7-Flash": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2e-07, "output_cost_per_token": 1.15e-06, @@ -66277,7 +63548,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.5-35B-A3B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 1e-06, @@ -66293,7 +63564,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/ByteDance/Seed-1.8": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 2e-06, @@ -66309,7 +63580,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/tencent/Hy3": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 5.8e-07, @@ -66325,7 +63596,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/ByteDance/Seed-2.0-code": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 5e-07, "output_cost_per_token": 3e-06, @@ -66341,7 +63612,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/ByteDance/Seed-2.0-pro": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 5e-07, "output_cost_per_token": 3e-06, @@ -66357,7 +63628,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-5": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 6e-07, "output_cost_per_token": 2.08e-06, @@ -66373,7 +63644,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/nvidia/Nemotron-3-Nano-30B-A3B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 5e-08, "output_cost_per_token": 2e-07, @@ -66388,7 +63659,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/moonshotai/Kimi-K2.7-Code": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 6.8e-07, "output_cost_per_token": 3.4e-06, @@ -66407,7 +63678,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1e-05, "prompt_cache_min_tokens": 1024, @@ -66421,7 +63692,7 @@ "supports_vision": true }, "deepinfra/Qwen/Qwen3.5-397B-A17B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4.5e-07, "output_cost_per_token": 3e-06, @@ -66437,7 +63708,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 8e-08, "output_cost_per_token": 1.8e-07, @@ -66453,7 +63724,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-E4B-it": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 2e-08, "output_cost_per_token": 1e-07, @@ -66466,7 +63737,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3.2": { - "max_tokens": 163840, + "max_output_tokens": 163840, "max_input_tokens": 163840, "input_cost_per_token": 2.6e-07, "output_cost_per_token": 3.8e-07, @@ -66481,7 +63752,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.8-Max": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 1.65e-06, "output_cost_per_token": 4.951e-06, @@ -66499,7 +63770,7 @@ "input_cost_per_token": 1e-05, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 5e-05, "prompt_cache_min_tokens": 512, @@ -66514,7 +63785,7 @@ "thinking_always_on": true }, "deepinfra/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 5e-07, "output_cost_per_token": 2.2e-06, @@ -66530,7 +63801,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.5-122B-A10B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.9e-07, "output_cost_per_token": 2.4e-06, @@ -66544,7 +63815,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-5.1": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 1.05e-06, "output_cost_per_token": 3.5e-06, @@ -66560,7 +63831,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4-Pro": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1.3e-06, "output_cost_per_token": 2.6e-06, @@ -66575,7 +63846,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 8.5e-08, "output_cost_per_token": 4e-07, @@ -66589,7 +63860,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-5.2": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 7.5e-07, "output_cost_per_token": 2.4e-06, @@ -66605,7 +63876,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/moonshotai/Kimi-K3": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 2.85e-06, "output_cost_per_token": 1.425e-05, @@ -66623,7 +63894,7 @@ "input_cost_per_token": 5e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-05, "prompt_cache_min_tokens": 2048, @@ -66637,7 +63908,7 @@ "supports_vision": true }, "deepinfra/Qwen/Qwen3.6-27B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 3.2e-07, "output_cost_per_token": 3.2e-06, @@ -66651,7 +63922,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-26B-A4B-it": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7e-08, "output_cost_per_token": 3.4e-07, @@ -66665,7 +63936,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemini-3.1-pro": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 2e-06, "output_cost_per_token": 1.2e-05, @@ -66680,7 +63951,7 @@ "supports_audio_input": true }, "deepinfra/XiaomiMiMo/MiMo-V2.5-Pro": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1e-06, "output_cost_per_token": 3e-06, @@ -66696,7 +63967,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/anthropic/claude-haiku-4-5": { - "max_tokens": 200000, + "max_output_tokens": 200000, "max_input_tokens": 200000, "input_cost_per_token": 1e-06, "output_cost_per_token": 5e-06, @@ -66711,7 +63982,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4-Flash": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 9e-08, "output_cost_per_token": 1.8e-07, @@ -66727,7 +63998,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4.1-Flash": { - "max_tokens": 131072, "max_input_tokens": 1048576, "max_output_tokens": 131072, "input_cost_per_token": 1.4e-07, @@ -66744,7 +64014,6 @@ "source": "https://deepinfra.com/deepseek-ai/DeepSeek-V4.1-Flash" }, "deepinfra/deepseek-ai/DeepSeek-V4.1-Flash-1017": { - "max_tokens": 131072, "max_input_tokens": 1048576, "max_output_tokens": 131072, "input_cost_per_token": 1.4e-07, @@ -66761,7 +64030,7 @@ "source": "https://deepinfra.com/deepseek-ai/DeepSeek-V4.1-Flash-1017" }, "deepinfra/openai/gpt-oss-120b-Ultra": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 2e-07, "output_cost_per_token": 9.5e-07, @@ -66775,7 +64044,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.5-9B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1e-07, "output_cost_per_token": 1.5e-07, @@ -66788,7 +64057,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/MiniMaxAI/MiniMax-M2.7-Turbo": { - "max_tokens": 196608, + "max_output_tokens": 196608, "max_input_tokens": 196608, "input_cost_per_token": 3.8e-07, "output_cost_per_token": 1.7e-06, @@ -66804,7 +64073,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.7": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 4e-07, "output_cost_per_token": 1.75e-06, @@ -66820,7 +64089,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-31B-it": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.3e-07, "output_cost_per_token": 3.8e-07, @@ -66839,7 +64108,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -66874,7 +64142,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -66902,7 +64169,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -66930,7 +64196,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67206,7 +64471,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67233,7 +64497,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67260,7 +64523,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67291,7 +64553,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67319,7 +64580,6 @@ "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://console.groq.com/docs/model/qwen/qwen3.8-27b", @@ -67335,7 +64595,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -67356,7 +64615,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -67377,7 +64635,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -67398,7 +64655,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6e-07, "reasoning_effort_levels": [ @@ -67419,7 +64675,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", @@ -67434,7 +64689,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", @@ -67449,7 +64703,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://mistral.ai/news/devstral-2-vibe-cli", @@ -67550,7 +64803,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/models/model-cards/leanstral-1-5", @@ -67566,7 +64818,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_priority": 5.5e-06, @@ -67585,7 +64836,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.6e-06, "output_cost_per_token_priority": 8.25e-06, @@ -67604,7 +64854,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_priority": 5.5e-06, @@ -67623,7 +64872,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.6e-06, "output_cost_per_token_priority": 8.25e-06, @@ -67639,7 +64887,7 @@ "input_cost_per_token": 2.1e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -67681,7 +64929,7 @@ "input_cost_per_token": 2.1e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -67698,7 +64946,7 @@ "input_cost_per_token_priority": 1.875e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_priority": 6.25e-07, @@ -67716,7 +64964,7 @@ "input_cost_per_token_priority": 2.8125e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 7.5e-07, "output_cost_per_token_priority": 9.375e-07, @@ -67733,7 +64981,7 @@ "input_cost_per_token_priority": 1.875e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_priority": 6.25e-07, @@ -67751,7 +64999,7 @@ "input_cost_per_token_priority": 2.8125e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 7.5e-07, "output_cost_per_token_priority": 9.375e-07, @@ -67769,7 +65017,7 @@ "input_cost_per_token_priority": 1e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.05e-06, "output_cost_per_token_priority": 4.05e-06, @@ -67785,7 +65033,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 40960, "max_output_tokens": 40960, - "max_tokens": 40960, "mode": "embedding", "source": "https://api.fireworks.ai/v1/serverless/models" }, @@ -67808,7 +65055,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://api.together.ai/v1/models" @@ -67868,7 +65115,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 131072, "max_output_tokens": 40960, - "max_tokens": 40960, "mode": "chat", "output_cost_per_token": 1.49e-06, "source": "https://api.cerebras.ai/public/v1/models/gemma-4-31b", @@ -67894,7 +65140,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", @@ -67908,7 +65153,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", @@ -67924,7 +65168,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -67947,7 +65190,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": true, @@ -67989,7 +65231,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68004,7 +65245,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.1e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68019,7 +65259,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68034,7 +65273,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.25e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68050,7 +65288,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68067,7 +65304,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68080,7 +65316,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68155,7 +65390,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.88e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -68173,7 +65407,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", @@ -68189,7 +65422,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", @@ -68204,7 +65436,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -68222,7 +65453,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", @@ -68242,7 +65472,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", @@ -68267,7 +65496,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, @@ -68306,7 +65534,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 1024, @@ -68344,7 +65571,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, @@ -68381,7 +65607,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.4e-05, "prompt_cache_min_tokens": 512, @@ -68419,7 +65644,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_adaptive_thinking": true, @@ -68454,7 +65678,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.88e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -68472,7 +65695,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", @@ -68488,7 +65710,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", @@ -68503,7 +65724,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -68521,7 +65741,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", @@ -68541,7 +65760,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", @@ -68566,7 +65784,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, @@ -68600,7 +65817,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 1024, @@ -68633,7 +65849,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, @@ -68665,7 +65880,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.4e-05, "prompt_cache_min_tokens": 512, @@ -68698,7 +65912,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_adaptive_thinking": true, @@ -68727,7 +65940,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68763,7 +65975,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68799,7 +66010,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68830,7 +66040,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1048576, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -68854,7 +66063,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -68873,7 +66081,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68892,7 +66099,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68911,7 +66117,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68931,7 +66136,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ @@ -68951,7 +66155,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ @@ -68974,7 +66177,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.4e-05, "prompt_cache_min_tokens": 512, @@ -69010,7 +66212,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "supports_forced_tool_use": false, @@ -69039,7 +66240,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -69072,7 +66272,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -69092,7 +66291,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ @@ -69112,7 +66310,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ @@ -69136,7 +66333,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 8.4e-07, "output_cost_per_token_priority": 2.94e-06, @@ -69158,7 +66354,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_priority": 4.38e-06, @@ -69180,7 +66375,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 4.4e-07, "output_cost_per_token_priority": 1.54e-06, @@ -69202,7 +66396,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 32000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1.05e-06, @@ -69225,7 +66418,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1.05e-06, @@ -69247,7 +66439,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 9e-07, "output_cost_per_token_priority": 3.15e-06, @@ -69268,7 +66459,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 6e-07, "output_cost_per_token_priority": 2.1e-06, @@ -69290,7 +66480,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 1.33e-06, "output_cost_per_token_priority": 4.66e-06, @@ -69309,7 +66498,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.375e-05, "supported_endpoints": [ @@ -69343,7 +66531,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.05e-06, "supports_prompt_caching": true, @@ -69356,7 +66543,6 @@ "input_cost_per_token": 1.63e-07, "litellm_provider": "azure", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -69364,7 +66550,6 @@ "input_cost_per_token": 2.5e-08, "litellm_provider": "azure", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -69395,7 +66580,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_image": 0.04, "output_cost_per_token": 0, @@ -69420,7 +66604,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -69445,7 +66628,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 0, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -69470,7 +66652,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 0, "source": "https://ai.google.dev/gemini-api/docs/models/lyria-realtime-exp", @@ -69843,7 +67024,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -69867,7 +67047,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://inference.baseten.co/v1/models", @@ -69892,7 +67071,7 @@ "output_cost_per_token": 1.5e-07, "source": "https://api.together.ai/v1/models", "max_input_tokens": 128000, - "max_tokens": 128000 + "max_output_tokens": 128000 }, "vertex_ai/gemini-2.5-flash-native-audio": { "deprecation_date": "2026-12-13", @@ -69953,7 +67132,7 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai", "max_output_tokens": 57920, - "max_tokens": 57920, + "max_input_tokens": 57920, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -70016,7 +67195,7 @@ "output_cost_per_token": 3.5e-06, "source": "https://api.together.ai/v1/models", "max_input_tokens": 4096, - "max_tokens": 4096 + "max_output_tokens": 4096 }, "together_ai/meta-llama/Llama-3.2-1B-Instruct": { "input_cost_per_token": 6e-08, @@ -70025,7 +67204,7 @@ "output_cost_per_token": 6e-08, "source": "https://api.together.ai/v1/models", "max_input_tokens": 131072, - "max_tokens": 131072 + "max_output_tokens": 131072 }, "together_ai/meta-llama/Llama-3.2-3B-Instruct": { "input_cost_per_token": 6e-08, @@ -70034,7 +67213,7 @@ "output_cost_per_token": 6e-08, "source": "https://api.together.ai/v1/models", "max_input_tokens": 131072, - "max_tokens": 131072 + "max_output_tokens": 131072 }, "together_ai/Qwen/Qwen2-1.5B-Instruct": { "input_cost_per_token": 2e-08, @@ -70043,7 +67222,7 @@ "output_cost_per_token": 2e-08, "source": "https://api.together.ai/v1/models", "max_input_tokens": 32768, - "max_tokens": 32768 + "max_output_tokens": 32768 }, "together_ai/Qwen/Qwen2.5-14B-Instruct": { "input_cost_per_token": 8e-07, @@ -70052,7 +67231,7 @@ "output_cost_per_token": 8e-07, "source": "https://api.together.ai/v1/models", "max_input_tokens": 32768, - "max_tokens": 32768 + "max_output_tokens": 32768 }, "together_ai/Qwen/Qwen2.5-72B-Instruct": { "input_cost_per_token": 1.2e-06, @@ -70061,14 +67240,13 @@ "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", "max_input_tokens": 32768, - "max_tokens": 32768 + "max_output_tokens": 32768 }, "together_ai/Salesforce/Llama-Rank-V1": { "deprecation_date": "2026-02-06", "input_cost_per_token": 1e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://api.together.ai/v1/models" @@ -70077,7 +67255,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 16384, - "max_tokens": 16384, + "max_output_tokens": 16384, "mode": "completion", "output_cost_per_token": 2e-07, "source": "https://api.together.xyz/v1/models" @@ -70095,7 +67273,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://api.together.ai/v1/models" @@ -70105,7 +67283,7 @@ "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" @@ -70115,7 +67293,7 @@ "input_cost_per_token": 9e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://api.together.ai/v1/models" @@ -70125,7 +67303,7 @@ "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" @@ -70135,7 +67313,7 @@ "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" @@ -70145,7 +67323,7 @@ "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "max_input_tokens": 16384, - "max_tokens": 16384, + "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://api.together.ai/v1/models" @@ -70155,7 +67333,7 @@ "input_cost_per_token": 1.95e-06, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://api.together.ai/v1/models" @@ -70165,7 +67343,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://api.together.ai/v1/models" @@ -70175,7 +67353,7 @@ "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" @@ -70185,7 +67363,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://api.together.ai/v1/models" @@ -70195,7 +67373,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://api.together.ai/v1/models" @@ -70205,7 +67383,7 @@ "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://api.together.ai/v1/models" @@ -70215,7 +67393,7 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://api.together.ai/v1/models" @@ -70226,7 +67404,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.6e-06, "source": "https://api.together.ai/v1/models" @@ -70236,7 +67414,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://api.together.ai/v1/models" @@ -70246,7 +67424,7 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.8e-07, "source": "https://api.together.ai/v1/models" @@ -70256,7 +67434,7 @@ "input_cost_per_token": 1.6e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://api.together.ai/v1/models" @@ -70266,7 +67444,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.7e-06, "source": "https://api.together.ai/v1/models" @@ -70276,7 +67454,7 @@ "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "max_input_tokens": 16384, - "max_tokens": 16384, + "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://api.together.ai/v1/models" @@ -70286,7 +67464,7 @@ "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://api.together.ai/v1/models" @@ -70296,7 +67474,7 @@ "input_cost_per_token": 3.9e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 9.7e-07, "source": "https://api.together.ai/v1/models" @@ -70306,7 +67484,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://api.together.ai/v1/models" @@ -70316,7 +67494,7 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5.9e-07, "source": "https://api.together.ai/v1/models" @@ -70326,7 +67504,7 @@ "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://api.together.ai/v1/models" @@ -70336,7 +67514,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://api.together.ai/v1/models" @@ -70346,7 +67524,7 @@ "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://api.together.ai/v1/models" @@ -70356,7 +67534,7 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.8e-07, "source": "https://api.together.ai/v1/models" @@ -70366,7 +67544,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://api.together.ai/v1/models" @@ -70376,7 +67554,7 @@ "input_cost_per_token": 1e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://api.together.ai/v1/models" @@ -70386,7 +67564,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://api.together.ai/v1/models" @@ -70397,7 +67575,7 @@ "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.5e-06, "source": "https://api.together.ai/v1/models" @@ -70408,7 +67586,7 @@ "input_cost_per_token": 9.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.together.ai/v1/models" @@ -70418,7 +67596,7 @@ "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://api.together.ai/v1/models" @@ -70429,7 +67607,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 512288, - "max_tokens": 512288, + "max_output_tokens": 512288, "mode": "chat", "output_cost_per_token": 3.6e-06, "source": "https://api.together.ai/v1/models" @@ -70439,7 +67617,7 @@ "input_cost_per_token": 5e-08, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://api.together.ai/v1/models" @@ -70449,7 +67627,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.1e-06, "source": "https://api.together.ai/v1/models" @@ -70459,7 +67637,7 @@ "input_cost_per_token": 4.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 202752, - "max_tokens": 202752, + "max_output_tokens": 202752, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://api.together.ai/v1/models" @@ -70469,7 +67647,7 @@ "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", "max_input_tokens": 202752, - "max_tokens": 202752, + "max_output_tokens": 202752, "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://api.together.ai/v1/models" @@ -70480,7 +67658,7 @@ "input_cost_per_token": 1.4e-06, "litellm_provider": "together_ai", "max_input_tokens": 202752, - "max_tokens": 202752, + "max_output_tokens": 202752, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://api.together.ai/v1/models" @@ -70547,7 +67725,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_batches": 8.25e-06, @@ -70720,7 +67897,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_above_272k_tokens": 9e-05, @@ -70764,7 +67940,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_above_272k_tokens": 9e-07, @@ -70813,7 +67988,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -70869,7 +68043,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -70883,7 +68056,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -70906,7 +68078,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -70920,7 +68091,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -70955,7 +68125,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -70977,7 +68146,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -71053,7 +68221,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_batches": 8.25e-06, @@ -71266,7 +68433,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, @@ -71314,7 +68480,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "audio_transcription", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ @@ -71355,7 +68520,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ @@ -71373,7 +68537,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 512000, "max_output_tokens": 65500, - "max_tokens": 65500, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71386,7 +68549,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71399,7 +68561,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 200000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.2e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71414,7 +68575,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71435,7 +68595,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71453,7 +68612,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71473,7 +68631,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71494,7 +68651,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71513,7 +68669,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.2e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71528,7 +68683,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.61333e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71557,7 +68711,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.6e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71573,7 +68726,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71587,7 +68739,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71602,7 +68753,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.84e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71617,7 +68767,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.38e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71632,7 +68781,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.411e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71648,7 +68796,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.4246e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71665,7 +68812,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.0136e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71682,7 +68828,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.411e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71699,7 +68844,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.32375e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71716,7 +68860,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 119000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.5353e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71729,7 +68872,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71746,7 +68888,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71763,7 +68904,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71780,7 +68920,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71797,7 +68936,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.499999e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71815,7 +68953,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.75e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71832,7 +68969,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 131100, - "max_tokens": 131100, "mode": "chat", "output_cost_per_token": 3.9998e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71844,7 +68980,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 131100, - "max_tokens": 131100, "mode": "chat", "output_cost_per_token": 3.9998e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71857,7 +68992,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 7.889e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71872,7 +69006,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.9438e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71887,7 +69020,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.9438e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71903,7 +69035,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 200000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.09848e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71917,7 +69048,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71933,7 +69063,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71950,7 +69079,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71967,7 +69095,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71984,7 +69111,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72001,7 +69127,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72017,7 +69142,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 0.00018, "source": "https://aihubmix.com/api/v1/models", @@ -72034,7 +69158,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72051,7 +69174,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72068,7 +69190,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72084,7 +69205,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72129,7 +69249,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72145,7 +69264,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72161,7 +69279,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.248e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72177,7 +69294,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.535e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72193,7 +69309,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.9995e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72209,7 +69324,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.999e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72225,7 +69339,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72241,7 +69354,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.0984e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72254,7 +69366,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72290,7 +69401,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 204800, "max_output_tokens": 204800, - "max_tokens": 204800, "mode": "chat", "output_cost_per_token": 1.1832e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72304,7 +69414,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.152e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72329,7 +69438,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 5.48e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72341,7 +69449,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 9.008e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72356,7 +69463,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 9.864e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72371,7 +69477,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.532e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72386,7 +69491,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.524e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72403,7 +69507,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.608e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72420,7 +69523,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.128e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72437,7 +69539,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72454,7 +69555,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.80025e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72472,7 +69572,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5.07e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72657,7 +69756,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "wandb", "max_input_tokens": 1049000, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.5e-07, "source": "https://wandb.ai/site/pricing/tokens/", @@ -72670,7 +69769,7 @@ "input_cost_per_token": 1e-07, "litellm_provider": "wandb", "max_input_tokens": 262000, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://wandb.ai/site/pricing/tokens/", @@ -72711,7 +69810,6 @@ "token_count_tokenizer": "kimi_k3", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_flex": 8.25e-06, @@ -72743,7 +69841,6 @@ "token_count_tokenizer": "kimi_k3", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_flex": 7.5e-06, @@ -72775,7 +69872,6 @@ "token_count_tokenizer": "kimi_k3", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_flex": 8.25e-06, @@ -72797,7 +69893,6 @@ "litellm_provider": "xiaomi_mimo", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.7e-07, "source": "https://platform.xiaomimimo.com/static/docs/price/pay-as-you-go.md", @@ -72820,7 +69915,6 @@ "litellm_provider": "xiaomi_mimo", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://platform.xiaomimimo.com/static/docs/price/pay-as-you-go.md", @@ -72845,7 +69939,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -72871,7 +69964,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72893,7 +69985,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72915,7 +70006,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72937,7 +70027,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72958,7 +70047,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72980,7 +70068,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73001,7 +70088,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73023,7 +70109,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73045,7 +70130,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73066,7 +70150,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73088,7 +70171,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73110,7 +70192,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73131,7 +70212,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73153,7 +70233,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73178,7 +70257,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73203,7 +70281,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73228,7 +70305,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73249,7 +70325,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73271,7 +70346,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -73289,7 +70363,6 @@ "litellm_provider": "baseten", "max_input_tokens": 262000, "max_output_tokens": 262000, - "max_tokens": 262000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://inference.baseten.co/v1/models", @@ -73307,7 +70380,6 @@ "litellm_provider": "baseten", "max_input_tokens": 262000, "max_output_tokens": 262000, - "max_tokens": 262000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://inference.baseten.co/v1/models", @@ -73324,7 +70396,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://inference.baseten.co/v1/models", @@ -73341,7 +70412,6 @@ "litellm_provider": "baseten", "max_input_tokens": 202800, "max_output_tokens": 202800, - "max_tokens": 202800, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -73359,7 +70429,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4.05e-06, "source": "https://inference.baseten.co/v1/models", @@ -73377,7 +70446,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -73394,7 +70462,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -73411,7 +70478,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inference.baseten.co/v1/models", @@ -73428,7 +70494,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.6e-07, "source": "https://inference.baseten.co/v1/models", @@ -73446,7 +70511,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.48e-06, "source": "https://inference.baseten.co/v1/models", @@ -73463,7 +70527,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.96e-06, "source": "https://inference.baseten.co/v1/models", @@ -73480,7 +70543,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://inference.baseten.co/v1/models", @@ -73500,7 +70562,6 @@ "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, @@ -73521,7 +70582,6 @@ "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, @@ -73542,7 +70602,6 @@ "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, @@ -73563,7 +70622,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.2e-06, @@ -73581,7 +70639,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 196608, "max_output_tokens": 196608, - "max_tokens": 196608, "mode": "chat", "output_cost_per_token_priority": 1.2e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -73647,7 +70704,6 @@ "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -73665,7 +70721,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -73684,7 +70739,6 @@ "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#veo", @@ -73700,7 +70754,6 @@ "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -73717,7 +70770,6 @@ "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.5, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#veo", @@ -73764,7 +70816,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73797,7 +70848,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73830,7 +70880,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73863,7 +70912,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73901,7 +70949,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73934,7 +70981,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73970,7 +71016,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74009,7 +71054,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74049,7 +71093,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74087,7 +71130,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74126,7 +71168,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74159,7 +71200,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "supports_mid_conversation_system": true, @@ -74197,7 +71237,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "thinking_always_on": true, "supports_function_calling": true, @@ -74217,7 +71256,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "thinking_always_on": true, "supports_function_calling": true, @@ -74237,7 +71275,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "thinking_always_on": true, "supports_function_calling": true, @@ -74259,7 +71296,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-r1.html", @@ -74272,7 +71308,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -74285,7 +71320,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.5e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/", @@ -74321,7 +71355,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -74359,7 +71392,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "supports_mid_conversation_system": true, @@ -74405,7 +71437,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -74447,7 +71478,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -74484,7 +71514,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "supports_forced_tool_use": false, @@ -74521,7 +71550,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "supports_forced_tool_use": false, @@ -74562,7 +71590,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -74609,7 +71636,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -74651,7 +71677,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -74783,7 +71808,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "supports_forced_tool_use": false, @@ -74829,7 +71853,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -74874,7 +71897,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -74914,7 +71936,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -74947,7 +71968,6 @@ "litellm_provider": "prism", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 6.3e-07, "source": "https://prisminference.com/pricing", @@ -74972,7 +71992,6 @@ "litellm_provider": "prism", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.1e-07, "source": "https://prisminference.com/pricing", @@ -75005,7 +72024,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 524288, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_flex": 3e-06, @@ -75031,7 +72049,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 524288, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6.6e-06, "output_cost_per_token_flex": 3.3e-06, @@ -75049,7 +72066,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -75104,7 +72120,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -75172,7 +72187,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -75218,7 +72232,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "responses", "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, @@ -75266,7 +72279,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, @@ -75301,7 +72313,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 9e-06, "output_cost_per_token": 9e-06, @@ -75315,7 +72326,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 6e-06, "output_cost_per_token": 6e-06, @@ -75332,7 +72342,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 524288, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -75350,7 +72359,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", @@ -75383,7 +72391,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.28e-06, "output_cost_per_token_batches": 2.64e-06, @@ -75416,7 +72423,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.808e-06, "output_cost_per_token_batches": 2.904e-06, @@ -75439,7 +72445,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2.75e-06, @@ -75454,7 +72459,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/model-router/", @@ -75474,7 +72478,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -75519,7 +72522,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -75565,7 +72567,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -75609,7 +72610,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -75637,7 +72637,6 @@ "token_count_tokenizer": "kimi_k3", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_flex": 8.25e-06, @@ -75670,7 +72669,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, @@ -75724,7 +72722,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -75771,7 +72768,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_forced_tool_use": true, @@ -75816,7 +72812,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "search_context_cost_per_query": { @@ -75859,7 +72854,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "supports_mid_conversation_system": true, @@ -75910,7 +72904,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -75957,7 +72950,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "search_context_cost_per_query": { @@ -75999,7 +72991,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_forced_tool_use": true, @@ -76041,7 +73032,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_forced_tool_use": true, @@ -76087,7 +73077,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -76139,7 +73128,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "search_context_cost_per_query": { @@ -76186,7 +73174,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -76245,7 +73232,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_forced_tool_use": true, @@ -76296,7 +73282,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -76346,7 +73331,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, @@ -76391,7 +73375,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, @@ -76430,7 +73413,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 261120, "max_output_tokens": 98304, - "max_tokens": 98304, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", @@ -76441,7 +73423,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 261120, "max_output_tokens": 98304, - "max_tokens": 98304, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", @@ -76452,7 +73433,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 261120, "max_output_tokens": 98304, - "max_tokens": 98304, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", @@ -76488,7 +73468,6 @@ "input_cost_per_token": 8e-08, "litellm_provider": "azure_ai", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048.0, @@ -76508,7 +73487,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048.0, @@ -76577,7 +73555,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, @@ -76623,7 +73600,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, @@ -76673,7 +73649,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_reasoning_token": 1e-05, "output_cost_per_token": 1e-05, @@ -76732,7 +73707,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_reasoning_token": 1e-05, "output_cost_per_token": 1e-05, @@ -76790,7 +73764,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_reasoning_token": 2e-05, "output_cost_per_token": 2e-05, @@ -76838,7 +73811,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -76851,7 +73823,6 @@ "supports_vision": true }, "openinfer/@oi/GLM-5.3-Flash": { - "max_tokens": 393216, "max_input_tokens": 393216, "max_output_tokens": 393216, "input_cost_per_token": 1.95e-07, @@ -76863,7 +73834,6 @@ "source": "https://api.openinfer.ai/v1/models" }, "openinfer/@oi/Qwen3.5-27B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1.95e-07, @@ -76875,7 +73845,6 @@ "source": "https://api.openinfer.ai/v1/models" }, "openinfer/@oi/Qwen3.5-9B": { - "max_tokens": 100000, "max_input_tokens": 100000, "max_output_tokens": 100000, "input_cost_per_token": 5e-06, @@ -76954,7 +73923,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://api.together.ai/v1/models", @@ -76977,7 +73946,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.311e-06, "output_cost_per_token_batches": 1.6555e-06, @@ -76999,7 +73967,6 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -77023,7 +73990,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-05, "prompt_cache_min_tokens": 512.0, @@ -77053,7 +74019,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.25e-06, "source": "https://inference.baseten.co/v1/models", @@ -77071,7 +74036,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inference.baseten.co/v1/models", @@ -77087,7 +74051,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "wandb", "max_input_tokens": 1049000, - "max_tokens": 943718, + "max_output_tokens": 943718, "mode": "chat", "output_cost_per_token": 7.5e-07, "source": "https://openrouter.ai/api/v1/models/nvidia/nemotron-3.5-super-vl/endpoints", @@ -77103,7 +74067,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", @@ -77117,7 +74081,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -77131,7 +74094,7 @@ "input_cost_per_token": 2.5e-07, "litellm_provider": "nebius", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://tokenfactory.nebius.com/models/catalog/text2text/reflection%2Fbeam", @@ -77144,7 +74107,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -77161,7 +74123,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -77180,7 +74141,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.5e-06, @@ -77200,7 +74160,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -77222,7 +74181,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.25e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -77244,7 +74202,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -77266,7 +74223,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -77288,7 +74244,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -77307,7 +74262,7 @@ "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", @@ -77344,7 +74299,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 9e-07, @@ -77380,7 +74334,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.8e-06, "output_cost_per_token_priority": 2.25e-06, @@ -77429,7 +74382,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -77459,7 +74411,6 @@ "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 5e-07, - "max_tokens": 500000, "max_output_tokens": 500000, "max_input_tokens": 500000, "supports_function_calling": true, @@ -77497,7 +74448,6 @@ "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, "cache_creation_input_token_cost": 1.25e-07, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 922000, "supports_function_calling": true, @@ -77537,7 +74487,6 @@ "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 2e-08, "cache_creation_input_token_cost": 2.5e-07, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 922000, "supports_function_calling": true, @@ -77577,7 +74526,6 @@ "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, "cache_creation_input_token_cost": 1.25e-07, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77616,7 +74564,6 @@ "input_cost_per_token": 2e-06, "output_cost_per_token": 1e-05, "cache_read_input_token_cost": 1e-07, - "max_tokens": 65536, "max_output_tokens": 65536, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77633,7 +74580,6 @@ "input_cost_per_token": 1.5e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 3e-08, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77650,7 +74596,6 @@ "input_cost_per_token": 1.4e-06, "output_cost_per_token": 4.4e-06, "cache_read_input_token_cost": 2.6e-07, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77667,7 +74612,6 @@ "input_cost_per_token": 1.4e-06, "output_cost_per_token": 4.4e-06, "cache_read_input_token_cost": 2.6e-07, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77685,7 +74629,6 @@ "output_cost_per_token": 1e-05, "cache_read_input_token_cost": 2.0000000000000002e-07, "cache_creation_input_token_cost": 2e-06, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77703,7 +74646,6 @@ "output_cost_per_token": 1.5e-05, "cache_read_input_token_cost": 3e-07, "cache_creation_input_token_cost": 3e-06, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77720,7 +74662,6 @@ "input_cost_per_token": 9.499999999999999e-07, "output_cost_per_token": 4e-06, "cache_read_input_token_cost": 1.9e-07, - "max_tokens": 262144, "max_output_tokens": 262144, "max_input_tokens": 262144, "supports_function_calling": true, @@ -77737,7 +74678,6 @@ "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 6e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77754,7 +74694,6 @@ "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 6e-09, - "max_tokens": 384000, "max_output_tokens": 384000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77771,7 +74710,6 @@ "input_cost_per_token": 1.32e-06, "output_cost_per_token": 3.96e-06, "cache_read_input_token_cost": 4.4e-08, - "max_tokens": 384000, "max_output_tokens": 384000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77788,7 +74726,6 @@ "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, "cache_read_input_token_cost": 2.8e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77805,7 +74742,6 @@ "input_cost_per_token": 4.35e-07, "output_cost_per_token": 8.7e-07, "cache_read_input_token_cost": 3.625e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77822,7 +74758,6 @@ "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, "cache_read_input_token_cost": 2.8e-09, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77839,7 +74774,6 @@ "input_cost_per_token": 4.35e-07, "output_cost_per_token": 8.7e-07, "cache_read_input_token_cost": 3.625e-09, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77856,7 +74790,6 @@ "input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07, "cache_read_input_token_cost": 1.5e-08, - "max_tokens": 512000, "max_output_tokens": 512000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77873,7 +74806,6 @@ "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 6e-08, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77890,7 +74822,6 @@ "input_cost_per_token": 1.0000000000000001e-07, "output_cost_per_token": 2.0000000000000002e-07, "cache_read_input_token_cost": 2e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77907,7 +74838,6 @@ "input_cost_per_token": 1.0000000000000001e-07, "output_cost_per_token": 2.0000000000000002e-07, "cache_read_input_token_cost": 2e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77925,7 +74855,6 @@ "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 2.5e-07, "cache_creation_input_token_cost": 2.5e-06, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77943,7 +74872,6 @@ "output_cost_per_token": 4.6999999999999995e-07, "cache_read_input_token_cost": 1.6e-08, "cache_creation_input_token_cost": 2.0000000000000002e-07, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77961,7 +74889,6 @@ "output_cost_per_token": 1.6000000000000001e-06, "cache_read_input_token_cost": 4e-08, "cache_creation_input_token_cost": 5e-07, - "max_tokens": 65536, "max_output_tokens": 65536, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -78000,7 +74927,6 @@ "input_cost_per_token": 8.34e-07, "output_cost_per_token": 2.501e-06, "cache_read_input_token_cost": 4.2000000000000006e-08, - "max_tokens": 64000, "max_output_tokens": 64000, "max_input_tokens": 1024000, "supports_function_calling": true, @@ -78017,7 +74943,6 @@ "input_cost_per_token": 1.4e-07, "output_cost_per_token": 5.8e-07, "cache_read_input_token_cost": 3.5e-08, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 192000, "supports_function_calling": true, @@ -78034,7 +74959,6 @@ "input_cost_per_token": 0, "output_cost_per_token": 0, "cache_read_input_token_cost": 0, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -78051,7 +74975,6 @@ "input_cost_per_token": 1e-06, "output_cost_per_token": 2.7e-06, "cache_read_input_token_cost": 1.0000000000000001e-07, - "max_tokens": 65536, "max_output_tokens": 65536, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -78068,7 +74991,6 @@ "input_cost_per_token": 0, "output_cost_per_token": 0, "cache_read_input_token_cost": 0, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -78089,7 +75011,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -78134,7 +75055,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -78178,7 +75098,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -78211,7 +75130,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, @@ -78261,7 +75179,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78298,7 +75215,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78341,7 +75257,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78385,7 +75300,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78430,7 +75344,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78475,7 +75388,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_batches": 1e-05, @@ -78520,7 +75432,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -78555,7 +75466,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -78598,7 +75508,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -78642,7 +75551,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -78712,7 +75620,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_above_272k_tokens": 7.5e-06, @@ -78761,7 +75668,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -78788,7 +75694,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_above_272k_tokens": 7.5e-06, @@ -78827,7 +75732,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-06, "output_cost_per_token_above_272k_tokens": 8.25e-06, @@ -78884,7 +75788,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 5e-06, "output_cost_per_token_above_272k_tokens": 7.5e-06, diff --git a/litellm/proxy/_experimental/mcp_server/sampling_handler.py b/litellm/proxy/_experimental/mcp_server/sampling_handler.py index e1469f2fff7..d049ada4dfd 100644 --- a/litellm/proxy/_experimental/mcp_server/sampling_handler.py +++ b/litellm/proxy/_experimental/mcp_server/sampling_handler.py @@ -222,7 +222,7 @@ def _select_model_by_priority( input_cost = info.get("input_cost_per_token") or 0.0 output_cost = info.get("output_cost_per_token") or 0.0 total_cost = input_cost + output_cost - max_output = info.get("max_output_tokens") or info.get("max_tokens") or 0 + max_output = info.get("max_output_tokens") or 0 output_tps = info.get("output_tokens_per_second") or 0.0 scored.append( _ScoredModel( diff --git a/litellm/router.py b/litellm/router.py index d5093384a17..548df8e6721 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12264,7 +12264,6 @@ class Router: _fallback_entry = litellm.model_cost.get(_azure_fallback_key) _fallback_resolves = _fallback_entry is not None and ( (_fallback_entry.get("max_input_tokens") or 0) > 0 - or (_fallback_entry.get("max_tokens") or 0) > 0 or (_fallback_entry.get("input_cost_per_token") or 0) > 0 ) if _fallback_resolves: diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index 086f5517c29..2e350a2f87b 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -148,7 +148,7 @@ def _model_max_output_tokens(model_name: str) -> int | None: # this lookup is a fallback default and must never fail the request. with contextlib.suppress(Exception): info: Final = litellm.get_model_info(model=model_name) - model_max: Final = info.get("max_output_tokens") or info.get("max_tokens") + model_max: Final = info.get("max_output_tokens") if model_max is not None: return max(0, int(model_max)) return None diff --git a/litellm/utils.py b/litellm/utils.py index 1cc7c63e6a5..1aed15ba83e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6079,11 +6079,8 @@ def get_max_tokens(model: str) -> int | None: return None try: - if model in litellm.model_cost: - if "max_output_tokens" in litellm.model_cost[model]: - return litellm.model_cost[model]["max_output_tokens"] - elif "max_tokens" in litellm.model_cost[model]: - return litellm.model_cost[model]["max_tokens"] + if model in litellm.model_cost and "max_output_tokens" in litellm.model_cost[model]: + return litellm.model_cost[model]["max_output_tokens"] get_llm_provider: Final = litellm_utils.get_llm_provider model, custom_llm_provider, _, _ = get_llm_provider(model=model) if custom_llm_provider == "huggingface": @@ -6092,8 +6089,6 @@ def get_max_tokens(model: str) -> int | None: if model in litellm.model_cost: # check if extracted model is in model_list if "max_output_tokens" in litellm.model_cost[model]: return litellm.model_cost[model]["max_output_tokens"] - elif "max_tokens" in litellm.model_cost[model]: - return litellm.model_cost[model]["max_tokens"] else: raise Exception() return None @@ -6742,7 +6737,7 @@ def get_model_info_helper( returned_model_info: Final = ModelInfoBase( key=key, - max_tokens=_model_info.get("max_tokens", None), + max_tokens=_model_info.get("max_output_tokens") or _model_info.get("max_input_tokens"), max_input_tokens=_model_info.get("max_input_tokens", None), max_output_tokens=_model_info.get("max_output_tokens", None), input_cost_per_token=_input_cost_per_token, @@ -8242,9 +8237,7 @@ def trim_messages( if max_tokens is None: # Check if model is valid if model in litellm.model_cost: - max_tokens_for_model: Final = litellm.model_cost[model].get( - "max_input_tokens", litellm.model_cost[model]["max_tokens"] - ) + max_tokens_for_model: Final = litellm.model_cost[model]["max_input_tokens"] max_tokens = int(max_tokens_for_model * trim_ratio) else: # if user did not specify max (input) tokens diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d9df9d97d7b..c913c879ccf 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -9,9 +9,8 @@ "input_cost_per_audio_token": 0.0, "input_cost_per_token": 0.0, "litellm_provider": "one of https://docs.litellm.ai/docs/providers", - "max_input_tokens": "max input tokens, if the provider specifies it. if not default to max_tokens", - "max_output_tokens": "max output tokens, if the provider specifies it. if not default to max_tokens", - "max_tokens": "LEGACY parameter. set to max_output_tokens if provider specifies it. IF not set to max_input_tokens, if provider specifies it.", + "max_input_tokens": "max input tokens, if the provider specifies it", + "max_output_tokens": "max output tokens, if the provider specifies it", "mode": "one of: chat, embedding, completion, image_generation, audio_transcription, audio_speech, image_generation, moderation, rerank, search", "output_cost_per_reasoning_token": 0.0, "output_cost_per_token": 0.0, @@ -49,28 +48,24 @@ "1024-x-1024/50-steps/stability.stable-diffusion-xl-v1": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.04 }, "1024-x-1024/max-steps/stability.stable-diffusion-xl-v1": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.08 }, "512-x-512/50-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.018 }, "512-x-512/max-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.036 }, @@ -79,7 +74,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8191, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.25e-05, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -89,7 +83,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8191, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.88e-05, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -100,7 +93,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-large.md" @@ -111,7 +103,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-mini.md" @@ -121,7 +112,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 70000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -298,7 +288,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", @@ -311,7 +300,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md", @@ -324,7 +312,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", @@ -337,7 +324,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md", @@ -352,7 +338,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -368,7 +353,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.4e-07, "output_cost_per_token_batches": 1.2e-07, @@ -389,7 +373,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, @@ -414,7 +397,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -435,7 +417,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.311e-06, "supports_function_calling": true, @@ -455,7 +436,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.1e-05, "source": "https://aws.amazon.com/nova/pricing/", @@ -474,7 +454,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.597e-06, "supports_function_calling": true, @@ -494,7 +473,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.1e-05, "source": "https://aws.amazon.com/nova/pricing/", @@ -518,7 +496,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.75e-06, "output_cost_per_token_batches": 1.375e-06, @@ -544,7 +521,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -562,7 +538,6 @@ "amazon.nova-2-multimodal-embeddings-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 8172, - "max_tokens": 8172, "mode": "embedding", "input_cost_per_token": 1.35e-07, "input_cost_per_image": 6e-05, @@ -583,7 +558,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.4e-07, "output_cost_per_token_batches": 7e-08, @@ -602,7 +576,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.25e-05, "output_cost_per_token_batches": 6.25e-06, @@ -626,7 +599,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.2e-06, "output_cost_per_token_batches": 1.6e-06, @@ -646,7 +618,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2.75e-06, @@ -660,7 +631,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -669,7 +639,6 @@ "input_cost_per_token": 8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128, - "max_tokens": 128, "metadata": { "notes": "'supports_image_input' is a deprecated field. Use 'supports_embedding_image_input' instead." }, @@ -684,7 +653,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536 @@ -693,7 +661,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536 @@ -702,7 +669,6 @@ "input_cost_per_token": 2e-08, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -740,7 +706,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -755,7 +720,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -770,7 +734,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -784,7 +747,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 500, - "max_tokens": 500, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 512, @@ -798,7 +760,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 500, - "max_tokens": 500, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 512, @@ -812,7 +773,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 500, - "max_tokens": 500, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 512, @@ -860,7 +820,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 6e-07 }, @@ -869,7 +828,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 2e-07 }, @@ -878,7 +836,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06 }, @@ -889,7 +846,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, @@ -909,7 +865,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", @@ -941,7 +896,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -964,7 +918,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -982,7 +935,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -1001,7 +953,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -1021,7 +972,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -1039,7 +989,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -1054,7 +1003,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-06, "supports_tool_choice": true @@ -1067,7 +1015,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -1097,7 +1044,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -1125,7 +1071,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1164,7 +1109,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1204,7 +1148,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1244,7 +1187,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1284,7 +1226,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1324,7 +1265,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1364,7 +1304,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1405,7 +1344,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "thinking_always_on": true, "supports_function_calling": true, @@ -1427,7 +1365,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1470,7 +1407,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1513,7 +1449,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1556,7 +1491,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1596,7 +1530,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -1639,7 +1572,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -1678,7 +1610,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -1721,7 +1652,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -1760,7 +1690,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "search_context_cost_per_query": { @@ -1803,7 +1732,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "search_context_cost_per_query": { @@ -1842,7 +1770,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "search_context_cost_per_query": { @@ -1880,7 +1807,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "search_context_cost_per_query": { @@ -1922,7 +1848,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1965,7 +1890,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "search_context_cost_per_query": { @@ -2010,7 +1934,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -2053,7 +1976,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "search_context_cost_per_query": { @@ -2098,7 +2020,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2141,7 +2062,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -2186,7 +2106,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2229,7 +2148,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -2269,7 +2187,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2312,7 +2229,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -2357,7 +2273,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2395,7 +2310,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -2435,7 +2349,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -2480,7 +2393,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -2525,7 +2437,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2570,7 +2481,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2615,7 +2525,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2660,7 +2569,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2703,7 +2611,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2745,7 +2652,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -2790,7 +2696,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -2835,7 +2740,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -2880,7 +2784,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -2925,7 +2828,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -2970,7 +2872,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -3011,7 +2912,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3050,7 +2950,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3089,7 +2988,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3128,7 +3026,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3167,7 +3064,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3206,7 +3102,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3246,7 +3141,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3286,7 +3180,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3319,7 +3212,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05 }, @@ -3328,7 +3220,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -3338,7 +3229,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-07 }, @@ -3347,7 +3237,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -3356,7 +3245,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/codellama-CodeLlama-70b-Instruct-hf" @@ -3366,7 +3254,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/google-gemma-7b-it" @@ -3376,7 +3263,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07 }, @@ -3385,7 +3271,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -3394,7 +3279,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07 }, @@ -3403,7 +3287,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/meta-llama-Meta-Llama-3-70B-Instruct" @@ -3413,7 +3296,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/meta-llama-Meta-Llama-3-8B-Instruct" @@ -3423,7 +3305,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mistral-7B-Instruct-v0.1", @@ -3434,7 +3315,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mixtral-8x22B-Instruct-v0.1", @@ -3445,7 +3325,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mixtral-8x7B-Instruct-v0.1", @@ -3457,7 +3336,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.52e-07, "supports_function_calling": true, @@ -3473,7 +3351,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.48e-07, "supports_function_calling": true, @@ -3489,7 +3366,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.36e-06, "output_cost_per_token_flex": 1.68e-06, @@ -3508,7 +3384,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -3543,7 +3418,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3595,7 +3469,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3628,7 +3501,6 @@ "input_cost_per_token": 1.1e-07, "litellm_provider": "azure", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -3640,7 +3512,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 6e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -3670,7 +3541,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -3685,7 +3555,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -3709,7 +3578,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": true, @@ -3734,7 +3602,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3765,7 +3632,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3797,7 +3663,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3830,7 +3695,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3868,7 +3732,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3899,7 +3762,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3938,7 +3800,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3971,7 +3832,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -3996,7 +3856,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -4030,7 +3889,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": false, @@ -4052,7 +3910,6 @@ "litellm_provider": "azure", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -4087,7 +3944,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, @@ -4105,7 +3961,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -4154,7 +4009,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, @@ -4203,7 +4057,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -4252,7 +4105,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -4304,7 +4156,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -4345,7 +4196,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", @@ -4395,7 +4245,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 6e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", @@ -4439,7 +4288,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -4486,7 +4334,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -4533,7 +4380,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -4581,7 +4427,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -4629,7 +4474,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -4675,7 +4519,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -4717,7 +4560,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "output_cost_per_token_priority": 9e-06, @@ -4759,7 +4601,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "output_cost_per_token_priority": 9e-06, @@ -4801,7 +4642,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_priority": 2.5e-06, @@ -4843,7 +4683,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_priority": 2.5e-06, @@ -4883,7 +4722,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" @@ -4895,7 +4733,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Catalog-name twin of azure_ai/model_router: the flat $0.14 per M input tokens is the router's own fee, the routed model is priced on top of it" @@ -4908,7 +4745,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -4929,7 +4765,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -4949,7 +4784,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -4968,7 +4802,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -4987,7 +4820,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5006,7 +4838,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2.2e-05, "output_cost_per_token": 2.64e-06, @@ -5026,7 +4857,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 0.00022, "output_cost_per_token": 2.2e-05, @@ -5046,7 +4876,6 @@ "litellm_provider": "azure", "max_input_tokens": 16000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 8.8e-05, "output_cost_per_token": 2.2e-05, @@ -5076,7 +4905,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -5116,7 +4944,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, "output_cost_per_token_batches": 1.1e-06, @@ -5155,7 +4982,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "supported_endpoints": [ @@ -5196,7 +5022,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5228,7 +5053,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5261,7 +5085,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-07, "output_cost_per_token_batches": 2.2e-07, @@ -5298,7 +5121,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, "output_cost_per_token_batches": 3.3e-05, @@ -5317,7 +5139,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, "output_cost_per_token_batches": 3.3e-05, @@ -5335,7 +5156,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -5353,7 +5173,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -5371,7 +5190,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -5389,7 +5207,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -5409,7 +5226,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -5426,7 +5242,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -5443,7 +5258,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -5463,7 +5277,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -5484,7 +5297,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "supported_endpoints": [ @@ -5525,7 +5337,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5557,7 +5368,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5587,7 +5397,6 @@ "litellm_provider": "azure", "max_input_tokens": 4097, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -5597,7 +5406,7 @@ "input_cost_per_token": 1.65e-06, "litellm_provider": "azure_text", "max_input_tokens": 4097, - "max_tokens": 4097, + "max_output_tokens": 4097, "mode": "completion", "output_cost_per_token": 2.2e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -5607,7 +5416,6 @@ "litellm_provider": "azure", "max_input_tokens": 4097, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -5619,7 +5427,6 @@ "litellm_provider": "azure", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5631,7 +5438,6 @@ "litellm_provider": "azure", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5642,7 +5448,7 @@ "input_cost_per_token": 1.65e-06, "litellm_provider": "azure_text", "max_input_tokens": 4097, - "max_tokens": 4097, + "max_output_tokens": 4097, "mode": "completion", "output_cost_per_token": 2.2e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -5651,7 +5457,7 @@ "input_cost_per_token": 1.65e-06, "litellm_provider": "azure_text", "max_input_tokens": 4097, - "max_tokens": 4097, + "max_output_tokens": 4097, "mode": "completion", "output_cost_per_token": 2.2e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -5662,7 +5468,6 @@ "litellm_provider": "azure", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -5674,7 +5479,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_function_calling": true, @@ -5687,7 +5491,6 @@ "litellm_provider": "azure", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5699,7 +5502,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_function_calling": true, @@ -5712,7 +5514,6 @@ "litellm_provider": "azure", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.00012, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5724,7 +5525,6 @@ "litellm_provider": "azure", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.00012, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5735,7 +5535,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.3e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/", @@ -5748,7 +5547,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.3e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/", @@ -5762,7 +5560,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.3e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/", @@ -5779,7 +5576,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -5817,7 +5613,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -5855,7 +5650,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, @@ -5893,7 +5687,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, @@ -5929,7 +5722,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -5963,7 +5755,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -5997,7 +5788,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 0.00015, "output_cost_per_token_batches": 7.5e-05, @@ -6017,7 +5807,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -6034,7 +5823,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -6053,7 +5841,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -6073,7 +5860,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -6092,7 +5878,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -6125,7 +5910,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -6157,7 +5941,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -6190,7 +5973,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -6225,7 +6007,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6257,7 +6038,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6290,7 +6070,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -6322,7 +6101,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -6340,7 +6118,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -6358,7 +6135,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 6e-07, @@ -6392,7 +6168,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6414,7 +6189,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -6449,7 +6223,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -6483,7 +6256,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -6518,7 +6290,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -6552,7 +6323,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, @@ -6587,7 +6357,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6622,7 +6391,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6656,7 +6424,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6720,7 +6487,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 0.0002, "output_cost_per_token": 2e-05, @@ -6739,7 +6505,6 @@ "litellm_provider": "azure", "max_input_tokens": 16000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 2e-05, @@ -6793,7 +6558,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "audio_transcription", "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ @@ -6817,7 +6581,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, @@ -6860,7 +6623,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, @@ -6895,7 +6657,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, "output_cost_per_token_priority": 3.6e-06, @@ -6932,7 +6693,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -6971,7 +6731,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -7008,7 +6767,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7044,7 +6802,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -7083,7 +6840,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -7122,7 +6878,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -7158,7 +6913,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -7194,7 +6948,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, @@ -7228,7 +6981,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "supported_endpoints": [ @@ -7269,7 +7021,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7294,7 +7045,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7316,7 +7066,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7337,7 +7086,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7358,7 +7106,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7380,7 +7127,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.54e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7401,7 +7147,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.54e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7422,7 +7167,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7444,7 +7188,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.54e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7465,7 +7208,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.54e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7486,7 +7228,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.6e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7502,7 +7243,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.6e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7518,7 +7258,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7550,7 +7289,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7582,7 +7320,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7618,7 +7355,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_batches": 7e-06, @@ -7658,7 +7394,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_batches": 7e-06, @@ -7696,7 +7431,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7730,7 +7464,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, @@ -7762,7 +7495,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, "output_cost_per_token_batches": 8.4e-05, @@ -7801,7 +7533,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, "output_cost_per_token_batches": 8.4e-05, @@ -7850,7 +7581,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -7906,7 +7636,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -7956,7 +7685,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -8006,7 +7734,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -8062,7 +7789,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -8112,7 +7838,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -8163,7 +7888,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -8212,7 +7936,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -8266,7 +7989,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -8327,7 +8049,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -8391,7 +8112,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -8455,7 +8175,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -8519,7 +8238,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -8583,7 +8301,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, @@ -8647,7 +8364,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, @@ -8699,7 +8415,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -8748,7 +8463,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -8797,7 +8511,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, @@ -8846,7 +8559,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, @@ -8901,7 +8613,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -8954,7 +8665,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -9001,7 +8711,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -9050,7 +8759,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -9099,7 +8807,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, @@ -9148,7 +8855,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -9197,7 +8903,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -9242,7 +8947,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "reasoning_effort_levels": [ @@ -9279,7 +8983,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "reasoning_effort_levels": [ @@ -9325,7 +9028,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, @@ -9380,7 +9082,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, @@ -9436,7 +9137,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, @@ -9492,7 +9192,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, @@ -9542,7 +9241,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "output_cost_per_token_above_272k_tokens": 8.25e-05, @@ -9591,7 +9289,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "output_cost_per_token_above_272k_tokens": 8.25e-07, @@ -9646,7 +9343,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, @@ -9689,7 +9385,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.3e-05, "reasoning_effort_levels": [ @@ -9735,7 +9430,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, @@ -9790,7 +9484,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, @@ -9846,7 +9539,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, @@ -9902,7 +9594,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, @@ -9961,7 +9652,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -10020,7 +9710,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, "output_cost_per_token_batches": 1.65e-05, @@ -10075,7 +9764,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, "output_cost_per_token_batches": 1.65e-05, @@ -10127,7 +9815,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -10187,7 +9874,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_batches": 1.5e-05, @@ -10246,7 +9932,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -10299,7 +9984,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -10353,7 +10037,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -10406,7 +10089,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -10453,7 +10135,6 @@ "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -10497,7 +10178,6 @@ "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -10539,7 +10219,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "search_context_cost_per_query": { @@ -10590,7 +10269,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "search_context_cost_per_query": { @@ -10642,7 +10320,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "search_context_cost_per_query": { @@ -10690,7 +10367,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "search_context_cost_per_query": { @@ -11066,7 +10742,7 @@ "input_cost_per_token": 8e-06, "litellm_provider": "azure", "max_input_tokens": 32000, - "max_tokens": 32000, + "max_output_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_function_calling": true @@ -11075,7 +10751,7 @@ "input_cost_per_token": 8e-06, "litellm_provider": "azure", "max_input_tokens": 32000, - "max_tokens": 32000, + "max_output_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_function_calling": true @@ -11087,7 +10763,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_function_calling": true, @@ -11105,7 +10780,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_batches": 3e-05, @@ -11124,7 +10798,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -11142,7 +10815,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -11160,7 +10832,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "supported_endpoints": [ @@ -11191,7 +10862,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -11223,7 +10893,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 4e-05, "search_context_cost_per_query": { @@ -11261,7 +10930,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supports_prompt_caching": true, @@ -11278,7 +10946,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -11295,7 +10962,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, @@ -11326,7 +10992,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, @@ -11357,7 +11022,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supported_endpoints": [ @@ -11388,7 +11052,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -11433,7 +11096,6 @@ "input_cost_per_token": 1.43e-07, "litellm_provider": "azure", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -11443,7 +11105,6 @@ "input_cost_per_token": 2.2e-08, "litellm_provider": "azure", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -11453,7 +11114,6 @@ "input_cost_per_token": 1.1e-07, "litellm_provider": "azure", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -11504,7 +11164,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -11542,7 +11201,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.76e-06, "output_cost_per_token_batches": 8.8e-07, @@ -11578,7 +11236,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4.4e-07, "output_cost_per_token_batches": 2.2e-07, @@ -11612,7 +11269,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -11633,7 +11289,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -11653,7 +11308,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -11672,7 +11326,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -11691,7 +11344,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -11710,7 +11362,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2.2e-05, "output_cost_per_token": 2.64e-06, @@ -11730,7 +11381,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 0.00022, "output_cost_per_token": 2.2e-05, @@ -11750,7 +11400,6 @@ "litellm_provider": "azure", "max_input_tokens": 16000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 8.8e-05, "output_cost_per_token": 2.2e-05, @@ -11780,7 +11429,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -11820,7 +11468,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, "output_cost_per_token_batches": 1.1e-06, @@ -11858,7 +11505,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-07, "output_cost_per_token_batches": 2.2e-07, @@ -11896,7 +11542,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "supported_endpoints": [ @@ -11937,7 +11582,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -11969,7 +11613,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12002,7 +11645,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, "output_cost_per_token_batches": 3.3e-05, @@ -12021,7 +11663,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, "output_cost_per_token_batches": 3.3e-05, @@ -12039,7 +11680,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12057,7 +11697,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -12090,7 +11729,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -12123,7 +11761,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12141,7 +11778,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12159,7 +11795,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12180,7 +11815,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12204,7 +11838,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "azure_ai", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -12215,7 +11848,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "azure_ai", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -12227,7 +11859,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12245,7 +11876,6 @@ "azure_ai/FLUX-1.1-pro": { "litellm_provider": "azure_ai", "max_input_tokens": 5000, - "max_tokens": 5000, "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659", @@ -12256,7 +11886,6 @@ "azure_ai/FLUX.1-Kontext-pro": { "litellm_provider": "azure_ai", "max_input_tokens": 5000, - "max_tokens": 5000, "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/black-forest-labs/", @@ -12267,7 +11896,6 @@ "azure_ai/flux.2-pro": { "litellm_provider": "azure_ai", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -12279,7 +11907,6 @@ "input_cost_per_pixel": 5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "image_generation", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/black-forest-labs/", "supported_endpoints": [ @@ -12301,7 +11928,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.85e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12316,7 +11942,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.828e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12332,7 +11957,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.52e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12348,7 +11972,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.84e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12363,7 +11986,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.84e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12378,7 +12000,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 7.26e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12393,7 +12014,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.46e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12415,7 +12035,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.3e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12438,7 +12057,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12461,7 +12079,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12484,7 +12101,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ @@ -12513,7 +12129,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12528,7 +12143,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 512000, "max_output_tokens": 512000, - "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12571,7 +12185,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.64e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12637,7 +12250,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12660,7 +12272,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.1e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -12672,7 +12283,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12685,7 +12295,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 10000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://azure.microsoft.com/en-us/blog/introducing-the-llama-4-herd-in-azure-ai-foundry-and-azure-databricks/", @@ -12699,7 +12308,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 8192, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.7e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12711,7 +12319,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.54e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12723,7 +12330,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12736,7 +12342,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12749,7 +12354,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12762,7 +12366,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12775,7 +12378,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12788,7 +12390,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12801,7 +12402,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12814,7 +12414,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12827,7 +12426,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12840,7 +12438,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12854,7 +12451,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12867,7 +12463,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.2e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12881,7 +12476,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12893,7 +12487,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12935,7 +12528,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 8182, - "max_tokens": 8182, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12975,7 +12567,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12989,7 +12580,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" @@ -13001,7 +12591,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" @@ -13012,7 +12601,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-cohere-rerank-4-0-in-microsoft-foundry/4477076" @@ -13023,7 +12611,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-cohere-rerank-4-0-in-microsoft-foundry/4477076" @@ -13033,7 +12620,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13048,7 +12634,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13064,7 +12649,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.56e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13076,7 +12660,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.48e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13092,7 +12675,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 5.1e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13109,7 +12691,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13122,7 +12703,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 3072, @@ -13142,7 +12722,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13159,7 +12738,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -13180,7 +12758,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -13200,7 +12777,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13219,7 +12795,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13236,7 +12811,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, @@ -13250,7 +12824,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, @@ -13264,7 +12837,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13279,7 +12851,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.00971, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" @@ -13290,7 +12861,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 70000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13302,7 +12872,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13319,7 +12888,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13343,7 +12911,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-08, "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview", @@ -13356,7 +12923,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13369,7 +12935,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13381,7 +12946,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", @@ -13393,7 +12957,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13407,7 +12970,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "supports_assistant_prefill": true, @@ -13420,7 +12982,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13432,7 +12993,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13445,7 +13005,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -13459,7 +13018,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -13471,7 +13029,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_tool_choice": true }, @@ -13481,7 +13038,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_tool_choice": true }, @@ -13491,7 +13047,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_tool_choice": true }, @@ -13501,7 +13056,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_tool_choice": true }, @@ -13526,7 +13080,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -13536,7 +13089,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/ap-northeast-1/1-month-commitment/anthropic.claude-v2:1": { @@ -13545,7 +13097,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -13555,7 +13106,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -13565,7 +13115,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/ap-northeast-1/6-month-commitment/anthropic.claude-v2:1": { @@ -13574,7 +13123,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -13583,7 +13131,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 7.55e-06, "supports_tool_choice": true @@ -13593,7 +13140,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -13603,7 +13149,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -13614,7 +13159,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -13634,7 +13178,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13652,7 +13195,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -13672,7 +13214,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, "output_cost_per_token_flex": 1.52e-06, @@ -13686,7 +13227,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -13704,7 +13244,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13723,7 +13262,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, @@ -13737,7 +13275,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, @@ -13757,7 +13294,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.2e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -13768,7 +13304,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -13779,7 +13314,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -13799,7 +13333,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13817,7 +13350,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -13837,7 +13369,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.94e-06, "output_cost_per_token_flex": 1.47e-06, @@ -13851,7 +13382,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -13869,7 +13399,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13887,7 +13416,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -13906,7 +13434,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -13926,7 +13453,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13944,7 +13470,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -13963,7 +13488,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -13981,7 +13505,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13999,7 +13522,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.03e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14010,7 +13532,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.9e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14021,7 +13542,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -14041,7 +13561,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14059,7 +13578,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14078,7 +13596,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -14096,7 +13613,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14106,7 +13622,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-v2:1": { @@ -14115,7 +13630,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14125,7 +13639,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14135,7 +13648,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/eu-central-1/6-month-commitment/anthropic.claude-v2:1": { @@ -14144,7 +13656,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14153,7 +13664,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 8.38e-06, "supports_tool_choice": true @@ -14163,7 +13673,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05 }, @@ -14172,7 +13681,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -14184,7 +13692,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14202,7 +13709,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14221,7 +13727,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14239,7 +13744,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.78e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14250,7 +13754,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14262,7 +13765,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14280,7 +13782,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14299,7 +13800,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14317,7 +13817,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.55e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14328,7 +13827,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14340,7 +13838,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, "output_cost_per_token_flex": 9.3e-07, @@ -14358,7 +13855,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14377,7 +13873,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.01e-06, "output_cost_per_token_flex": 5.05e-07, @@ -14396,7 +13891,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.86e-06, "output_cost_per_token_flex": 9.3e-07, @@ -14414,7 +13908,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", @@ -14425,7 +13918,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.56e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", @@ -14437,7 +13929,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 9.1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", @@ -14450,7 +13941,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14468,7 +13958,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14487,7 +13976,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14504,7 +13992,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Anthropic via Invoke route does not currently support pdf input." }, @@ -14523,7 +14010,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.88e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14534,7 +14020,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.01e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14545,7 +14030,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -14565,7 +14049,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14583,7 +14066,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14603,7 +14085,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, "output_cost_per_token_flex": 1.52e-06, @@ -14617,7 +14098,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -14635,7 +14115,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14653,7 +14132,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14663,7 +14141,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-v2:1": { @@ -14672,7 +14149,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14682,7 +14158,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14692,7 +14167,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/us-east-1/6-month-commitment/anthropic.claude-v2:1": { @@ -14701,7 +14175,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14710,7 +14183,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-06, "supports_tool_choice": true @@ -14720,7 +14192,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -14730,7 +14201,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -14741,7 +14211,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14752,7 +14221,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14763,7 +14231,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", @@ -14774,7 +14241,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", @@ -14786,7 +14252,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", @@ -14798,7 +14263,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_flex": 9.25e-07, @@ -14818,7 +14282,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -14838,7 +14301,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -14855,7 +14317,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, @@ -14869,7 +14330,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, @@ -14887,7 +14347,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -14905,7 +14364,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_flex": 9.25e-07, @@ -14925,7 +14383,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -14943,7 +14400,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14963,7 +14419,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, @@ -14977,7 +14432,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, @@ -14995,7 +14449,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -15013,7 +14466,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.84e-06, "supports_function_calling": true, @@ -15027,7 +14479,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536 @@ -15036,7 +14487,6 @@ "input_cost_per_token": 2e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024 @@ -15046,7 +14496,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.7e-06 }, @@ -15055,7 +14504,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 4e-07 }, @@ -15064,7 +14512,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06 }, @@ -15077,7 +14524,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -15106,7 +14552,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -15127,7 +14572,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.5e-06, "supports_pdf_input": true @@ -15137,7 +14581,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 2.65e-06, "supports_pdf_input": true @@ -15145,7 +14588,6 @@ "bedrock/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 8172, - "max_tokens": 8172, "mode": "embedding", "input_cost_per_token": 1.62e-07, "input_cost_per_image": 7.2e-05, @@ -15164,7 +14606,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.88e-07, "supports_function_calling": true, @@ -15180,7 +14621,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.68e-07, "supports_function_calling": true, @@ -15194,7 +14634,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.84e-06, "supports_function_calling": true, @@ -15208,7 +14647,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536 @@ -15217,7 +14655,6 @@ "input_cost_per_token": 1.1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024 @@ -15227,7 +14664,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.6e-06 }, @@ -15236,7 +14672,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 4e-07 }, @@ -15245,7 +14680,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06 }, @@ -15258,7 +14692,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -15287,7 +14720,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -15308,7 +14740,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.5e-06, "supports_pdf_input": true @@ -15318,7 +14749,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 6e-07, "supports_pdf_input": true @@ -15329,7 +14759,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -15340,7 +14769,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -15351,7 +14779,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -15361,7 +14788,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/us-west-2/1-month-commitment/anthropic.claude-v2:1": { @@ -15370,7 +14796,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -15380,7 +14805,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -15390,7 +14814,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/us-west-2/6-month-commitment/anthropic.claude-v2:1": { @@ -15399,7 +14822,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -15408,7 +14830,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-06, "supports_tool_choice": true @@ -15418,7 +14839,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -15428,7 +14848,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -15439,7 +14858,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", @@ -15450,7 +14868,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", @@ -15462,7 +14879,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", @@ -15474,7 +14890,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_flex": 9.25e-07, @@ -15494,7 +14909,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -15514,7 +14928,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -15531,7 +14944,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, @@ -15545,7 +14957,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, @@ -15563,7 +14974,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -15582,7 +14992,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, @@ -15672,7 +15081,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -15683,7 +15091,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -15694,7 +15101,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": true, @@ -15705,7 +15111,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -15716,7 +15121,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -15727,7 +15131,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.5e-07, "source": "https://www.cerebras.ai/blog/openai-gpt-oss-120b-runs-fastest-on-cerebras", @@ -15742,7 +15145,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://inference-docs.cerebras.ai/support/pricing", @@ -15755,7 +15157,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.49e-06, "source": "https://api.cerebras.ai/public/v1/models/qwen-3.8-27b", @@ -15771,7 +15172,6 @@ "litellm_provider": "nlp_cloud", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 5e-07 }, @@ -15802,7 +15202,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -15829,7 +15228,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -15858,7 +15256,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -15892,7 +15289,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -15928,7 +15324,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -15972,7 +15367,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -16019,7 +15413,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -16063,7 +15456,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -16092,7 +15484,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16126,7 +15517,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16163,7 +15553,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16206,7 +15595,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16249,7 +15637,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16293,7 +15680,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16337,7 +15723,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -16382,7 +15767,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -16428,7 +15812,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_batches": 1e-05, @@ -16474,7 +15857,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16522,7 +15904,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16638,7 +16019,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 3072, "max_output_tokens": 3072, - "max_tokens": 3072, "mode": "chat", "output_cost_per_token": 1.923e-06, "rpm": 300 @@ -16648,7 +16028,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 2048, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.923e-06, "rpm": 300 @@ -16658,7 +16037,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.923e-06, "rpm": 300 @@ -16668,7 +16046,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.923e-06, "rpm": 300 @@ -16678,7 +16055,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-07, "rpm": 300, @@ -16690,7 +16066,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "rpm": 300 @@ -16700,7 +16075,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 80000, "max_output_tokens": 80000, - "max_tokens": 80000, "mode": "chat", "output_cost_per_token": 3.35e-07, "rpm": 300 @@ -16710,7 +16084,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-08, "rpm": 300 @@ -16720,7 +16093,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 15000, "max_output_tokens": 15000, - "max_tokens": 15000, "mode": "chat", "output_cost_per_token": 0.0, "rpm": 300 @@ -16731,7 +16103,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "rpm": 20, @@ -16743,7 +16114,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 80000, "max_output_tokens": 80000, - "max_tokens": 80000, "mode": "chat", "output_cost_per_token": 4.881e-06, "rpm": 300, @@ -16754,7 +16124,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.87e-07, "rpm": 300 @@ -16764,7 +16133,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 60000, "max_output_tokens": 60000, - "max_tokens": 60000, "mode": "chat", "output_cost_per_token": 2.01e-07, "rpm": 300 @@ -16775,7 +16143,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "rpm": 20, @@ -16787,7 +16154,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-07, "rpm": 300, @@ -16799,7 +16165,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "rpm": 300 @@ -16809,7 +16174,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 24000, "max_output_tokens": 24000, - "max_tokens": 24000, "mode": "chat", "output_cost_per_token": 2.253e-06, "rpm": 300, @@ -16820,7 +16184,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1.12e-07, "rpm": 300, @@ -16831,7 +16194,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1e-06, "rpm": 300 @@ -16842,7 +16204,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "rpm": 20, @@ -16854,7 +16215,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.5e-06, "rpm": 300, @@ -16866,7 +16226,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.55e-07, "rpm": 300 @@ -16876,7 +16235,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.35e-07, "rpm": 300, @@ -16888,7 +16246,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 3500, "max_output_tokens": 3500, - "max_tokens": 3500, "mode": "chat", "output_cost_per_token": 0.0, "rpm": 300 @@ -16898,7 +16255,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3e-07, "rpm": 300, @@ -16910,7 +16266,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.55e-07, "rpm": 300, @@ -16921,7 +16276,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.76e-07, "rpm": 300, @@ -16932,7 +16286,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "rpm": 300, @@ -16944,7 +16297,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 8.5e-07, "rpm": 300, @@ -16955,7 +16307,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 24000, "max_output_tokens": 24000, - "max_tokens": 24000, "mode": "chat", "output_cost_per_token": 1e-06, "rpm": 300, @@ -16966,7 +16317,6 @@ "litellm_provider": "codestral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/capabilities/code_generation/", @@ -16978,7 +16328,6 @@ "litellm_provider": "codestral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/capabilities/code_generation/", @@ -16990,7 +16339,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -17001,7 +16349,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -17011,7 +16358,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -17021,7 +16367,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -17031,7 +16376,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -17042,7 +16386,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -17053,7 +16396,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -17064,7 +16406,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "cohere", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -17076,7 +16417,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "cohere", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048, @@ -17087,7 +16427,6 @@ "input_cost_per_token": 8e-08, "litellm_provider": "cohere", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048, @@ -17109,7 +16448,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-cohere-rerank-3-5.md" @@ -17119,7 +16457,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -17131,7 +16468,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "source": "https://docs.cohere.com/docs/models" }, @@ -17140,7 +16476,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.cohere.com/docs/command-a-plus", @@ -17155,7 +16490,6 @@ "litellm_provider": "cohere", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 2e-06 }, @@ -17164,7 +16498,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -17175,7 +16508,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -17186,7 +16518,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.cohere.com/v2/docs/command-r7b", @@ -17199,7 +16530,7 @@ "input_cost_per_token": 1.12e-06, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17222,7 +16553,7 @@ "input_cost_per_token": 1.12e-06, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17245,7 +16576,7 @@ "input_cost_per_token": 1.12e-06, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17268,7 +16599,7 @@ "input_cost_per_token": 1e-08, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17291,7 +16622,7 @@ "input_cost_per_token": 1e-08, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17314,7 +16645,7 @@ "input_cost_per_token": 1e-08, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17337,7 +16668,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, "supported_endpoints": [ @@ -17367,7 +16697,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, "supported_endpoints": [ @@ -17398,7 +16727,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -17420,7 +16748,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -17442,7 +16769,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17458,7 +16784,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17474,7 +16799,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4.8e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17490,7 +16814,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 202745, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17506,7 +16829,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17522,7 +16844,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 229376, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17538,7 +16859,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17550,7 +16870,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17579,7 +16898,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17609,7 +16927,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 30720, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17622,7 +16939,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17635,7 +16951,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17648,7 +16963,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -17662,7 +16976,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -17675,7 +16988,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17706,7 +17018,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17737,7 +17048,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17769,7 +17079,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -17783,7 +17092,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17796,7 +17104,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -17810,7 +17117,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -17823,7 +17129,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17834,7 +17139,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17883,7 +17187,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17928,7 +17231,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17977,7 +17279,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18022,7 +17323,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18059,7 +17359,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18096,7 +17395,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18134,7 +17432,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18146,7 +17443,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18159,7 +17455,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18172,7 +17467,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18186,7 +17480,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18199,7 +17492,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.87e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18212,7 +17504,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 260096, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18250,7 +17541,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18284,7 +17574,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -18298,7 +17587,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18334,7 +17622,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -18352,7 +17639,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://docs.modelstudio.console.alibabacloud.com/en/model-studio/model-pricing", @@ -18371,7 +17657,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://docs.modelstudio.console.alibabacloud.com/en/model-studio/model-pricing", @@ -18390,7 +17675,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 98304, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -18711,7 +17995,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.qwencloud.com/models", @@ -18727,7 +18010,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.qwencloud.com/models", @@ -18743,7 +18025,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4.8e-06, "source": "https://www.qwencloud.com/models", @@ -18759,7 +18040,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 202745, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.qwencloud.com/models", @@ -18775,7 +18055,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.qwencloud.com/models", @@ -18791,7 +18070,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 229376, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.qwencloud.com/models", @@ -18807,7 +18085,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://www.qwencloud.com/models", @@ -18819,7 +18096,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -18848,7 +18124,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -18878,7 +18153,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 30720, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.4e-06, "source": "https://www.qwencloud.com/models", @@ -18891,7 +18165,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.qwencloud.com/models", @@ -18904,7 +18177,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.qwencloud.com/models", @@ -18917,7 +18189,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -18931,7 +18202,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -18944,7 +18214,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -18975,7 +18244,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19006,7 +18274,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19038,7 +18305,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -19052,7 +18318,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://www.qwencloud.com/models", @@ -19065,7 +18330,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -19079,7 +18343,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -19092,7 +18355,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19103,7 +18365,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19152,7 +18413,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19197,7 +18457,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19246,7 +18505,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19291,7 +18549,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19328,7 +18585,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19365,7 +18621,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19403,7 +18658,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.qwencloud.com/models", @@ -19415,7 +18669,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.qwencloud.com/models", @@ -19428,7 +18681,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://www.qwencloud.com/models", @@ -19441,7 +18693,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.qwencloud.com/models", @@ -19455,7 +18706,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://www.qwencloud.com/models", @@ -19468,7 +18718,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.87e-06, "source": "https://www.qwencloud.com/models", @@ -19481,7 +18730,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 260096, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19519,7 +18767,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19553,7 +18800,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://www.qwencloud.com/models", @@ -19567,7 +18813,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19603,7 +18848,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://www.qwencloud.com/models", @@ -19619,7 +18863,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 98304, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://www.qwencloud.com/models", @@ -19940,7 +19183,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -19956,7 +19198,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -19972,7 +19213,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4.8e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -19988,7 +19228,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 202745, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20004,7 +19243,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20020,7 +19258,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 229376, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20036,7 +19273,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20048,7 +19284,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20077,7 +19312,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20107,7 +19341,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 30720, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20120,7 +19353,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20133,7 +19365,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20146,7 +19377,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -20160,7 +19390,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -20173,7 +19402,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20204,7 +19432,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20235,7 +19462,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20267,7 +19493,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -20281,7 +19506,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20294,7 +19518,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -20308,7 +19531,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -20321,7 +19543,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20332,7 +19553,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20381,7 +19601,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20426,7 +19645,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20475,7 +19693,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20520,7 +19737,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20557,7 +19773,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20594,7 +19809,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20632,7 +19846,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20644,7 +19857,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20657,7 +19869,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20670,7 +19881,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20684,7 +19894,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20697,7 +19906,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.87e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20710,7 +19918,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 260096, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20748,7 +19955,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20782,7 +19988,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20796,7 +20001,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20832,7 +20036,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20850,7 +20053,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://docs.modelstudio.console.alibabacloud.com/en/model-studio/model-pricing", @@ -20869,7 +20071,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://docs.modelstudio.console.alibabacloud.com/en/model-studio/model-pricing", @@ -20888,7 +20089,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 98304, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -21210,7 +20410,6 @@ "input_dbu_cost_per_token": 1.429e-06, "litellm_provider": "databricks", "max_input_tokens": 512, - "max_tokens": 512, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21228,7 +20427,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21257,7 +20455,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21287,7 +20484,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21311,7 +20507,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21335,7 +20530,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21359,7 +20553,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21384,7 +20577,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21409,7 +20601,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21436,7 +20627,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21464,7 +20654,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21493,7 +20682,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21528,7 +20716,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21552,7 +20739,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21575,7 +20761,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21599,7 +20784,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21624,7 +20808,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields. Introductory launch rates of 28.571 input / 142.857 output / 35.714 cache write / 2.857 cache read DBU run through 2026-08-31; the standard rates are listed here because entries carry no expiry date." }, @@ -21653,7 +20836,6 @@ "token_count_framing": 97, "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference. Context/max output are the DeepSeek-published model limits (1M context, 384K max output)." }, @@ -21686,7 +20868,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference. Context/max output are the DeepSeek-published model limits (1M context, 384K max output)." }, @@ -21715,7 +20896,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21749,7 +20929,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21784,7 +20963,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21812,7 +20990,6 @@ "litellm_provider": "databricks", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "metadata": { "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." }, @@ -21833,7 +21010,6 @@ "litellm_provider": "databricks", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "metadata": { "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." }, @@ -21858,7 +21034,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21891,7 +21066,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21923,7 +21097,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21939,7 +21112,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." }, @@ -21964,7 +21136,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." }, @@ -21993,7 +21164,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22025,7 +21195,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22056,7 +21225,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22089,7 +21257,6 @@ "litellm_provider": "databricks", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22106,7 +21273,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference." }, @@ -22134,7 +21300,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22165,7 +21330,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22195,7 +21359,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22215,7 +21378,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22235,7 +21397,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22255,7 +21416,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22275,7 +21435,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22304,7 +21463,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22333,7 +21491,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22362,7 +21519,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields. Rates reflect OpenAI's promotional pricing in effect through November 21, 2026; afterwards input, cache and Batch rates are 25% higher and output rates 50% higher." }, @@ -22391,7 +21547,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22420,7 +21575,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22449,7 +21603,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22478,7 +21631,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22507,7 +21659,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22527,7 +21678,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22547,7 +21697,6 @@ "litellm_provider": "databricks", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22564,7 +21713,6 @@ "litellm_provider": "databricks", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22606,7 +21754,6 @@ "input_dbu_cost_per_token": 1.857e-06, "litellm_provider": "databricks", "max_input_tokens": 8192, - "max_tokens": 8192, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22653,7 +21800,6 @@ "token_count_framing": 100, "max_input_tokens": 1000000, "max_output_tokens": 1048576, - "max_tokens": 1048576, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference." }, @@ -22687,7 +21833,6 @@ "litellm_provider": "databricks", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)." }, @@ -22705,7 +21850,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22722,7 +21866,6 @@ "litellm_provider": "databricks", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22740,7 +21883,6 @@ "litellm_provider": "databricks", "max_input_tokens": 262144, "max_output_tokens": 25000, - "max_tokens": 25000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22790,7 +21932,6 @@ "input_dbu_cost_per_token": 2.86e-07, "litellm_provider": "databricks", "max_input_tokens": 32768, - "max_tokens": 32768, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22836,7 +21977,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -23431,7 +22571,6 @@ ] }, "deepinfra/Gryphe/MythoMax-L2-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 4e-07, @@ -23443,7 +22582,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/NousResearch/Hermes-3-Llama-3.1-405B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-06, @@ -23454,7 +22592,6 @@ "supports_function_calling": true }, "deepinfra/NousResearch/Hermes-3-Llama-3.1-70B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 7e-07, @@ -23465,7 +22602,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/QwQ-32B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.5e-07, @@ -23476,7 +22612,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen2.5-72B-Instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 3.6e-07, @@ -23488,7 +22623,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen2.5-7B-Instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 4e-08, @@ -23498,7 +22632,6 @@ "supports_tool_choice": false }, "deepinfra/Qwen/Qwen2.5-VL-32B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -23510,7 +22643,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen3-14B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1.2e-07, @@ -23522,7 +22654,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-235B-A22B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1.8e-07, @@ -23533,7 +22664,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen3-235B-A22B-Instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-08, @@ -23545,7 +22675,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-235B-A22B-Thinking-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 3e-07, @@ -23556,7 +22685,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen3-30B-A3B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1.2e-07, @@ -23568,7 +22696,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-32B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 8e-08, @@ -23580,7 +22707,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 4e-07, @@ -23591,7 +22717,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct-Turbo": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 3e-07, @@ -23605,7 +22730,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Next-80B-A3B-Instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-08, @@ -23617,7 +22741,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Next-80B-A3B-Thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.4e-07, @@ -23628,7 +22751,6 @@ "supports_function_calling": true }, "deepinfra/Sao10K/L3-8B-Lunaris-v1-Turbo": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 4e-08, @@ -23638,7 +22760,6 @@ "supports_tool_choice": false }, "deepinfra/Sao10K/L3.1-70B-Euryale-v2.2": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 8.5e-07, @@ -23649,7 +22770,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Sao10K/L3.3-70B-Euryale-v2.3": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6.5e-07, @@ -23659,7 +22779,6 @@ "supports_tool_choice": false }, "deepinfra/allenai/olmOCR-7B-0725-FP8": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2.7e-07, @@ -23669,7 +22788,6 @@ "supports_tool_choice": false }, "deepinfra/anthropic/claude-3-7-sonnet-latest": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 3.3e-06, @@ -23681,7 +22799,6 @@ "supports_function_calling": true }, "deepinfra/anthropic/claude-4-opus": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 1.65e-05, @@ -23692,7 +22809,6 @@ "supports_function_calling": true }, "deepinfra/anthropic/claude-4-sonnet": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 3.3e-06, @@ -23703,7 +22819,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 7e-07, @@ -23714,7 +22829,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1-0528": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 5e-07, @@ -23726,7 +22840,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1-0528-Turbo": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-06, @@ -23737,7 +22850,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -23747,7 +22859,6 @@ "supports_tool_choice": false }, "deepinfra/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2.7e-07, @@ -23758,7 +22869,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1-Turbo": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1e-06, @@ -23769,7 +22879,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-V3": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 3.2e-07, @@ -23781,7 +22890,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3-0324": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 2.4e-07, @@ -23795,7 +22903,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3.1": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 2.5e-07, @@ -23809,7 +22916,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3.1-Terminus": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 2.7e-07, @@ -23821,7 +22927,6 @@ "supports_function_calling": true }, "deepinfra/google/gemini-2.5-flash": { - "max_tokens": 1000000, "max_input_tokens": 1000000, "max_output_tokens": 1000000, "input_cost_per_token": 3e-07, @@ -23837,7 +22942,6 @@ "supports_audio_input": true }, "deepinfra/google/gemini-2.5-pro": { - "max_tokens": 1000000, "max_input_tokens": 1000000, "max_output_tokens": 1000000, "input_cost_per_token": 1.25e-06, @@ -23852,7 +22956,6 @@ "supports_audio_input": true }, "deepinfra/google/gemma-3-12b-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-08, @@ -23864,7 +22967,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-3-27b-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 8e-08, @@ -23876,7 +22978,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-3-4b-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-08, @@ -23888,7 +22989,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-3.2-11B-Vision-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4.9e-08, @@ -23898,7 +22998,6 @@ "supports_tool_choice": false }, "deepinfra/meta-llama/Llama-3.2-3B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-08, @@ -23909,7 +23008,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Llama-3.3-70B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2.3e-07, @@ -23920,7 +23018,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Llama-3.3-70B-Instruct-Turbo": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -23932,7 +23029,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 2e-07, @@ -23944,7 +23040,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-4-Scout-17B-16E-Instruct": { - "max_tokens": 327680, "max_input_tokens": 327680, "max_output_tokens": 327680, "input_cost_per_token": 1e-07, @@ -23956,7 +23051,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-Guard-3-8B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5.5e-08, @@ -23966,7 +23060,6 @@ "supports_tool_choice": false }, "deepinfra/meta-llama/Llama-Guard-4-12B": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 1.8e-07, @@ -23976,7 +23069,6 @@ "supports_tool_choice": false }, "deepinfra/meta-llama/Meta-Llama-3-8B-Instruct": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 3e-08, @@ -23987,7 +23079,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Meta-Llama-3.1-70B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4e-07, @@ -23998,7 +23089,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4e-07, @@ -24010,7 +23100,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, @@ -24021,7 +23110,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-08, @@ -24033,7 +23121,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/microsoft/WizardLM-2-8x22B": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 4.8e-07, @@ -24043,7 +23130,6 @@ "supports_tool_choice": false }, "deepinfra/microsoft/phi-4": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 7e-08, @@ -24054,7 +23140,6 @@ "supports_function_calling": true }, "deepinfra/mistralai/Mistral-Nemo-Instruct-2407": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.9e-08, @@ -24066,7 +23151,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/mistralai/Mistral-Small-24B-Instruct-2501": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-08, @@ -24077,7 +23161,6 @@ "supports_function_calling": true }, "deepinfra/mistralai/Mistral-Small-3.2-24B-Instruct-2506": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 7.5e-08, @@ -24088,7 +23171,6 @@ "supports_function_calling": true }, "deepinfra/mistralai/Mixtral-8x7B-Instruct-v0.1": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 4e-07, @@ -24099,7 +23181,6 @@ "supports_function_calling": true }, "deepinfra/moonshotai/Kimi-K2-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-07, @@ -24110,7 +23191,6 @@ "supports_function_calling": true }, "deepinfra/moonshotai/Kimi-K2-Instruct-0905": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 5e-07, @@ -24122,7 +23202,6 @@ "supports_function_calling": true }, "deepinfra/nvidia/Llama-3.1-Nemotron-70B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6e-07, @@ -24133,7 +23212,6 @@ "supports_function_calling": true }, "deepinfra/nvidia/Llama-3.3-Nemotron-Super-49B-v1.5": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -24157,7 +23235,6 @@ "supports_prompt_caching": true }, "deepinfra/nvidia/NVIDIA-Nemotron-Nano-9B-v2": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4e-08, @@ -24168,7 +23245,6 @@ "supports_function_calling": true }, "deepinfra/openai/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3.7e-08, @@ -24180,7 +23256,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/openai/gpt-oss-20b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, @@ -24192,7 +23267,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.5": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4e-07, @@ -24211,7 +23285,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -24234,7 +23307,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-07, "supports_assistant_prefill": true, @@ -24249,7 +23321,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.19e-06, "supports_assistant_prefill": true, @@ -24266,7 +23337,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -24291,7 +23361,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.1e-06, "supports_assistant_prefill": true, @@ -24306,7 +23375,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 4e-07, "supports_assistant_prefill": true, @@ -24321,7 +23389,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-1.html", @@ -24338,7 +23405,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 164000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_batches": 9.25e-07, @@ -24358,7 +23424,6 @@ "litellm_provider": "nlp_cloud", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "completion", "output_cost_per_token": 5e-07 }, @@ -24367,7 +23432,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 98304, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "supports_assistant_prefill": true, @@ -24381,7 +23445,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "supports_assistant_prefill": true, @@ -24395,7 +23458,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 229376, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "supports_assistant_prefill": true, @@ -24408,7 +23470,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - standard version with 2560 dimensions" }, @@ -24420,7 +23481,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - large version with 2048 dimensions" }, @@ -24432,7 +23492,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - text-240915 version with 4096 dimensions" }, @@ -24444,7 +23503,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - text-250515 version with 2048 dimensions" }, @@ -24456,7 +23514,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - text-240715 version with 2560 dimensions" }, @@ -24569,7 +23626,6 @@ "mode": "search" }, "sail/moonshotai/Kimi-K3": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 2.5e-06, @@ -24591,7 +23647,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/zai-org/GLM-5.3": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 9.8e-07, @@ -24613,7 +23668,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/zai-org/GLM-5.3-Flash": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1.1e-07, @@ -24635,7 +23689,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/deepseek-ai/DeepSeek-V4-Pro-0813": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 9.2e-07, @@ -24657,7 +23710,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 9e-08, @@ -24679,7 +23731,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/deepseek-ai/DeepSeek-V4.1-Flash": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 9e-08, @@ -24701,7 +23752,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/moonshotai/Kimi-K2.6": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1e-06, @@ -24724,7 +23774,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/google/gemma-4-31B-it": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 4e-07, @@ -24747,7 +23796,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/nvidia/Gemma-4-31B-IT-NVFP4": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.4e-07, @@ -24770,7 +23818,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/google/gemma-4-12B-it": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 3e-07, @@ -24792,7 +23839,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/openai/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6e-08, @@ -24808,7 +23854,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/Qwen/Qwen3.6-35B-A3B": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 5e-08, @@ -25047,7 +24092,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "cohere", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -25056,7 +24100,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "cohere", "max_input_tokens": 512, - "max_tokens": 512, "metadata": { "notes": "'supports_image_input' is a deprecated field. Use 'supports_embedding_image_input' instead." }, @@ -25069,7 +24112,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "cohere", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "supports_embedding_image_input": true @@ -25078,7 +24120,6 @@ "input_cost_per_token": 0.0001, "litellm_provider": "cohere", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "supports_embedding_image_input": true @@ -25089,7 +24130,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.12e-07, "supports_function_calling": true, @@ -25105,7 +24145,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.84e-07, "supports_function_calling": true, @@ -25121,7 +24160,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 4.2e-06, "output_cost_per_token_flex": 2.1e-06, @@ -25138,7 +24176,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, @@ -25161,7 +24198,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -25189,7 +24225,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -25205,7 +24240,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -25224,7 +24258,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -25244,7 +24277,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -25261,7 +24293,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -25292,7 +24323,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -25323,7 +24353,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -25363,7 +24392,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -25397,7 +24425,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.3e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", @@ -25410,7 +24437,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.9e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", @@ -25422,7 +24448,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-pixtral-large.md", @@ -27688,14 +26713,12 @@ "litellm_provider": "featherless_ai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat" }, "featherless_ai/featherless-ai/Qwerky-QwQ-32B": { "litellm_provider": "featherless_ai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat" }, "fireworks-ai-4.1b-to-16b": { @@ -27743,7 +26766,6 @@ "input_cost_per_token": 1.6e-08, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -27753,7 +26775,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://fireworks.ai/pricing", @@ -27766,7 +26787,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 20480, - "max_tokens": 20480, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://fireworks.ai/pricing", @@ -27778,7 +26798,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 160000, - "max_tokens": 160000, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://fireworks.ai/pricing", @@ -27790,7 +26809,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 20480, - "max_tokens": 20480, "mode": "chat", "output_cost_per_token": 2.19e-06, "source": "https://fireworks.ai/pricing", @@ -27802,7 +26820,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -27814,7 +26831,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/models/fireworks/deepseek-v3-0324", @@ -27826,7 +26842,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://fireworks.ai/pricing", @@ -27839,7 +26854,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://fireworks.ai/pricing", @@ -27852,7 +26866,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://fireworks.ai/models/fireworks/deepseek-v3p2", @@ -27867,7 +26880,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -27886,7 +26898,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.96e-06, "output_cost_per_token_priority": 4.95e-06, @@ -27906,7 +26917,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.96e-06, "output_cost_per_token_priority": 4.95e-06, @@ -27922,7 +26932,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -27935,7 +26944,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 96000, - "max_tokens": 96000, "mode": "chat", "output_cost_per_token": 2.19e-06, "source": "https://fireworks.ai/models/fireworks/glm-4p5", @@ -27949,7 +26957,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 96000, - "max_tokens": 96000, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://artificialanalysis.ai/models/glm-4-5-air", @@ -27964,7 +26971,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 202800, - "max_tokens": 202800, "mode": "chat", "source": "https://fireworks.ai/pricing", "supports_function_calling": true, @@ -27978,7 +26984,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 202800, - "max_tokens": 202800, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://fireworks.ai/models/fireworks/glm-4p7", @@ -27993,7 +26998,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28012,7 +27016,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_priority": 5.5e-06, @@ -28032,7 +27035,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 7.2e-07, @@ -28049,7 +27051,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28064,7 +27065,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://fireworks.ai/models/fireworks/kimi-k2-instruct", @@ -28077,7 +27077,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://app.fireworks.ai/models/fireworks/kimi-k2-instruct-0905", @@ -28090,7 +27089,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://fireworks.ai/pricing", @@ -28105,7 +27103,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -28122,7 +27119,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "output_cost_per_token_priority": 6e-06, @@ -28142,7 +27138,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "output_cost_per_token_priority": 6e-06, @@ -28158,7 +27153,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -28171,7 +27165,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://fireworks.ai/pricing", @@ -28184,7 +27177,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://fireworks.ai/pricing", @@ -28198,7 +27190,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://fireworks.ai/pricing", @@ -28211,7 +27202,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://fireworks.ai/pricing", @@ -28224,7 +27214,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -28237,7 +27226,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://fireworks.ai/pricing", @@ -28249,7 +27237,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://fireworks.ai/pricing", @@ -28262,7 +27249,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 196608, "max_output_tokens": 204800, - "max_tokens": 204800, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://fireworks.ai/models/fireworks/minimax-m2p1", @@ -28278,7 +27264,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 512000, "max_output_tokens": 512000, - "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.8e-06, @@ -28294,7 +27279,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://fireworks.ai/pricing", @@ -28307,7 +27291,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -28320,7 +27303,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -28333,7 +27315,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -28347,7 +27328,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28363,7 +27343,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 202800, - "max_tokens": 202800, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://fireworks.ai/models/fireworks/glm-4p7", @@ -28378,7 +27357,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28394,7 +27372,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28413,7 +27390,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_priority": 5.5e-06, @@ -28433,7 +27409,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 7.2e-07, @@ -28450,7 +27425,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28466,7 +27440,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -28483,7 +27456,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "output_cost_per_token_priority": 6e-06, @@ -28501,7 +27473,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28520,7 +27491,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "output_cost_per_token_priority": 6e-06, @@ -28538,7 +27508,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28554,7 +27523,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 196608, "max_output_tokens": 204800, - "max_tokens": 204800, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://fireworks.ai/models/fireworks/minimax-m2p1", @@ -28570,7 +27538,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 512000, "max_output_tokens": 512000, - "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.8e-06, @@ -28587,7 +27554,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -28601,7 +27567,6 @@ "input_cost_per_token": 8e-09, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -28610,7 +27575,6 @@ "input_cost_per_token": 8e-09, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -28619,7 +27583,6 @@ "input_cost_per_token": 8e-09, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -28628,7 +27591,6 @@ "input_cost_per_token": 1.6e-08, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -28637,7 +27599,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 3e-08, @@ -28665,7 +27626,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "input_cost_per_token": 1.26e-06, "output_cost_per_token": 3.96e-06, "cache_read_input_token_cost": 2.34e-07, @@ -28693,7 +27653,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 4e-07, "supports_prompt_caching": false, @@ -28716,7 +27675,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "input_cost_per_token": 1.4e-06, "output_cost_per_token": 4.4e-06, "cache_read_input_token_cost": 2.6e-07, @@ -28743,7 +27701,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "input_cost_per_token": 5e-07, "output_cost_per_token": 1.5e-06, "cache_read_input_token_cost": 2.5e-07, @@ -28767,7 +27724,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 196608, "max_output_tokens": 196608, - "max_tokens": 196608, "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 6e-08, @@ -28791,7 +27747,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 202752, "max_output_tokens": 202752, - "max_tokens": 202752, "input_cost_per_token": 1.4e-06, "output_cost_per_token": 4.4e-06, "cache_read_input_token_cost": 2.6e-07, @@ -28818,7 +27773,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 9e-07, @@ -28831,7 +27785,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, @@ -28844,7 +27797,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_batches": 3e-06, @@ -28858,7 +27810,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "supports_system_messages": true, @@ -28870,7 +27821,6 @@ "litellm_provider": "openai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "supports_system_messages": true, @@ -28882,7 +27832,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "supports_system_messages": true, @@ -28894,7 +27843,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "source": "OpenAI needs to add pricing for this ft model, will be updated when added by OpenAI. Defaulting to base model pricing", @@ -28911,7 +27859,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -28932,7 +27879,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -28952,7 +27898,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_batches": 6e-07, @@ -28974,7 +27919,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, @@ -28995,7 +27939,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.2e-06, "output_cost_per_token_batches": 1.6e-06, @@ -29016,7 +27959,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, "output_cost_per_token_batches": 4e-07, @@ -29037,7 +27979,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 1.6e-05, "output_cost_per_token_batches": 8e-06, @@ -29060,7 +28001,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -29117,7 +28057,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, @@ -29175,7 +28114,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -29229,7 +28167,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -29281,7 +28218,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, @@ -29327,7 +28263,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, @@ -29376,7 +28311,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -29416,7 +28350,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -29475,7 +28408,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -29535,7 +28467,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -29591,7 +28522,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -29629,7 +28559,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, @@ -29684,7 +28613,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, @@ -29731,7 +28659,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -29778,7 +28705,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -29824,7 +28750,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -29868,7 +28793,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -29920,7 +28844,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -29982,7 +28905,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_image": 0.00012, "output_cost_per_token": 1.2e-05, @@ -30035,7 +28957,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -30088,7 +29009,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -30149,7 +29069,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, @@ -30196,7 +29115,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -30259,7 +29177,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -30320,7 +29237,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -30384,7 +29300,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -30440,7 +29355,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 7.5e-06, "output_cost_per_token": 7.5e-06, @@ -30488,7 +29402,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -30531,7 +29444,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_image": 0.00012, "output_cost_per_token": 1.2e-05, @@ -30584,7 +29496,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -30641,7 +29552,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, @@ -30674,7 +29584,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_reasoning_token": 2.5e-06, @@ -30712,7 +29621,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, @@ -30759,7 +29667,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -30783,7 +29690,6 @@ "input_cost_per_token_batches": 1.2e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30800,7 +29706,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30818,7 +29723,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30837,7 +29741,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30856,7 +29759,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30870,7 +29772,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30882,7 +29783,6 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "gemini", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30901,7 +29801,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "gemini", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30922,7 +29821,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -30968,7 +29866,6 @@ "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "image_generation", "output_cost_per_image": 0.045, "output_cost_per_image_token": 6e-05, @@ -31020,7 +29917,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -31080,7 +29976,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "gemini", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -31097,7 +29992,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "rpm": 1000, @@ -31134,7 +30028,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "rpm": 1000, @@ -31180,7 +30073,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -31243,7 +30135,6 @@ "supports_reasoning": false, "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, @@ -31295,7 +30186,6 @@ "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -31344,7 +30234,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -31387,7 +30276,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.045, "output_cost_per_image_token": 6e-05, @@ -31429,7 +30317,6 @@ "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -31468,7 +30355,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, @@ -31513,7 +30399,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, @@ -31570,7 +30455,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -31629,7 +30513,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -31688,7 +30571,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, @@ -31718,7 +30600,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -31772,7 +30653,6 @@ "litellm_provider": "gemini", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -31815,7 +30695,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -31877,7 +30756,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -31940,7 +30818,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, @@ -32002,7 +30879,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -32067,7 +30943,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32129,7 +31004,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32192,7 +31066,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32250,7 +31123,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -32288,7 +31160,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -32352,7 +31223,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -32421,7 +31291,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, @@ -32466,7 +31335,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -32501,7 +31369,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -32564,7 +31431,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32625,7 +31491,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32689,7 +31554,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32745,7 +31609,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 7.5e-06, "output_cost_per_token": 7.5e-06, @@ -32792,7 +31655,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, @@ -32827,7 +31689,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "metadata": { "notes": "Rate limits not documented for gemini-exp-1114. Assuming same as gemini-1.5-pro.", "supports_tool_choice": true @@ -32850,7 +31711,6 @@ "litellm_provider": "gemini", "max_input_tokens": 2097152, "max_output_tokens": 8192, - "max_tokens": 8192, "metadata": { "notes": "Rate limits not documented for gemini-exp-1206. Assuming same as gemini-1.5-pro.", "supports_tool_choice": true @@ -32871,7 +31731,7 @@ "input_cost_per_token": 3.5e-07, "litellm_provider": "gemini", "max_output_tokens": 8192, - "max_tokens": 8192, + "max_input_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.05e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", @@ -32885,7 +31745,7 @@ "input_cost_per_token": 3.5e-07, "litellm_provider": "gemini", "max_output_tokens": 8192, - "max_tokens": 8192, + "max_input_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.05e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", @@ -32909,7 +31769,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_character": 0, "output_cost_per_character_above_128k_tokens": 0, @@ -32929,7 +31788,6 @@ "litellm_provider": "gemini", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, @@ -32945,7 +31803,6 @@ "litellm_provider": "gemini", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, @@ -32981,7 +31838,6 @@ "litellm_provider": "gemini", "max_input_tokens": 32767, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_character": 0, "output_cost_per_character_above_128k_tokens": 0, @@ -33000,7 +31856,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_image": 0.04, "output_cost_per_token": 0, @@ -33025,7 +31880,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -33049,7 +31903,6 @@ "deprecation_date": "2026-10-22", "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -33066,7 +31919,6 @@ "deprecation_date": "2026-10-22", "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, @@ -33082,7 +31934,6 @@ "deprecation_date": "2026-10-22", "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.05, "output_cost_per_second_1080p": 0.08, @@ -33097,7 +31948,6 @@ "gemini/veo-3.1-fast-generate-001": { "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -33113,7 +31963,6 @@ "gemini/veo-3.1-generate-001": { "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, @@ -33132,7 +31981,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33155,7 +32003,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33178,7 +32025,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 5e-06, "supported_endpoints": [ @@ -33199,7 +32045,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_100k_tokens": 2.5e-06, @@ -33224,7 +32069,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33247,7 +32091,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33270,7 +32113,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33292,7 +32134,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33315,7 +32156,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33337,7 +32177,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33360,7 +32199,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.75e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33379,7 +32217,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.75e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33396,7 +32233,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true }, @@ -33404,7 +32240,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true }, @@ -33412,7 +32247,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true }, @@ -33420,7 +32254,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true }, @@ -33428,7 +32261,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -33438,7 +32270,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "supports_function_calling": true, @@ -33451,7 +32282,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "supports_function_calling": true, @@ -33467,7 +32297,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -33477,7 +32306,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -33487,7 +32315,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -33496,7 +32323,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -33506,7 +32332,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -33515,7 +32340,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -33527,7 +32351,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://github.blog/changelog/2026-10-19-github-copilot-models-deprecated-on-october-19-2026", @@ -33542,7 +32365,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "supported_endpoints": [ @@ -33563,7 +32385,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -33585,7 +32406,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 4.5e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33607,7 +32427,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -33631,7 +32450,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_200k_tokens": 1.8e-06, @@ -33655,7 +32473,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -33680,7 +32497,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -33704,7 +32520,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -33728,7 +32543,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, @@ -33752,7 +32566,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -33776,7 +32589,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -33796,7 +32608,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 917504, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33814,7 +32625,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.2e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33829,19 +32639,16 @@ "github_copilot/text-embedding-3-small": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding" }, "github_copilot/text-embedding-3-small-inference": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding" }, "github_copilot/text-embedding-ada-002": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding" }, "chatgpt/gpt-6-sol": { @@ -33853,7 +32660,6 @@ ], "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_max_reasoning_effort": true, @@ -33874,7 +32680,6 @@ ], "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_max_reasoning_effort": true, @@ -33895,7 +32700,6 @@ ], "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_max_reasoning_effort": true, @@ -33916,7 +32720,6 @@ ], "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_max_reasoning_effort": true, @@ -33933,7 +32736,6 @@ "source": "https://platform.openai.com/docs/models/gpt-5.5", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -33953,7 +32755,6 @@ "source": "https://platform.openai.com/docs/models/gpt-5.6-luna", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -33973,7 +32774,6 @@ "source": "https://platform.openai.com/docs/models/gpt-5.6-sol", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -33993,7 +32793,6 @@ "source": "https://platform.openai.com/docs/models/gpt-5.6-terra", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34012,7 +32811,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34032,7 +32830,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34050,7 +32847,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34068,7 +32864,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34082,7 +32877,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34097,7 +32891,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34113,7 +32906,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34128,7 +32920,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34148,7 +32939,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34163,7 +32953,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34179,7 +32968,6 @@ "litellm_provider": "gigachat", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true, @@ -34190,7 +32978,6 @@ "litellm_provider": "gigachat", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true, @@ -34202,7 +32989,6 @@ "litellm_provider": "gigachat", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true, @@ -34213,7 +32999,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024 @@ -34222,7 +33007,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024 @@ -34231,7 +33015,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 4096, - "max_tokens": 4096, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2560 @@ -34240,7 +33023,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 4096, - "max_tokens": 4096, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048 @@ -34250,7 +33032,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_function_calling": true, @@ -34262,7 +33043,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -34273,7 +33053,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -34284,7 +33063,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -34295,7 +33073,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.4e-05, "supports_function_calling": true @@ -34305,7 +33082,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true @@ -34315,7 +33091,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true @@ -34325,7 +33100,6 @@ "litellm_provider": "gmi", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -34336,7 +33110,6 @@ "litellm_provider": "gmi", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -34347,7 +33120,6 @@ "litellm_provider": "gmi", "max_input_tokens": 163840, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-07, "supports_function_calling": true @@ -34357,7 +33129,6 @@ "litellm_provider": "gmi", "max_input_tokens": 163840, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 8.8e-07, "supports_function_calling": true @@ -34367,7 +33138,6 @@ "litellm_provider": "gmi", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -34379,7 +33149,6 @@ "litellm_provider": "gmi", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06 }, @@ -34388,7 +33157,6 @@ "litellm_provider": "gmi", "max_input_tokens": 196608, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06 }, @@ -34420,7 +33188,6 @@ "litellm_provider": "baseten", "max_input_tokens": 200000, "max_output_tokens": 200000, - "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -34465,7 +33232,6 @@ "litellm_provider": "baseten", "max_input_tokens": 128072, "max_output_tokens": 128072, - "max_tokens": 128072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inference.baseten.co/v1/models", @@ -34495,7 +33261,6 @@ "litellm_provider": "gmi", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-06, "supports_vision": true @@ -34505,7 +33270,6 @@ "litellm_provider": "gmi", "max_input_tokens": 202752, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2e-06 }, @@ -34518,7 +33282,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.9e-07, "output_cost_per_token_batches": 1.5e-07, @@ -34540,7 +33303,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 3.8e-07, "output_cost_per_token_batches": 1.9e-07, @@ -34562,7 +33324,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 8e-08, "output_cost_per_token_batches": 4e-08, @@ -34594,7 +33355,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -34634,7 +33394,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -34668,7 +33427,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -34700,7 +33458,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, @@ -34721,7 +33478,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://developers.openai.com/api/docs/pricing", @@ -34737,7 +33493,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, @@ -34755,7 +33510,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 2e-06, @@ -34772,7 +33526,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-06, "supports_prompt_caching": true, @@ -34785,7 +33538,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 2e-06, "source": "https://developers.openai.com/api/docs/pricing" @@ -34795,7 +33547,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 8192, "max_output_tokens": 4097, - "max_tokens": 4097, "mode": "completion", "output_cost_per_token": 2e-06 }, @@ -34805,7 +33556,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "supports_function_calling": true, @@ -34820,7 +33570,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_batches": 3e-05, @@ -34836,7 +33585,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_function_calling": true, @@ -34851,7 +33599,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_function_calling": true, @@ -34869,7 +33616,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_batches": 1.5e-05, @@ -34891,7 +33637,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -34935,7 +33680,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_priority": 1.4e-05, @@ -34978,7 +33722,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, @@ -35022,7 +33765,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_priority": 2.8e-06, @@ -35066,7 +33808,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -35104,7 +33845,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_priority": 8e-07, @@ -35141,7 +33881,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -35164,7 +33903,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -35188,7 +33926,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 1.7e-05, @@ -35213,7 +33950,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 1.7e-05, @@ -35235,7 +33971,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -35253,7 +33988,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -35271,7 +34005,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -35308,7 +34041,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -35343,7 +34075,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -35381,7 +34112,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -35419,7 +34149,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -35456,7 +34185,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -35497,7 +34225,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -35527,7 +34254,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 1e-06, @@ -35554,7 +34280,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 6e-07, @@ -35574,7 +34299,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -35592,7 +34316,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -35619,7 +34342,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -35700,7 +34422,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -35727,7 +34448,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -36281,7 +35001,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -36327,7 +35046,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5-codex", @@ -36362,7 +35080,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, @@ -36412,7 +35129,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex-mini", @@ -36443,7 +35159,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex-max", @@ -36475,7 +35190,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex", @@ -36507,7 +35221,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-chat-latest", @@ -36542,7 +35255,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, @@ -36594,7 +35306,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, @@ -36645,7 +35356,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.2-codex", @@ -36677,7 +35387,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.2-chat-latest", @@ -36712,7 +35421,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, @@ -36762,7 +35470,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, "output_cost_per_token_batches": 8.4e-05, @@ -36804,7 +35511,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, "output_cost_per_token_batches": 8.4e-05, @@ -36884,7 +35590,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -36971,7 +35676,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -37062,7 +35766,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_decisions": 0.0, @@ -37148,7 +35851,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -37230,7 +35932,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -37314,7 +36015,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -37397,7 +36097,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, @@ -37455,7 +36154,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-05, "output_cost_per_token_above_272k_tokens": 0.0001125, @@ -37494,7 +36192,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-05, "output_cost_per_token_above_272k_tokens": 0.0001125, @@ -37533,7 +36230,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 7.5e-05, "output_cost_per_token_above_272k_tokens": 0.0001125, @@ -37571,7 +36267,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -37610,7 +36305,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -37644,7 +36338,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "reasoning_effort_levels": [ @@ -37690,7 +36383,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -37753,7 +36445,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -37809,7 +36500,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -37861,7 +36551,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -37920,7 +36609,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -37978,7 +36666,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -38029,7 +36716,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -38082,7 +36768,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -38138,7 +36823,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "output_cost_per_token_flex": 2.25e-06, @@ -38193,7 +36877,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "output_cost_per_token_flex": 2.25e-06, @@ -38246,7 +36929,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, @@ -38298,7 +36980,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, @@ -38345,7 +37026,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 272000, - "max_tokens": 272000, "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, @@ -38388,7 +37068,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 272000, - "max_tokens": 272000, "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, @@ -38437,7 +37116,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -38483,7 +37161,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supported_endpoints": [ @@ -38519,7 +37196,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5-chat-latest", @@ -38552,7 +37228,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, @@ -38596,7 +37271,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.3-chat-latest", @@ -38633,7 +37307,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -38685,7 +37358,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -38735,7 +37407,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -38785,7 +37456,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -38871,7 +37541,6 @@ "litellm_provider": "openai", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -38906,7 +37575,6 @@ "litellm_provider": "openai", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -38941,7 +37609,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, @@ -38976,7 +37643,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, @@ -39013,7 +37679,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -39051,7 +37716,6 @@ "litellm_provider": "openai", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -39087,7 +37751,6 @@ "litellm_provider": "openai", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -39113,7 +37776,6 @@ }, "gradient_ai/alibaba-qwen3-32b": { "litellm_provider": "gradient_ai", - "max_tokens": 40960, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" @@ -39128,7 +37790,6 @@ "gradient_ai/anthropic-claude-3-opus": { "input_cost_per_token": 1.5e-05, "litellm_provider": "gradient_ai", - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 7.5e-05, "supported_endpoints": [ @@ -39144,7 +37805,6 @@ "gradient_ai/anthropic-claude-3.5-haiku": { "input_cost_per_token": 8e-07, "litellm_provider": "gradient_ai", - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 4e-06, "supported_endpoints": [ @@ -39160,7 +37820,6 @@ "gradient_ai/anthropic-claude-3.5-sonnet": { "input_cost_per_token": 3e-06, "litellm_provider": "gradient_ai", - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.5e-05, "supported_endpoints": [ @@ -39176,7 +37835,6 @@ "gradient_ai/anthropic-claude-3.7-sonnet": { "input_cost_per_token": 3e-06, "litellm_provider": "gradient_ai", - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.5e-05, "supported_endpoints": [ @@ -39192,7 +37850,6 @@ "gradient_ai/deepseek-r1-distill-llama-70b": { "input_cost_per_token": 9.9e-07, "litellm_provider": "gradient_ai", - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 9.9e-07, "supported_endpoints": [ @@ -39208,7 +37865,6 @@ "gradient_ai/llama3-8b-instruct": { "input_cost_per_token": 2e-07, "litellm_provider": "gradient_ai", - "max_tokens": 512, "mode": "chat", "output_cost_per_token": 2e-07, "supported_endpoints": [ @@ -39224,7 +37880,6 @@ "gradient_ai/llama3.3-70b-instruct": { "input_cost_per_token": 6.5e-07, "litellm_provider": "gradient_ai", - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 6.5e-07, "supported_endpoints": [ @@ -39240,7 +37895,6 @@ "gradient_ai/mistral-nemo-instruct-2407": { "input_cost_per_token": 3e-07, "litellm_provider": "gradient_ai", - "max_tokens": 512, "mode": "chat", "output_cost_per_token": 3e-07, "supported_endpoints": [ @@ -39255,7 +37909,6 @@ }, "gradient_ai/openai-gpt-4o": { "litellm_provider": "gradient_ai", - "max_tokens": 16384, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" @@ -39269,7 +37922,6 @@ }, "gradient_ai/openai-gpt-4o-mini": { "litellm_provider": "gradient_ai", - "max_tokens": 16384, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" @@ -39284,7 +37936,6 @@ "gradient_ai/openai-o3": { "input_cost_per_token": 2e-06, "litellm_provider": "gradient_ai", - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "supported_endpoints": [ @@ -39300,7 +37951,6 @@ "gradient_ai/openai-o3-mini": { "input_cost_per_token": 1.1e-06, "litellm_provider": "gradient_ai", - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supported_endpoints": [ @@ -39316,7 +37966,6 @@ "lemonade/Qwen3-Coder-30B-A3B-Instruct-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 32768, "max_input_tokens": 262144, "max_output_tokens": 32768, "mode": "chat", @@ -39328,7 +37977,6 @@ "lemonade/gpt-oss-20b-mxfp4-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 32768, "max_input_tokens": 131072, "max_output_tokens": 32768, "mode": "chat", @@ -39340,7 +37988,6 @@ "lemonade/gpt-oss-120b-mxfp-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 32768, "max_input_tokens": 131072, "max_output_tokens": 32768, "mode": "chat", @@ -39352,7 +37999,6 @@ "lemonade/Gemma-3-4b-it-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 8192, "max_input_tokens": 128000, "max_output_tokens": 8192, "mode": "chat", @@ -39364,7 +38010,6 @@ "lemonade/Qwen3-4B-Instruct-2507-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 32768, "max_input_tokens": 262144, "max_output_tokens": 32768, "mode": "chat", @@ -39378,7 +38023,6 @@ "litellm_provider": "amazon_nova", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.4e-07, "supports_function_calling": true, @@ -39390,7 +38034,6 @@ "litellm_provider": "amazon_nova", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.4e-07, "supports_function_calling": true, @@ -39404,7 +38047,6 @@ "litellm_provider": "amazon_nova", "max_input_tokens": 1000000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.25e-05, "supports_function_calling": true, @@ -39418,7 +38060,6 @@ "litellm_provider": "amazon_nova", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.2e-06, "supports_function_calling": true, @@ -39432,7 +38073,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://console.groq.com/docs/model/llama-guard-3-8b" @@ -39442,7 +38083,6 @@ "litellm_provider": "groq", "max_input_tokens": 512, "max_output_tokens": 512, - "max_tokens": 512, "mode": "chat", "output_cost_per_token": 3e-08, "source": "https://console.groq.com/docs/models" @@ -39452,7 +38092,6 @@ "litellm_provider": "groq", "max_input_tokens": 512, "max_output_tokens": 512, - "max_tokens": 512, "mode": "chat", "output_cost_per_token": 4e-08, "source": "https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m" @@ -39463,7 +38102,6 @@ "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 6e-07, "search_context_cost_per_query": { @@ -39484,7 +38122,6 @@ "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, "search_context_cost_per_query": { @@ -39505,7 +38142,6 @@ "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, "search_context_cost_per_query": { @@ -39524,7 +38160,6 @@ "litellm_provider": "groq", "max_input_tokens": 4000, "max_output_tokens": 50000, - "max_tokens": 50000, "mode": "audio_speech", "source": "https://console.groq.com/docs/model/canopylabs/orpheus-v1-english" }, @@ -39533,7 +38168,6 @@ "litellm_provider": "groq", "max_input_tokens": 4000, "max_output_tokens": 50000, - "max_tokens": 50000, "mode": "audio_speech", "source": "https://console.groq.com/docs/models" }, @@ -39551,7 +38185,6 @@ }, "heroku/claude-3-5-haiku": { "litellm_provider": "heroku", - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -39561,7 +38194,6 @@ }, "heroku/claude-3-5-sonnet-latest": { "litellm_provider": "heroku", - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -39571,7 +38203,6 @@ }, "heroku/claude-3-7-sonnet": { "litellm_provider": "heroku", - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -39581,7 +38212,6 @@ }, "heroku/claude-4-sonnet": { "litellm_provider": "heroku", - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -39627,7 +38257,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39640,7 +38269,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": true, @@ -39653,7 +38281,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39666,7 +38293,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39679,7 +38305,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-06, "supports_function_calling": true, @@ -39692,7 +38317,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "supports_function_calling": true, @@ -39705,7 +38329,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_function_calling": true, @@ -39718,7 +38341,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": true, @@ -39731,7 +38353,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "supports_function_calling": true, @@ -39744,7 +38365,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39757,7 +38377,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39770,7 +38389,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39783,7 +38401,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39796,7 +38413,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39809,7 +38425,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39822,7 +38437,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-06, "supports_function_calling": true, @@ -39835,7 +38449,6 @@ "litellm_provider": "ai21", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 3e-06 }, @@ -39844,7 +38457,6 @@ "litellm_provider": "ai21", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 1e-05 }, @@ -39853,7 +38465,6 @@ "litellm_provider": "ai21", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 1.5e-05 }, @@ -39862,7 +38473,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39872,7 +38482,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -39882,7 +38491,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -39892,7 +38500,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39902,7 +38509,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39912,7 +38518,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -39922,7 +38527,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -39932,7 +38536,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39942,7 +38545,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39952,7 +38554,6 @@ "litellm_provider": "jina_ai", "max_input_tokens": 1024, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://api.jina.ai/v1/models" @@ -39972,7 +38573,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -40009,7 +38609,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -40037,7 +38636,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 7e-06, "supports_function_calling": false, @@ -40050,7 +38648,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -40063,7 +38660,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -40077,7 +38673,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": true, @@ -40090,7 +38685,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.5e-06, "supports_function_calling": false, @@ -40103,7 +38697,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -40116,7 +38709,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -40130,7 +38722,6 @@ "litellm_provider": "inception", "max_input_tokens": 128000, "max_output_tokens": 50000, - "max_tokens": 50000, "mode": "chat", "output_cost_per_token": 7.5e-07, "supports_function_calling": true, @@ -40145,7 +38736,6 @@ "litellm_provider": "inception", "max_input_tokens": 260000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.5e-07, "source": "https://docs.inceptionlabs.ai/get-started/models", @@ -40160,7 +38750,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40181,7 +38771,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40202,7 +38792,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40223,7 +38813,7 @@ "input_cost_per_token": 1.72e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40244,7 +38834,7 @@ "input_cost_per_token": 1.72e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40265,7 +38855,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://inferenceapis.com/pricing", @@ -40286,7 +38876,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40307,7 +38897,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://inferenceapis.com/pricing", @@ -40328,7 +38918,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40349,7 +38939,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40370,7 +38960,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40391,7 +38981,7 @@ "input_cost_per_token": 1.72e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40412,7 +39002,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://inferenceapis.com/pricing", @@ -40433,7 +39023,7 @@ "input_cost_per_token": 1.82e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048575, - "max_tokens": 1048575, + "max_output_tokens": 1048575, "mode": "chat", "output_cost_per_token": 5.72e-06, "source": "https://inferenceapis.com/pricing", @@ -40454,7 +39044,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048575, - "max_tokens": 1048575, + "max_output_tokens": 1048575, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://inferenceapis.com/pricing", @@ -40474,7 +39064,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://inferenceapis.com/pricing", @@ -40493,7 +39083,7 @@ "input_cost_per_token": 4e-08, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.9e-07, "source": "https://inferenceapis.com/pricing", @@ -40513,7 +39103,7 @@ "input_cost_per_token": 8.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.42e-06, "source": "https://inferenceapis.com/pricing", @@ -40533,7 +39123,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40552,7 +39142,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40571,7 +39161,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40590,7 +39180,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40609,7 +39199,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40628,7 +39218,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40648,7 +39238,7 @@ "input_cost_per_token": 3.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 524288, - "max_tokens": 524288, + "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://inferenceapis.com/pricing", @@ -40669,7 +39259,7 @@ "input_cost_per_token": 3.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 524288, - "max_tokens": 524288, + "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://inferenceapis.com/pricing", @@ -40690,7 +39280,7 @@ "input_cost_per_token": 8.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.42e-06, "source": "https://inferenceapis.com/pricing", @@ -40710,7 +39300,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://inferenceapis.com/pricing", @@ -40729,7 +39319,7 @@ "input_cost_per_token": 4e-08, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.9e-07, "source": "https://inferenceapis.com/pricing", @@ -40748,7 +39338,7 @@ "input_cost_per_token": 2.6e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40768,7 +39358,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40788,7 +39378,7 @@ "input_cost_per_token": 2.6e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40807,7 +39397,7 @@ "input_cost_per_token": 2.6e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40827,7 +39417,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40848,7 +39438,7 @@ "input_cost_per_token": 1.82e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048575, - "max_tokens": 1048575, + "max_output_tokens": 1048575, "mode": "chat", "output_cost_per_token": 5.72e-06, "source": "https://inferenceapis.com/pricing", @@ -40869,7 +39459,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048575, - "max_tokens": 1048575, + "max_output_tokens": 1048575, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://inferenceapis.com/pricing", @@ -40891,7 +39481,6 @@ "litellm_provider": "text-completion-inception", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 7.5e-07 }, @@ -40900,7 +39489,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -40914,7 +39502,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -40928,7 +39515,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -40942,7 +39528,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -40955,7 +39540,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -40968,7 +39552,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -40981,7 +39564,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, @@ -40994,7 +39576,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": true, @@ -41007,7 +39588,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, @@ -41020,7 +39600,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -41033,7 +39612,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 16384, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -41046,7 +39624,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -41059,7 +39636,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -41072,7 +39648,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, @@ -41085,7 +39660,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -41098,7 +39672,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-08, "supports_function_calling": true, @@ -41112,7 +39685,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-08, "supports_function_calling": true, @@ -41125,7 +39697,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -41138,7 +39709,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -41151,7 +39721,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -41196,14 +39765,12 @@ "max-x-max/50-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.036 }, "max-x-max/max-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.072 }, @@ -41299,7 +39866,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_character": 1.5e-05, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", @@ -41310,7 +39876,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_character": 1e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", @@ -41321,7 +39886,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -41331,7 +39895,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.56e-06, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -41342,7 +39905,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -41356,7 +39918,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -41371,7 +39932,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.2e-07, "output_cost_per_token_batches": 1.1e-07, @@ -41385,7 +39945,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.6e-07, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -41399,7 +39958,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", @@ -41412,7 +39970,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", @@ -41425,7 +39982,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -41440,7 +39996,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -41454,7 +40009,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -41465,7 +40019,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -41477,7 +40030,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9.7e-07, "output_cost_per_token_batches": 4.85e-07, @@ -41500,7 +40052,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 10000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_batches": 3.3e-07, @@ -41522,7 +40073,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.25e-06, "search_context_cost_per_query": { @@ -41563,7 +40113,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.25e-06, "search_context_cost_per_query": { @@ -41604,7 +40153,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "search_context_cost_per_query": { @@ -41645,7 +40193,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.25e-06, "search_context_cost_per_query": { @@ -41686,7 +40233,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "search_context_cost_per_query": { @@ -41743,7 +40289,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-05, "search_context_cost_per_query": { @@ -41784,7 +40329,6 @@ "litellm_provider": "meta_llama", "max_input_tokens": 128000, "max_output_tokens": 4028, - "max_tokens": 4028, "mode": "chat", "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ @@ -41800,7 +40344,6 @@ "litellm_provider": "meta_llama", "max_input_tokens": 128000, "max_output_tokens": 4028, - "max_tokens": 4028, "mode": "chat", "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ @@ -41816,7 +40359,6 @@ "litellm_provider": "meta_llama", "max_input_tokens": 1000000, "max_output_tokens": 4028, - "max_tokens": 4028, "mode": "chat", "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ @@ -41833,7 +40375,6 @@ "litellm_provider": "meta_llama", "max_input_tokens": 10000000, "max_output_tokens": 4028, - "max_tokens": 4028, "mode": "chat", "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ @@ -41855,7 +40396,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_batches": 6e-07, @@ -41878,7 +40418,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 196000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_batches": 6e-07, @@ -41902,7 +40441,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 196000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 6e-07, "output_cost_per_token_priority": 2.1e-06, @@ -42100,7 +40638,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -42122,7 +40659,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 40000, - "max_tokens": 40000, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, @@ -42144,7 +40680,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_batches": 1e-07, @@ -42166,7 +40701,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-07, "output_cost_per_token_batches": 5e-08, @@ -42188,7 +40722,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.5e-07, "output_cost_per_token_batches": 7e-08, @@ -42208,7 +40741,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", @@ -42219,7 +40751,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", @@ -42230,7 +40761,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -42245,7 +40775,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, @@ -42265,7 +40794,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, @@ -42278,7 +40806,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", @@ -42293,7 +40820,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-08, "output_cost_per_token_batches": 2e-08, @@ -42313,7 +40839,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-07, "output_cost_per_token_batches": 1.5e-07, @@ -42330,7 +40855,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", @@ -42345,7 +40869,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", @@ -42360,7 +40883,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.5e-07, "source": "https://mistral.ai/technology/", @@ -42373,7 +40895,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://docs.mistral.ai/models/devstral-small-2-25-12", @@ -42388,7 +40909,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://mistral.ai/news/devstral-2-vibe-cli", @@ -42403,7 +40923,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://mistral.ai/news/devstral-2-vibe-cli", @@ -42417,7 +40936,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.mistral.ai/models/ministral-3-14b-25-12", @@ -42433,7 +40951,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.mistral.ai/models/ministral-3-14b-25-12", @@ -42449,7 +40966,6 @@ "litellm_provider": "mistral", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://docs.mistral.ai/models/ministral-3-3b-25-12", @@ -42465,7 +40981,6 @@ "litellm_provider": "mistral", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://docs.mistral.ai/models/ministral-3-3b-25-12", @@ -42480,7 +40995,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "source": "https://docs.mistral.ai/models/mistral-embed-23-12" }, @@ -42490,7 +41004,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -42671,7 +41184,6 @@ "litellm_provider": "mistral", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://docs.mistral.ai/models/voxtral-small-25-07", @@ -42686,7 +41198,6 @@ "litellm_provider": "mistral", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://docs.mistral.ai/models/voxtral-small-25-07", @@ -42702,7 +41213,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42728,7 +41238,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42750,7 +41259,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "reasoning_effort_levels": [ @@ -42773,7 +41281,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42795,7 +41302,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42817,7 +41323,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42896,7 +41401,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", @@ -42913,7 +41417,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03", @@ -42928,7 +41431,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding" }, "mistral/codestral-embed": { @@ -42936,7 +41438,6 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding" }, "mistral/codestral-embed-2505": { @@ -42944,7 +41445,6 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding" }, "mistral/mistral-large-4": { @@ -42952,7 +41452,7 @@ "input_cost_per_token": 6.8e-07, "litellm_provider": "mistral", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 2.09e-06, "reasoning_effort_levels": [ @@ -42973,7 +41473,7 @@ "input_cost_per_token": 6.8e-07, "litellm_provider": "mistral", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 2.09e-06, "reasoning_effort_levels": [ @@ -42995,7 +41495,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://docs.mistral.ai/models/mistral-large-3-25-12", @@ -43011,7 +41510,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://docs.mistral.ai/models/mistral-large-3-25-12", @@ -43027,7 +41525,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://docs.mistral.ai/models/mistral-large-3-25-12", @@ -43043,7 +41540,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -43064,7 +41560,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -43085,7 +41580,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -43106,7 +41600,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -43127,7 +41620,6 @@ "litellm_provider": "mistral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-07, "supports_assistant_prefill": true, @@ -43141,7 +41633,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6e-07, "reasoning_effort_levels": [ @@ -43162,7 +41653,6 @@ "litellm_provider": "mistral", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://mistral.ai/pricing", @@ -43178,7 +41668,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://mistral.ai/pricing", @@ -43194,7 +41683,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://mistral.ai/pricing", @@ -43210,7 +41698,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://mistral.ai/pricing", @@ -43226,7 +41713,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://mistral.ai/pricing", @@ -43242,7 +41728,6 @@ "litellm_provider": "mistral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_assistant_prefill": true, @@ -43255,7 +41740,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://mistral.ai/technology/", @@ -43269,7 +41753,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-06, "supports_assistant_prefill": true, @@ -43286,7 +41769,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, @@ -43308,7 +41790,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, @@ -43328,7 +41809,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://platform.kimi.ai/docs/pricing/chat-k27-code", @@ -43346,7 +41826,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", @@ -43363,7 +41842,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://platform.kimi.ai/docs/pricing/chat-k26", @@ -43380,7 +41858,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ @@ -43401,7 +41878,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43414,7 +41890,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43428,7 +41903,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43441,7 +41915,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43455,7 +41928,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43468,7 +41940,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43482,7 +41953,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43495,7 +41965,6 @@ "litellm_provider": "morph", "max_input_tokens": 16000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": false, @@ -43509,7 +41978,6 @@ "litellm_provider": "morph", "max_input_tokens": 16000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 1.9e-06, "supports_function_calling": false, @@ -43527,7 +41995,6 @@ "input_cost_per_video_per_second_above_8s_interval": 0.001, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -43551,7 +42018,6 @@ "input_cost_per_video_per_second_above_8s_interval": 0.001, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -43711,7 +42177,6 @@ ] }, "nebius/deepseek-ai/DeepSeek-R1": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 8e-07, @@ -43723,7 +42188,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-R1-0528": { - "max_tokens": 164000, "max_input_tokens": 164000, "max_output_tokens": 164000, "input_cost_per_token": 8e-07, @@ -43735,7 +42199,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2.5e-07, @@ -43746,7 +42209,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V3": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 5e-07, @@ -43757,7 +42219,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V3-0324": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 5e-07, @@ -43768,7 +42229,6 @@ "source": "https://nebius.com/prices" }, "nebius/google/gemma-3-27b-it": { - "max_tokens": 110000, "max_input_tokens": 110000, "max_output_tokens": 110000, "input_cost_per_token": 1e-07, @@ -43780,7 +42240,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/google%2Fgemma-3-27b-it" }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.3e-07, @@ -43791,7 +42250,6 @@ "source": "https://nebius.com/prices" }, "nebius/meta-llama/Llama-Guard-3-8B": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-08, @@ -43801,7 +42259,6 @@ "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-8B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-08, @@ -43812,7 +42269,6 @@ "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-70B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.3e-07, @@ -43823,7 +42279,6 @@ "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-405B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1e-06, @@ -43834,7 +42289,6 @@ "source": "https://nebius.com/prices" }, "nebius/mistralai/Mistral-Nemo-Instruct-2407": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 4e-08, @@ -43845,7 +42299,6 @@ "source": "https://nebius.com/prices" }, "nebius/NousResearch/Hermes-3-Llama-3.1-405B": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1e-06, @@ -43856,7 +42309,6 @@ "source": "https://nebius.com/prices" }, "nebius/nvidia/Llama-3.1-Nemotron-Ultra-253B-v1": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 6e-07, @@ -43867,7 +42319,6 @@ "source": "https://nebius.com/prices" }, "nebius/nvidia/Llama-3.3-Nemotron-Super-49B-v1": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -43878,7 +42329,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-235B-A22B": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2e-07, @@ -43889,7 +42339,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-32B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1e-07, @@ -43900,7 +42349,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-32B" }, "nebius/Qwen/Qwen3-30B-A3B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -43911,7 +42359,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-14B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 8e-08, @@ -43922,7 +42369,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-4B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 8e-08, @@ -43933,7 +42379,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/QwQ-32B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1.5e-07, @@ -43945,7 +42390,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-72B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.3e-07, @@ -43956,7 +42400,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-32B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 6e-08, @@ -43967,7 +42410,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-Coder-7B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-08, @@ -43978,7 +42420,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-VL-72B-Instruct": { - "max_tokens": 32000, "max_input_tokens": 32000, "max_output_tokens": 32000, "input_cost_per_token": 2.5e-07, @@ -43990,7 +42431,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/Qwen%2FQwen2.5-VL-72B-Instruct" }, "nebius/Qwen/Qwen2-VL-72B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.3e-07, @@ -44002,7 +42442,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2-VL-7B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-08, @@ -44013,7 +42452,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V4-Flash": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1.4e-07, @@ -44025,7 +42463,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash" }, "nebius/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_tokens": 1024000, "max_input_tokens": 1024000, "max_output_tokens": 1024000, "input_cost_per_token": 1.4e-07, @@ -44037,7 +42474,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash-0731" }, "nebius/deepseek-ai/DeepSeek-V4-Pro": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1.75e-06, @@ -44063,7 +42499,6 @@ "litellm_provider": "nebius", "max_input_tokens": 1048000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://tokenfactory.nebius.com/endpoints?modals=endpoint-details&model-id=deepseek-ai/DeepSeek-V4.1-Flash", @@ -44072,7 +42507,6 @@ "supports_vision": true }, "nebius/MiniMaxAI/MiniMax-M2.5": { - "max_tokens": 196608, "max_input_tokens": 196608, "max_output_tokens": 196608, "input_cost_per_token": 3e-07, @@ -44084,7 +42518,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M2.5" }, "nebius/MiniMaxAI/MiniMax-M3": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 3e-07, @@ -44097,7 +42530,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M3" }, "nebius/moonshotai/Kimi-K2.6": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9.5e-07, @@ -44110,7 +42542,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K2.6" }, "nebius/moonshotai/Kimi-K2.7-Code": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9.5e-07, @@ -44123,7 +42554,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/moonshotai%2FKimi-K2.7-Code" }, "nebius/moonshotai/Kimi-K3": { - "max_tokens": 1024000, "max_input_tokens": 1024000, "max_output_tokens": 1024000, "input_cost_per_token": 3e-06, @@ -44136,7 +42566,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K3" }, "nebius/NousResearch/Hermes-4-405B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-06, @@ -44148,7 +42577,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-405B" }, "nebius/NousResearch/Hermes-4-70B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.3e-07, @@ -44160,7 +42588,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-70B" }, "nebius/nvidia/Cosmos3-Super-Reasoner": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1e-07, @@ -44173,7 +42600,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/nvidia%2FCosmos3-Super-Reasoner" }, "nebius/nvidia/Llama-3_1-Nemotron-Ultra-253B-v1": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6e-07, @@ -44184,7 +42610,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FLlama-3_1-Nemotron-Ultra-253B-v1" }, "nebius/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 6e-08, @@ -44196,7 +42621,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNVIDIA-Nemotron-3-Nano-30B-A3B" }, "nebius/nvidia/Nemotron-3-Nano-Omni": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 6e-08, @@ -44209,7 +42633,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Nano-Omni" }, "nebius/nvidia/nemotron-3-super-120b-a12b": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 3e-07, @@ -44221,7 +42644,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2Fnemotron-3-super-120b-a12b" }, "nebius/nvidia/Nemotron-3-Ultra-550b-a55b": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1e-06, @@ -44233,7 +42655,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Ultra-550b-a55b" }, "nebius/nvidia/Nemotron-3_5-Lightning": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 6e-08, @@ -44245,7 +42666,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3_5-Lightning" }, "nebius/openai/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.5e-07, @@ -44257,7 +42677,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/openai%2Fgpt-oss-120b" }, "nebius/openbmb/MiniCPM-V-4_5": { - "max_tokens": 32000, "max_input_tokens": 32000, "max_output_tokens": 32000, "input_cost_per_token": 6.58e-07, @@ -44268,7 +42687,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/openbmb%2FMiniCPM-V-4_5" }, "nebius/Qwen/Qwen3-235B-A22B-Instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2e-07, @@ -44279,7 +42697,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-235B-A22B-Instruct-2507" }, "nebius/Qwen/Qwen3-30B-A3B-Instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1e-07, @@ -44290,7 +42707,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-30B-A3B-Instruct-2507" }, "nebius/Qwen/Qwen3-Next-80B-A3B-Thinking": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.5e-07, @@ -44302,7 +42718,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-Next-80B-A3B-Thinking" }, "nebius/Qwen/Qwen3.5-397B-A17B": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 6e-07, @@ -44324,7 +42739,6 @@ "supports_reasoning": true }, "nebius/zai-org/GLM-5.1": { - "max_tokens": 202752, "max_input_tokens": 202752, "max_output_tokens": 202752, "input_cost_per_token": 1.4e-06, @@ -44336,7 +42750,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.1" }, "nebius/zai-org/GLM-5.2": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1.4e-06, @@ -44351,7 +42764,7 @@ "input_cost_per_token": 1.4e-06, "litellm_provider": "nebius", "max_input_tokens": 1024000, - "max_tokens": 1024000, + "max_output_tokens": 1024000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3", @@ -44359,7 +42772,6 @@ "supports_reasoning": true }, "nebius/zai-org/GLM-5.3-Flash": { - "max_tokens": 1024000, "max_input_tokens": 1024000, "max_output_tokens": 1024000, "input_cost_per_token": 1.5e-07, @@ -44372,7 +42784,6 @@ "supports_vision": true }, "nebius/BAAI/bge-en-icl": { - "max_tokens": 32768, "max_input_tokens": 32768, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -44381,7 +42792,6 @@ "source": "https://nebius.com/prices" }, "nebius/BAAI/bge-multilingual-gemma2": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -44390,7 +42800,6 @@ "source": "https://nebius.com/prices" }, "nebius/intfloat/e5-mistral-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -44399,7 +42808,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-Embedding-8B": { - "max_tokens": 40960, "max_input_tokens": 40960, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -44416,7 +42824,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -44438,7 +42845,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.3e-07, "output_cost_per_token_batches": 1.2e-07, @@ -44459,7 +42865,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.4e-07, "output_cost_per_token_batches": 1.2e-07, @@ -44482,7 +42887,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 6.5e-07, "output_cost_per_token_batches": 3.25e-07, @@ -44505,7 +42909,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_batches": 3e-05, @@ -44528,7 +42931,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_batches": 3e-05, @@ -44550,7 +42952,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 0.0006, "output_cost_per_token_batches": 0.0003, @@ -44584,7 +42985,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 0.0006, "output_cost_per_token_batches": 0.0003, @@ -44622,7 +43022,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -44663,7 +43062,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 4e-05, "source": "https://developers.openai.com/api/docs/models/o3-deep-research", @@ -44696,7 +43094,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 4e-05, "source": "https://developers.openai.com/api/docs/models/o3-deep-research", @@ -44733,7 +43130,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -44776,7 +43172,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -44797,7 +43192,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -44816,7 +43210,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, @@ -44854,7 +43247,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, @@ -44897,7 +43289,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -44925,7 +43316,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-06, "source": "https://developers.openai.com/api/docs/models/o4-mini-deep-research", @@ -44958,7 +43348,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-06, "source": "https://developers.openai.com/api/docs/models/o4-mini-deep-research", @@ -44995,7 +43384,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -45022,7 +43410,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45035,7 +43422,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45048,7 +43434,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.068e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45061,7 +43446,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45075,7 +43459,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45088,7 +43471,6 @@ "litellm_provider": "oci", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45102,7 +43484,6 @@ "litellm_provider": "oci", "max_input_tokens": 10485760, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45115,7 +43496,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45128,7 +43508,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45141,7 +43520,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45154,7 +43532,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45167,7 +43544,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45180,7 +43556,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45193,7 +43568,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45206,7 +43580,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45219,7 +43592,6 @@ "litellm_provider": "oci", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45239,7 +43611,6 @@ "litellm_provider": "oci", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45258,7 +43629,6 @@ "litellm_provider": "oci", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45278,7 +43648,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45292,7 +43661,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45314,7 +43682,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45326,7 +43693,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45339,7 +43705,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 9e-08, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45351,7 +43716,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45363,7 +43727,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45375,7 +43738,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45388,7 +43750,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45400,7 +43761,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45412,7 +43772,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45424,7 +43783,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [], @@ -45437,7 +43795,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45452,7 +43809,6 @@ "litellm_provider": "oci", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -45468,7 +43824,6 @@ "litellm_provider": "oci", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -45485,7 +43840,6 @@ "litellm_provider": "oci", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -45501,7 +43855,6 @@ "litellm_provider": "oci", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -45515,7 +43868,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45527,7 +43879,6 @@ "litellm_provider": "oci", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45542,7 +43893,6 @@ "litellm_provider": "oci", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45557,7 +43907,6 @@ "litellm_provider": "oci", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45572,7 +43921,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45585,7 +43933,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45597,7 +43944,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -45607,7 +43953,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 384, @@ -45617,7 +43962,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -45627,7 +43971,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 384, @@ -45637,7 +43980,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -45648,7 +43990,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 384, @@ -45659,7 +44000,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 384, @@ -45670,7 +44010,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "oci", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -45682,7 +44021,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": false @@ -45692,7 +44030,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45701,7 +44038,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45710,7 +44046,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45720,7 +44055,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45730,7 +44064,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45740,7 +44073,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45750,7 +44082,6 @@ "litellm_provider": "ollama", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45760,7 +44091,6 @@ "litellm_provider": "ollama", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45770,7 +44100,6 @@ "litellm_provider": "ollama", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45780,7 +44109,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45790,7 +44118,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45799,7 +44126,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45808,7 +44134,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45817,7 +44142,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45826,7 +44150,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45835,7 +44158,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45844,7 +44166,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45854,7 +44175,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45863,7 +44183,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45872,7 +44191,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45882,7 +44200,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45892,7 +44209,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45902,7 +44218,6 @@ "litellm_provider": "ollama", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45912,7 +44227,6 @@ "litellm_provider": "ollama", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45922,7 +44236,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45932,7 +44245,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45941,7 +44253,6 @@ "litellm_provider": "ollama", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45951,7 +44262,6 @@ "litellm_provider": "ollama", "max_input_tokens": 2048, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45960,7 +44270,6 @@ "litellm_provider": "openai", "max_input_tokens": 32768, "max_output_tokens": 0, - "max_tokens": 0, "mode": "moderation", "output_cost_per_token": 0.0 }, @@ -45969,7 +44278,6 @@ "litellm_provider": "openai", "max_input_tokens": 32768, "max_output_tokens": 0, - "max_tokens": 0, "mode": "moderation", "output_cost_per_token": 0.0 }, @@ -45987,7 +44295,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -46013,7 +44320,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-07, "output_cost_per_token_batches": 1.5e-07, @@ -46033,7 +44339,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -46055,7 +44360,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_batches": 1e-07, @@ -46074,7 +44378,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46088,7 +44391,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46101,7 +44403,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46114,7 +44415,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46127,7 +44427,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 127000, "max_output_tokens": 127000, - "max_tokens": 127000, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46140,7 +44439,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 118000, "max_output_tokens": 118000, - "max_tokens": 118000, "mode": "chat", "output_cost_per_token": 1.3e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46153,7 +44451,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46167,7 +44464,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 6.3e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46180,7 +44476,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 8.7e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46193,7 +44488,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 9.1e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46207,7 +44501,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.3e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46221,7 +44514,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46235,7 +44527,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46249,7 +44540,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.9e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46263,7 +44553,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.9e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46296,7 +44585,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-06 }, @@ -46305,7 +44593,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.8e-06 }, @@ -46314,7 +44601,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-06 }, @@ -46323,7 +44609,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -46332,7 +44617,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07 }, @@ -46341,7 +44625,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46350,7 +44633,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46359,7 +44641,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-06 }, @@ -46369,7 +44650,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-06 }, @@ -46378,7 +44658,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46388,7 +44667,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46444,7 +44722,7 @@ "input_cost_per_token": 1e-06, "litellm_provider": "perplexity", "max_input_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-06, "search_context_cost_per_query": { @@ -46459,7 +44737,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "perplexity", "max_input_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 128000, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 8e-06, @@ -46476,7 +44754,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.8e-06 }, @@ -46486,7 +44763,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 12000, "max_output_tokens": 12000, - "max_tokens": 12000, "mode": "chat", "output_cost_per_token": 1.8e-06 }, @@ -46495,7 +44771,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 200000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -46509,7 +44784,7 @@ "input_cost_per_token": 1e-06, "litellm_provider": "perplexity", "max_input_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-06, "search_context_cost_per_query": { @@ -46524,7 +44799,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "perplexity", "max_input_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 8e-06, "search_context_cost_per_query": { @@ -46540,7 +44815,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46550,7 +44824,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 12000, "max_output_tokens": 12000, - "max_tokens": 12000, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46559,7 +44832,6 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46571,7 +44843,6 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46583,7 +44854,6 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46595,7 +44865,6 @@ "litellm_provider": "publicai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46607,7 +44876,6 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46619,7 +44887,6 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46859,7 +45126,6 @@ "input_cost_per_token": 4e-09, "litellm_provider": "perplexity", "max_input_tokens": 32768, - "max_tokens": 32768, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -46869,7 +45135,6 @@ "input_cost_per_token": 3e-08, "litellm_provider": "perplexity", "max_input_tokens": 32768, - "max_tokens": 32768, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2560, @@ -46880,7 +45145,6 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46892,7 +45156,6 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46905,7 +45168,6 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46919,7 +45181,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.8e-06, "supports_function_calling": true, @@ -46934,7 +45195,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 8.8e-07, "output_cost_per_token_flex": 4.4e-07, @@ -46954,7 +45214,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -46973,7 +45232,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -46990,7 +45248,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -47006,7 +45263,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.45e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47022,7 +45278,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.41e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47038,7 +45293,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.236e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47054,7 +45308,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.41e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47070,7 +45323,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47086,7 +45338,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.45e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47104,7 +45355,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.66e-06, "output_cost_per_token_batches": 1.33e-06, @@ -47125,7 +45375,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_batches": 6e-07, @@ -47180,7 +45429,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, "supports_tool_choice": true @@ -47190,7 +45438,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, "supports_tool_choice": true @@ -47200,7 +45447,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.75e-06, "supports_tool_choice": true @@ -47210,7 +45456,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.75e-06, "supports_tool_choice": true @@ -47220,7 +45465,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47230,7 +45474,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47240,7 +45483,6 @@ "litellm_provider": "replicate", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.75e-06, "supports_tool_choice": true @@ -47250,7 +45492,6 @@ "litellm_provider": "replicate", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.75e-06, "supports_tool_choice": true @@ -47260,7 +45501,6 @@ "litellm_provider": "replicate", "max_input_tokens": 8086, "max_output_tokens": 8086, - "max_tokens": 8086, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47270,7 +45510,6 @@ "litellm_provider": "replicate", "max_input_tokens": 8086, "max_output_tokens": 8086, - "max_tokens": 8086, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47280,7 +45519,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47290,7 +45528,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47300,7 +45537,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06, "supports_tool_choice": true @@ -47429,7 +45665,6 @@ "mode": "chat", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_system_messages": true }, @@ -47584,7 +45819,6 @@ "mode": "chat", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true @@ -47605,7 +45839,6 @@ "mode": "chat", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_reasoning": true, "supports_system_messages": true }, @@ -47615,7 +45848,6 @@ "litellm_provider": "cohere", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -47625,7 +45857,6 @@ "litellm_provider": "cohere", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -47635,7 +45866,6 @@ "litellm_provider": "cohere", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -47645,7 +45875,6 @@ "litellm_provider": "cohere", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://cohere.com/pricing" @@ -47656,7 +45885,6 @@ "litellm_provider": "cohere", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://cohere.com/pricing" @@ -47687,7 +45915,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -47696,7 +45923,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -47705,7 +45931,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -47714,7 +45939,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -47723,7 +45947,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -47732,7 +45955,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -47741,7 +45963,6 @@ "litellm_provider": "sambanova", "max_input_tokens": 196608, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://cloud.sambanova.ai/plans/pricing", @@ -47754,7 +45975,6 @@ "litellm_provider": "sambanova", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7e-06, "source": "https://cloud.sambanova.ai/plans/pricing" @@ -47764,7 +45984,6 @@ "litellm_provider": "sambanova", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount" }, @@ -47781,7 +46000,6 @@ "litellm_provider": "sambanova", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.sambanova.ai/plans/pricing", @@ -47790,7 +46008,6 @@ "supports_tool_choice": true }, "sambanova/DeepSeek-V3.1": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-06, @@ -47803,7 +46020,6 @@ "source": "https://cloud.sambanova.ai/plans/pricing" }, "sambanova/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2.2e-07, @@ -47816,7 +46032,6 @@ "source": "https://cloud.sambanova.ai/plans/pricing" }, "sambanova/DeepSeek-V3.2": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 3e-06, @@ -47828,7 +46043,6 @@ "source": "https://cloud.sambanova.ai/plans/pricing" }, "sambanova/gemma-4-31B-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3.8e-07, @@ -47844,7 +46058,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1000000, "max_output_tokens": 375000, - "max_tokens": 375000, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://scx.ai/models/deepseek-v4-1-flash", @@ -47859,7 +46072,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.08e-06, "source": "https://scx.ai/models/glm-5-3", @@ -47875,7 +46087,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.413e-05, "source": "https://scx.ai/models/kimi-k3", @@ -47891,7 +46102,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.98e-06, "source": "https://scx.ai/pricing", @@ -47908,7 +46118,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.99e-06, "source": "https://scx.ai/pricing", @@ -47923,7 +46132,6 @@ "litellm_provider": "snowflake", "max_input_tokens": 200000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, @@ -47939,7 +46147,6 @@ "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 1.35e-06, "output_cost_per_token": 5.4e-06, @@ -47950,56 +46157,48 @@ "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/jamba-1.5-large": { "litellm_provider": "snowflake", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/jamba-1.5-mini": { "litellm_provider": "snowflake", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/jamba-instruct": { "litellm_provider": "snowflake", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama2-70b-chat": { "litellm_provider": "snowflake", "max_input_tokens": 4096, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3-70b": { "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3-8b": { "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.1-405b": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 1.2e-06, "output_cost_per_token": 1.2e-06, @@ -48010,7 +46209,6 @@ "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 7.2e-07, "output_cost_per_token": 7.2e-07, @@ -48021,7 +46219,6 @@ "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 2.4e-07, "output_cost_per_token": 2.4e-07, @@ -48031,18 +46228,15 @@ "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.2-3b": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.3-70b": { - "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 7.2e-07, @@ -48056,21 +46250,18 @@ "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/mistral-large": { "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/mistral-large2": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, @@ -48082,39 +46273,33 @@ "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/reka-core": { "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/reka-flash": { "litellm_provider": "snowflake", "max_input_tokens": 100000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/snowflake-arctic": { "litellm_provider": "snowflake", "max_input_tokens": 4096, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/snowflake-llama-3.1-405b": { "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/snowflake-llama-3.3-70b": { - "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 7.2e-07, @@ -48311,21 +46496,18 @@ "stability.sd3-5-large-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.08 }, "stability.sd3-large-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.08 }, "stability.stable-image-core-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.04 }, @@ -48501,21 +46683,18 @@ "stability.stable-image-core-v1:1": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.04 }, "stability.stable-image-ultra-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.14 }, "stability.stable-image-ultra-v1:1": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.14 }, @@ -48549,7 +46728,6 @@ "litellm_provider": "text-completion-codestral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "completion", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/capabilities/code_generation/" @@ -48559,7 +46737,6 @@ "litellm_provider": "text-completion-codestral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "completion", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/capabilities/code_generation/" @@ -48570,7 +46747,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -48582,7 +46758,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -48593,7 +46768,6 @@ "input_cost_per_token_batches": 6.5e-08, "litellm_provider": "openai", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "output_cost_per_token_batches": 0.0, @@ -48605,7 +46779,6 @@ "input_cost_per_token_batches": 1e-08, "litellm_provider": "openai", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "output_cost_per_token_batches": 0.0, @@ -48616,7 +46789,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "openai", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -48627,7 +46799,6 @@ "input_cost_per_token_batches": 5e-08, "litellm_provider": "openai", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "output_cost_per_token_batches": 0.0 @@ -48637,7 +46808,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -48647,7 +46817,6 @@ "input_cost_per_token": 6.25e-09, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 3072, - "max_tokens": 3072, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -48659,7 +46828,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -48670,7 +46838,6 @@ "litellm_provider": "vertex_ai-text-models", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "completion", "output_cost_per_token": 2.8e-05, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -48680,7 +46847,6 @@ "litellm_provider": "vertex_ai-text-models", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "completion", "output_cost_per_token": 2.8e-05, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -48706,7 +46872,8 @@ "together-ai-8.1b-21b": { "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", - "max_tokens": 1000, + "max_output_tokens": 1000, + "max_input_tokens": 1000, "mode": "chat", "output_cost_per_token": 3e-07 }, @@ -48759,7 +46926,6 @@ "litellm_provider": "together_ai", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "metadata": { "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" }, @@ -48775,7 +46941,7 @@ "input_cost_per_token": 1.04e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.04e-06, "source": "https://api.together.ai/v1/models", @@ -48802,7 +46968,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://api.together.ai/v1/models", @@ -48823,7 +46989,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 202752, - "max_tokens": 202752, + "max_output_tokens": 202752, "metadata": { "successor": "together_ai/zai-org/GLM-5.2" }, @@ -48840,7 +47006,7 @@ "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 524288, - "max_tokens": 524288, + "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", @@ -48857,7 +47023,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://www.together.ai/pricing", @@ -48867,7 +47033,7 @@ "input_cost_per_token": 0.0, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.together.ai/docs/serverless-models" @@ -48876,7 +47042,7 @@ "input_cost_per_token": 1.7e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.5e-07, "source": "https://api.together.ai/v1/models", @@ -48891,7 +47057,7 @@ "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://api.together.ai/v1/models", @@ -48902,7 +47068,7 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "together_ai", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 4.5e-06, "source": "https://www.together.ai/pricing", @@ -48912,7 +47078,7 @@ "input_cost_per_token": 3.2e-07, "litellm_provider": "together_ai", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.28e-06, "source": "https://api.together.ai/v1/models" @@ -48922,7 +47088,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 1010000, - "max_tokens": 1010000, + "max_output_tokens": 1010000, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://api.together.ai/v1/models", @@ -48932,7 +47098,7 @@ "input_cost_per_token": 1e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://api.together.ai/v1/models" @@ -48943,7 +47109,7 @@ "input_cost_per_token": 1.4e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://api.together.ai/v1/models", @@ -48958,7 +47124,7 @@ "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", @@ -48972,7 +47138,7 @@ "input_cost_per_token": 1.32e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.96e-06, "source": "https://api.together.ai/v1/models", @@ -48987,7 +47153,7 @@ "input_cost_per_token": 3.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://api.together.ai/v1/models", @@ -48998,7 +47164,7 @@ "input_cost_per_token": 1.25e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.25e-06, "source": "https://docs.together.ai/docs/serverless-models", @@ -49011,7 +47177,7 @@ "input_cost_per_token": 3e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ @@ -49033,7 +47199,7 @@ "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", "max_input_tokens": 524288, - "max_tokens": 524288, + "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 4.05e-06, "source": "https://api.together.ai/v1/models", @@ -49049,7 +47215,6 @@ "litellm_provider": "together_ai", "max_input_tokens": 1048575, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://api.together.ai/v1/models", @@ -49066,7 +47231,6 @@ "litellm_provider": "together_ai", "max_input_tokens": 1048575, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://api.together.ai/v1/models", @@ -49083,7 +47247,6 @@ "litellm_provider": "together_ai", "max_input_tokens": 1048575, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://api.together.ai/v1/models", @@ -49168,7 +47331,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.4e-07, "output_cost_per_token_batches": 1.2e-07, @@ -49187,7 +47349,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.4e-07, "output_cost_per_token_batches": 7e-08, @@ -49208,7 +47369,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.2e-06, "output_cost_per_token_batches": 1.6e-06, @@ -49229,7 +47389,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, @@ -49249,7 +47408,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -49277,7 +47435,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -49295,7 +47452,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -49314,7 +47470,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -49332,7 +47487,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -49350,7 +47504,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -49389,7 +47542,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -49431,7 +47583,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -49463,7 +47614,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, @@ -49503,7 +47653,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 1024, @@ -49542,7 +47691,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, @@ -49580,7 +47728,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.4e-05, "prompt_cache_min_tokens": 512, @@ -49618,7 +47765,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_adaptive_thinking": true, @@ -49654,7 +47800,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.88e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -49672,7 +47817,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", @@ -49688,7 +47832,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", @@ -49703,7 +47846,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -49721,7 +47863,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", @@ -49741,7 +47882,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", @@ -49768,7 +47908,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "supports_function_calling": true, "supports_prompt_caching": false, @@ -49785,7 +47924,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "supports_assistant_prefill": true, @@ -49815,7 +47953,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -49843,7 +47980,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -49880,7 +48016,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -49917,7 +48052,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -49957,7 +48091,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -49988,7 +48121,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-r1.html", @@ -50004,7 +48136,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_batches": 9.25e-07, @@ -50025,7 +48156,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -50043,7 +48173,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -50057,7 +48186,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -50072,7 +48200,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.2e-07, "output_cost_per_token_batches": 1.1e-07, @@ -50086,7 +48213,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.6e-07, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -50100,7 +48226,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", @@ -50113,7 +48238,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", @@ -50126,7 +48250,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -50141,7 +48264,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -50156,7 +48278,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9.7e-07, "output_cost_per_token_batches": 4.85e-07, @@ -50179,7 +48300,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 10000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_batches": 3.3e-07, @@ -50200,7 +48320,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-pixtral-large.md", @@ -50212,7 +48331,6 @@ "litellm_provider": "v0", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -50226,7 +48344,6 @@ "litellm_provider": "v0", "max_input_tokens": 512000, "max_output_tokens": 512000, - "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -50240,7 +48357,6 @@ "litellm_provider": "v0", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -50254,7 +48370,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.4e-07 }, @@ -50263,7 +48378,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07 }, @@ -50272,7 +48386,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-07 }, @@ -50281,7 +48394,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -50292,7 +48404,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 262144, "max_output_tokens": 66536, - "max_tokens": 66536, "mode": "chat", "output_cost_per_token": 1.6e-06, "supports_function_calling": true, @@ -50303,7 +48414,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 300000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-07, "supports_vision": true, @@ -50315,7 +48425,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.4e-07, "supports_function_calling": true, @@ -50326,7 +48435,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 300000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.2e-06, "supports_vision": true, @@ -50338,7 +48446,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -50349,7 +48456,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, "supports_vision": true, @@ -50364,7 +48470,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_vision": true, @@ -50379,7 +48484,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_vision": true, @@ -50394,7 +48498,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_vision": true, @@ -50409,7 +48512,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_vision": true, @@ -50424,7 +48526,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_vision": true, @@ -50439,7 +48540,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -50452,7 +48552,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50470,7 +48569,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50488,7 +48586,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50507,7 +48604,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -50527,7 +48623,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_assistant_prefill": true, @@ -50547,7 +48642,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_assistant_prefill": true, @@ -50567,7 +48661,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": true, @@ -50590,7 +48683,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": false, @@ -50611,7 +48703,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50631,7 +48722,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50648,7 +48738,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -50660,7 +48749,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -50671,7 +48759,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -50682,7 +48769,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -50691,7 +48777,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.19e-06, "supports_tool_choice": true @@ -50701,7 +48786,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 9.9e-07, "supports_function_calling": true, @@ -50713,7 +48797,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9e-07, "supports_tool_choice": true @@ -50723,7 +48806,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1000000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.5e-06, "supports_vision": true, @@ -50742,7 +48824,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "supports_vision": true, @@ -50763,7 +48844,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -50772,7 +48852,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "supports_vision": true, @@ -50784,7 +48863,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -50793,7 +48871,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -50802,7 +48879,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -50811,7 +48887,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.9e-07, "supports_tool_choice": true @@ -50821,7 +48896,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 8e-08, "supports_tool_choice": true @@ -50831,7 +48905,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_tool_choice": true @@ -50841,7 +48914,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-08, "supports_function_calling": true, @@ -50852,7 +48924,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.6e-07, "supports_vision": true, @@ -50864,7 +48935,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-07 }, @@ -50873,7 +48943,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-07, "supports_function_calling": true, @@ -50884,7 +48953,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_vision": true, @@ -50896,7 +48964,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_function_calling": true, @@ -50907,7 +48974,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "supports_tool_choice": true @@ -50917,7 +48983,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-07, "supports_vision": true, @@ -50929,7 +48994,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 9e-07, "supports_function_calling": true, @@ -50940,7 +49004,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -50949,7 +49012,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.8e-07, "supports_function_calling": true, @@ -50961,7 +49023,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "supports_function_calling": true, @@ -50973,7 +49034,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true @@ -50983,7 +49043,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, @@ -50994,7 +49053,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1e-07, "supports_vision": true, @@ -51006,7 +49064,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -51015,7 +49072,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -51026,7 +49082,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.9e-07 }, @@ -51035,7 +49090,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -51047,7 +49101,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 65536, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": true @@ -51057,7 +49110,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.5e-07, "supports_vision": true, @@ -51070,7 +49122,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 6e-06, "supports_vision": true, @@ -51083,7 +49134,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.2e-06, "supports_function_calling": true, @@ -51094,7 +49144,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06 }, @@ -51103,7 +49152,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.9e-06 }, @@ -51112,7 +49160,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -51123,7 +49170,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06 }, @@ -51132,7 +49178,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_vision": true, @@ -51146,7 +49191,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "supports_vision": true, @@ -51161,7 +49205,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "supports_vision": true, @@ -51176,7 +49219,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "supports_vision": true, @@ -51191,7 +49233,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_vision": true, @@ -51206,7 +49247,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "supports_vision": true, @@ -51221,7 +49261,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_vision": true, @@ -51236,7 +49275,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_vision": true, @@ -51251,7 +49289,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supports_function_calling": true, @@ -51265,7 +49302,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supports_vision": true, @@ -51278,7 +49314,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -51287,7 +49322,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -51296,7 +49330,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -51305,7 +49338,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 127000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -51314,7 +49346,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.5e-05 }, @@ -51323,7 +49354,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 127000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 5e-06 }, @@ -51332,7 +49362,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 127000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 8e-06 }, @@ -51341,7 +49370,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_vision": true, @@ -51353,7 +49381,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_vision": true, @@ -51365,7 +49392,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -51376,7 +49402,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1e-05, "supports_vision": true, @@ -51388,7 +49413,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -51399,7 +49423,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_function_calling": true @@ -51409,7 +49432,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "supports_function_calling": true, @@ -51420,7 +49442,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-06, "supports_function_calling": true, @@ -51431,7 +49452,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -51442,7 +49462,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.2e-06, "supports_function_calling": true, @@ -51453,7 +49472,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 96000, - "max_tokens": 96000, "mode": "chat", "output_cost_per_token": 1.1e-06, "supports_function_calling": true, @@ -51465,7 +49483,6 @@ "input_cost_per_token": 4.5e-07, "max_input_tokens": 200000, "max_output_tokens": 200000, - "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 1.8e-06, "source": "https://vercel.com/ai-gateway/models/glm-4.6", @@ -51516,7 +49533,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -51530,7 +49546,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -51550,7 +49565,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -51579,7 +49593,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -51602,7 +49615,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -51618,7 +49630,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -51633,7 +49644,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, "supports_assistant_prefill": true, @@ -51647,7 +49657,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, "supports_assistant_prefill": true, @@ -51661,7 +49670,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_assistant_prefill": true, @@ -51675,7 +49683,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_assistant_prefill": true, @@ -51688,7 +49695,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -51701,7 +49707,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -51721,7 +49726,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51756,7 +49760,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51795,7 +49798,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51833,7 +49835,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51870,7 +49871,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51908,7 +49908,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51946,7 +49945,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -51984,7 +49982,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -52028,7 +50025,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -52066,7 +50062,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -52111,7 +50106,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52150,7 +50144,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52187,7 +50180,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52228,7 +50220,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52271,7 +50262,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52310,7 +50300,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52350,7 +50339,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -52380,7 +50368,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -52419,7 +50406,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -52458,7 +50444,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -52482,7 +50467,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -52494,7 +50478,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -52506,7 +50489,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -52518,7 +50500,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -52530,7 +50511,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -52541,7 +50521,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -52552,7 +50531,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -52566,7 +50544,6 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 163840, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.7e-06, "output_cost_per_token_batches": 8.5e-07, @@ -52588,7 +50565,6 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 163840, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.68e-06, "output_cost_per_token_batches": 8.4e-07, @@ -52609,7 +50585,6 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 65336, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, "output_cost_per_token_batches": 2.7e-06, @@ -52632,7 +50607,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, @@ -52691,7 +50665,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -52726,7 +50699,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -52759,7 +50731,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, @@ -52787,7 +50758,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, @@ -52817,7 +50787,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -52857,7 +50826,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -52916,7 +50884,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -52977,7 +50944,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -53034,7 +51000,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -53048,7 +51013,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -53059,7 +51023,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -53070,7 +51033,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -53081,7 +51043,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -53092,7 +51053,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -53122,7 +51082,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "audio_speech", "output_cost_per_image": 0.04, "output_cost_per_token": 0, @@ -53158,7 +51117,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "audio_speech", "output_cost_per_image": 0.08, "output_cost_per_token": 0, @@ -53195,7 +51153,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.6e-05, "source": "https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama-3.2-90b-vision-instruct-maas", @@ -53208,7 +51165,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama-3.2-90b-vision-instruct-maas", @@ -53221,7 +51177,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "metadata": { "notes": "VertexAI states that The Llama 3.1 API service for llama-3.1-70b-instruct-maas and llama-3.1-8b-instruct-maas are in public preview and at no cost." }, @@ -53237,7 +51192,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "metadata": { "notes": "VertexAI states that The Llama 3.2 API service is at no cost during public preview, and will be priced as per dollar-per-1M-tokens at GA." }, @@ -53254,7 +51208,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.15e-06, "output_cost_per_token_batches": 5.75e-07, @@ -53276,7 +51229,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.15e-06, "output_cost_per_token_batches": 5.75e-07, @@ -53298,7 +51250,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 10000000, "max_output_tokens": 10000000, - "max_tokens": 10000000, "mode": "chat", "output_cost_per_token": 7e-07, "output_cost_per_token_batches": 3.5e-07, @@ -53320,7 +51271,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 10000000, "max_output_tokens": 10000000, - "max_tokens": 10000000, "mode": "chat", "output_cost_per_token": 7e-07, "output_cost_per_token_batches": 3.5e-07, @@ -53341,7 +51291,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", @@ -53352,7 +51301,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", @@ -53363,7 +51311,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", @@ -53376,7 +51323,6 @@ "litellm_provider": "vertex_ai-minimax_models", "max_input_tokens": 196608, "max_output_tokens": 196608, - "max_tokens": 196608, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53390,7 +51336,6 @@ "litellm_provider": "vertex_ai-moonshot_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53405,7 +51350,6 @@ "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53423,7 +51367,6 @@ "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53440,7 +51383,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53452,7 +51394,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53464,7 +51405,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53476,7 +51416,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53488,7 +51427,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -53499,7 +51437,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -53510,7 +51447,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -53521,7 +51457,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -53532,7 +51467,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -53543,7 +51477,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-07, "supports_function_calling": true, @@ -53554,7 +51487,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -53567,7 +51499,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -53601,7 +51532,6 @@ "litellm_provider": "vertex_ai-openai_models", "max_input_tokens": 262144, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/maas/google/gemma-4-26b-a4b-it", @@ -53618,7 +51548,6 @@ "litellm_provider": "vertex_ai-openai_models", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.6e-07, "output_cost_per_token_batches": 1.8e-07, @@ -53631,7 +51560,6 @@ "litellm_provider": "vertex_ai-openai_models", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.5e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53646,7 +51574,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53663,7 +51590,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53683,7 +51609,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 2000000, "max_output_tokens": 2000000, - "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -53703,7 +51628,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 2000000, "max_output_tokens": 2000000, - "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -53724,7 +51648,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 200000, "max_output_tokens": 200000, - "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -53744,7 +51667,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 524288, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -53763,7 +51685,6 @@ "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 8.8e-07, "output_cost_per_token_batches": 4.4e-07, @@ -53783,7 +51704,6 @@ "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.8e-06, "output_cost_per_token_batches": 9e-07, @@ -53800,7 +51720,6 @@ "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53816,7 +51735,6 @@ "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53829,7 +51747,6 @@ "vertex_ai/veo-3.1-generate-preview": { "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, @@ -53844,7 +51761,6 @@ "vertex_ai/veo-3.1-fast-generate-preview": { "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -53861,7 +51777,6 @@ "deprecation_date": "2026-11-17", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, @@ -53877,7 +51792,6 @@ "deprecation_date": "2026-11-17", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -53893,7 +51807,6 @@ "vertex_ai/veo-3.1-lite-generate-001": { "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.05, "output_cost_per_second_1080p": 0.08, @@ -53911,7 +51824,6 @@ "litellm_provider": "voyage", "max_input_tokens": 16000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -53920,7 +51832,6 @@ "litellm_provider": "voyage", "max_input_tokens": 8000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -53929,7 +51840,6 @@ "litellm_provider": "voyage", "max_input_tokens": 8000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.voyageai.com/docs/pricing" @@ -53939,7 +51849,6 @@ "litellm_provider": "voyage", "max_input_tokens": 4000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.voyageai.com/docs/pricing" @@ -53949,7 +51858,6 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -53958,7 +51866,6 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -53967,7 +51874,6 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.voyageai.com/docs/pricing" @@ -53977,7 +51883,6 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.voyageai.com/docs/pricing" @@ -53986,7 +51891,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "voyage", "max_input_tokens": 4000, - "max_tokens": 4000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -53994,7 +51898,6 @@ "input_cost_per_token": 6e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54002,7 +51905,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -54012,7 +51914,6 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54020,7 +51921,6 @@ "input_cost_per_token": 2e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54028,7 +51928,6 @@ "input_cost_per_token": 6e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54036,7 +51935,6 @@ "input_cost_per_token": 2e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54044,7 +51942,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 16000, - "max_tokens": 16000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54052,7 +51949,6 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54060,7 +51956,6 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "voyage", "max_input_tokens": 120000, - "max_tokens": 120000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54068,7 +51963,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54076,7 +51970,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 16000, - "max_tokens": 16000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54084,7 +51977,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 16000, - "max_tokens": 16000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -54094,7 +51986,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 16000, - "max_tokens": 16000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54102,7 +51993,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "voyage", "max_input_tokens": 4096, - "max_tokens": 4096, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54110,7 +52000,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "voyage", "max_input_tokens": 4000, - "max_tokens": 4000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54118,13 +52007,11 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, "wandb/openai/gpt-oss-120b": { "supports_reasoning": true, - "max_tokens": 131072, "max_input_tokens": 131000, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, @@ -54135,7 +52022,6 @@ }, "wandb/openai/gpt-oss-20b": { "supports_reasoning": true, - "max_tokens": 131072, "max_input_tokens": 131000, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, @@ -54145,7 +52031,6 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/moonshotai/Kimi-K2.5": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "cache_read_input_token_cost": 1e-07, @@ -54160,7 +52045,6 @@ "supports_vision": true }, "wandb/meta-llama/Llama-3.1-8B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 131000, "max_output_tokens": 128000, "input_cost_per_token": 2.2e-07, @@ -54171,7 +52055,6 @@ }, "wandb/deepseek-ai/DeepSeek-V3.1": { "supports_reasoning": true, - "max_tokens": 128000, "max_input_tokens": 161000, "max_output_tokens": 128000, "input_cost_per_token": 5.5e-07, @@ -54181,7 +52064,6 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/meta-llama/Llama-3.3-70B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 7.1e-07, @@ -54195,7 +52077,6 @@ "litellm_provider": "watsonx", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 2e-07, "supports_audio_input": false, @@ -54213,7 +52094,6 @@ "litellm_provider": "watsonx", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_audio_input": false, @@ -54227,7 +52107,6 @@ "supports_vision": false }, "watsonx/bigscience/mt0-xxl-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.908e-06, @@ -54240,7 +52119,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/bigscience/mt0-xxl": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.908e-06, @@ -54253,7 +52131,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/core42/jais-13b-chat": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 0.0005, @@ -54265,7 +52142,6 @@ "supports_vision": false }, "watsonx/google/flan-t5-xl-3b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 6e-07, @@ -54277,7 +52153,6 @@ "supports_vision": false }, "watsonx/ibm/granite-13b-chat-v2": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 6e-07, @@ -54289,7 +52164,6 @@ "supports_vision": false }, "watsonx/ibm/granite-13b-instruct-v2": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 6e-07, @@ -54301,7 +52175,6 @@ "supports_vision": false }, "watsonx/ibm/granite-3-3-8b-instruct": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -54313,7 +52186,6 @@ "supports_vision": false }, "watsonx/ibm/granite-4-h-small": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6.36e-08, @@ -54326,7 +52198,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/ibm/granite-guardian-3-2-2b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -54338,7 +52209,6 @@ "supports_vision": false }, "watsonx/ibm/granite-guardian-3-3-8b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -54350,7 +52220,6 @@ "supports_vision": false }, "watsonx/ibm/granite-ttm-1024-96-r2": { - "max_tokens": 512, "max_input_tokens": 512, "max_output_tokens": 512, "input_cost_per_token": 3.8e-07, @@ -54362,7 +52231,6 @@ "supports_vision": false }, "watsonx/ibm/granite-ttm-1536-96-r2": { - "max_tokens": 512, "max_input_tokens": 512, "max_output_tokens": 512, "input_cost_per_token": 3.8e-07, @@ -54374,7 +52242,6 @@ "supports_vision": false }, "watsonx/ibm/granite-ttm-512-96-r2": { - "max_tokens": 512, "max_input_tokens": 512, "max_output_tokens": 512, "input_cost_per_token": 3.8e-07, @@ -54386,7 +52253,6 @@ "supports_vision": false }, "watsonx/ibm/granite-vision-3-2-2b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -54398,7 +52264,6 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-2-11b-vision-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 3.5e-07, @@ -54410,7 +52275,6 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-2-1b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1e-07, @@ -54422,7 +52286,6 @@ "supports_vision": false }, "watsonx/meta-llama/llama-3-2-3b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.5e-07, @@ -54434,7 +52297,6 @@ "supports_vision": false }, "watsonx/meta-llama/llama-3-2-90b-vision-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-06, @@ -54446,7 +52308,6 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-3-70b-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 7.526e-07, @@ -54459,7 +52320,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b": { - "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 3.71e-07, @@ -54472,7 +52332,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b-128e-instruct-fp8": { - "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 3.71e-07, @@ -54485,7 +52344,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-guard-3-11b-vision": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 3.5e-07, @@ -54497,7 +52355,6 @@ "supports_vision": true }, "watsonx/mistralai/mistral-medium-2505": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, @@ -54509,7 +52366,6 @@ "supports_vision": false }, "watsonx/mistralai/mistral-small-2503": { - "max_tokens": 32000, "max_input_tokens": 32000, "max_output_tokens": 32000, "input_cost_per_token": 1e-07, @@ -54521,7 +52377,6 @@ "supports_vision": false }, "watsonx/mistralai/mistral-small-3-1-24b-instruct-2503": { - "max_tokens": 16384, "max_input_tokens": 131072, "max_output_tokens": 16384, "input_cost_per_token": 1.06e-07, @@ -54534,7 +52389,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/mistralai/pixtral-12b-2409": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 3.5e-07, @@ -54546,7 +52400,6 @@ "supports_vision": true }, "watsonx/openai/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.59e-07, @@ -54559,7 +52412,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/sdaia/allam-1-13b-instruct": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1.8e-06, @@ -54596,7 +52448,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -54621,7 +52472,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -54645,7 +52495,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -54671,7 +52520,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -54699,7 +52547,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -54727,7 +52574,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -54751,7 +52597,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54773,7 +52618,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54795,7 +52639,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54817,7 +52660,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54839,7 +52681,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54861,7 +52702,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, "output_cost_per_token_batches": 1.1e-06, @@ -54884,7 +52724,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_batches": 1.6e-06, "output_cost_per_token_priority": 5.6e-06, @@ -54907,7 +52746,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -55505,7 +53343,6 @@ } }, "fireworks_ai/accounts/fireworks/models/qwen3-coder-480b-a35b-instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 4.5e-07, @@ -55515,7 +53352,6 @@ "supports_reasoning": true }, "fireworks_ai/accounts/fireworks/models/flux-kontext-pro": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 4e-08, @@ -55524,7 +53360,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/SSD-1B": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -55533,7 +53368,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/chronos-hermes-13b-v2": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55542,7 +53376,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-13b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55551,7 +53384,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-13b-instruct": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55560,7 +53392,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-13b-python": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55569,7 +53400,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-34b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -55578,7 +53408,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-34b-instruct": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -55587,7 +53416,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-34b-python": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -55596,7 +53424,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-70b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -55605,7 +53432,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-70b-instruct": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -55614,7 +53440,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-70b-python": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -55623,7 +53448,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-7b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55632,7 +53456,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-7b-instruct": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55641,7 +53464,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-7b-python": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55650,7 +53472,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-qwen-1p5-7b": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 2e-07, @@ -55659,7 +53480,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/codegemma-2b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -55668,7 +53488,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/codegemma-7b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -55677,7 +53496,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-671b-v2-p1": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 1.2e-06, @@ -55686,7 +53504,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-llama-3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -55695,7 +53512,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-llama-70b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55704,7 +53520,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-llama-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55713,7 +53528,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-qwen-14b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55722,7 +53536,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-qwen-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55731,7 +53544,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/flux-kontext-max": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 8e-08, @@ -55740,7 +53552,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/dbrx-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1.2e-06, @@ -55749,7 +53560,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-1b-base": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 1e-07, @@ -55758,7 +53568,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-33b-instruct": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -55767,7 +53576,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-7b-base": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55776,7 +53584,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-7b-base-v1p5": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55785,7 +53592,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-7b-instruct-v1p5": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55794,7 +53600,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-v2-lite-base": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 5e-07, @@ -55803,7 +53608,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-v2-lite-instruct": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 5e-07, @@ -55812,7 +53616,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-prover-v2": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 1.2e-06, @@ -55821,7 +53624,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-0528-distill-qwen3-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55830,7 +53632,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-llama-70b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55839,7 +53640,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-llama-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55848,7 +53648,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-qwen-14b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55857,7 +53656,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-qwen-1p5b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -55866,7 +53664,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-qwen-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55875,7 +53672,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-qwen-7b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55884,7 +53680,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-v2-lite-chat": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 5e-07, @@ -55893,7 +53688,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-v2p5": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1.2e-06, @@ -55902,7 +53696,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/devstral-small-2505": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55911,7 +53704,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/dobby-mini-unhinged-plus-llama-3-1-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55920,7 +53712,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/dobby-unhinged-llama-3-3-70b-new": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55929,7 +53720,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/dolphin-2-9-2-qwen2-72b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55938,7 +53728,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/dolphin-2p6-mixtral-8x7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -55947,7 +53736,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ernie-4p5-21b-a3b-pt": { - "max_tokens": 4096, "max_input_tokens": 131072, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -55956,7 +53744,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ernie-4p5-300b-a47b-pt": { - "max_tokens": 4096, "max_input_tokens": 131072, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -55965,7 +53752,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/fare-20b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55974,7 +53760,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/firefunction-v1": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -55983,7 +53768,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/firellava-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55992,7 +53776,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/firesearch-ocr-v6": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56001,7 +53784,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/fireworks-asr-large": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 0.0, @@ -56010,7 +53792,6 @@ "mode": "audio_transcription" }, "fireworks_ai/accounts/fireworks/models/fireworks-asr-v2": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 0.0, @@ -56019,7 +53800,6 @@ "mode": "audio_transcription" }, "fireworks_ai/accounts/fireworks/models/flux-1-dev": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56028,7 +53808,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/flux-1-dev-controlnet-union": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-09, @@ -56037,7 +53816,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/flux-1-dev-fp8": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 5e-10, @@ -56046,7 +53824,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/flux-1-schnell": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56055,7 +53832,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/flux-1-schnell-fp8": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 3.5e-10, @@ -56064,7 +53840,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/gemma-2b-it": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -56073,7 +53848,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gemma-3-27b-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56082,7 +53856,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gemma-7b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56091,7 +53864,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gemma-7b-it": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56100,7 +53872,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gemma2-9b-it": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56109,7 +53880,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/glm-4p5v": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.2e-06, @@ -56119,7 +53889,6 @@ "supports_reasoning": true }, "fireworks_ai/accounts/fireworks/models/gpt-oss-safeguard-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.2e-06, @@ -56128,7 +53897,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gpt-oss-safeguard-20b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-07, @@ -56137,7 +53905,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/hermes-2-pro-mistral-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56146,7 +53913,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/internvl3-38b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56155,7 +53921,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/internvl3-78b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56164,7 +53929,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/internvl3-8b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -56173,7 +53937,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/japanese-stable-diffusion-xl": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -56182,7 +53945,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/kat-coder": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-07, @@ -56191,7 +53953,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/kat-dev-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56200,7 +53961,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/kat-dev-72b-exp": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56209,7 +53969,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-guard-2-8b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56218,7 +53977,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-guard-3-1b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -56227,7 +53985,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-guard-3-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56236,7 +53993,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56245,7 +54001,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-13b-chat": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56254,7 +54009,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-70b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56263,7 +54017,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-70b-chat": { - "max_tokens": 2048, "max_input_tokens": 2048, "max_output_tokens": 2048, "input_cost_per_token": 9e-07, @@ -56272,7 +54025,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-7b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56281,7 +54033,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-7b-chat": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56290,7 +54041,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 9e-07, @@ -56299,7 +54049,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct-hf": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 9e-07, @@ -56308,7 +54057,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3-8b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56317,7 +54065,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3-8b-instruct-hf": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56326,7 +54073,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct-long": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56335,7 +54081,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56344,7 +54089,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct-1b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56353,7 +54097,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-nemotron-70b-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56362,7 +54105,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p2-1b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -56371,7 +54113,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p2-3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -56380,7 +54121,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56389,7 +54129,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llamaguard-7b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56398,7 +54137,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llava-yi-34b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -56407,7 +54145,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/minimax-m1-80k": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56416,7 +54153,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/minimax-m2": { - "max_tokens": 4096, "max_input_tokens": 196608, "max_output_tokens": 4096, "input_cost_per_token": 3e-07, @@ -56425,7 +54161,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ministral-3-14b-instruct-2512": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 2e-07, @@ -56434,7 +54169,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ministral-3-3b-instruct-2512": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 1e-07, @@ -56443,7 +54177,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ministral-3-8b-instruct-2512": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 2e-07, @@ -56452,7 +54185,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56461,7 +54193,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b-instruct-4k": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56470,7 +54201,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b-instruct-v0p2": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56479,7 +54209,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b-instruct-v3": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56488,7 +54217,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b-v0p2": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56497,7 +54225,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-large-3-fp8": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 1.2e-06, @@ -56506,7 +54233,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-nemo-base-2407": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -56515,7 +54241,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-nemo-instruct-2407": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -56524,7 +54249,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-small-24b-instruct-2501": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56533,7 +54257,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 1.2e-06, @@ -56542,7 +54265,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 1.2e-06, @@ -56551,7 +54273,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -56560,7 +54281,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -56569,7 +54289,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x7b-instruct-hf": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -56578,7 +54297,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mythomax-l2-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56587,7 +54305,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nemotron-nano-v2-12b-vl": { - "max_tokens": 4096, "max_input_tokens": 131072, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56596,7 +54313,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-capybara-7b-v1p9": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56605,7 +54321,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-2-mixtral-8x7b-dpo": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -56614,7 +54329,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-2-yi-34b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -56623,7 +54337,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-llama2-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56632,7 +54345,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-llama2-70b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -56641,7 +54353,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-llama2-7b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56650,7 +54361,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nvidia-nemotron-nano-12b-v2": { - "max_tokens": 131072, "max_input_tokens": 128000, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56659,7 +54369,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nvidia-nemotron-nano-9b-v2": { - "max_tokens": 131072, "max_input_tokens": 128000, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56668,7 +54377,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/openchat-3p5-0106-7b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56677,7 +54385,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/openhermes-2-mistral-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56686,7 +54393,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/openhermes-2p5-mistral-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56695,7 +54401,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/openorca-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56704,7 +54409,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phi-2-3b": { - "max_tokens": 2048, "max_input_tokens": 2048, "max_output_tokens": 2048, "input_cost_per_token": 1e-07, @@ -56713,7 +54417,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phi-3-mini-128k-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -56722,7 +54425,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phi-3-vision-128k-instruct": { - "max_tokens": 32064, "max_input_tokens": 32064, "max_output_tokens": 32064, "input_cost_per_token": 2e-07, @@ -56731,7 +54433,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phind-code-llama-34b-python-v1": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56740,7 +54441,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phind-code-llama-34b-v1": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56749,7 +54449,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phind-code-llama-34b-v2": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56758,7 +54457,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/playground-v2-1024px-aesthetic": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -56767,7 +54465,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/playground-v2-5-1024px-aesthetic": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -56776,7 +54473,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/pythia-12b": { - "max_tokens": 2048, "max_input_tokens": 2048, "max_output_tokens": 2048, "input_cost_per_token": 2e-07, @@ -56785,7 +54481,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen-qwq-32b-preview": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56794,7 +54489,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen-v2p5-14b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56803,7 +54497,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen-v2p5-7b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56812,7 +54505,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen1p5-72b-chat": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56821,7 +54513,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56830,7 +54521,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2-vl-2b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56839,7 +54529,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2-vl-72b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56848,7 +54537,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2-vl-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56857,7 +54545,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-0p5b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56866,7 +54553,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-14b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56875,7 +54561,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-1p5b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56884,7 +54569,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56893,7 +54577,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-32b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56902,7 +54585,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-72b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56911,7 +54593,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-72b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56920,7 +54601,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56929,7 +54609,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-0p5b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56938,7 +54617,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-0p5b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56947,7 +54625,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-14b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56956,7 +54633,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-14b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56965,7 +54641,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-1p5b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56974,7 +54649,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-1p5b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56983,7 +54657,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56992,7 +54665,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct-128k": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -57001,7 +54673,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct-32k-rope": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -57010,7 +54681,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct-64k": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 9e-07, @@ -57019,7 +54689,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-3b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -57028,7 +54697,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-3b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -57037,7 +54705,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57046,7 +54713,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57055,7 +54721,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-math-72b-instruct": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57064,7 +54729,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-vl-32b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 9e-07, @@ -57073,7 +54737,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-vl-3b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -57082,7 +54745,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-vl-72b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 9e-07, @@ -57091,7 +54753,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-vl-7b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -57100,7 +54761,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-0p6b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1e-07, @@ -57109,7 +54769,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-14b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 2e-07, @@ -57118,7 +54777,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-1p7b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -57127,7 +54785,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-1p7b-fp8-draft": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1e-07, @@ -57136,7 +54793,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-1p7b-fp8-draft-131072": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -57145,7 +54801,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-1p7b-fp8-draft-40960": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1e-07, @@ -57154,7 +54809,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-235b-a22b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2.2e-07, @@ -57163,7 +54817,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-235b-a22b-instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.2e-07, @@ -57172,7 +54825,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-235b-a22b-thinking-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.2e-07, @@ -57181,7 +54833,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-30b-a3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.5e-07, @@ -57190,7 +54841,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-30b-a3b-instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 5e-07, @@ -57199,7 +54849,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-30b-a3b-thinking-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-07, @@ -57208,7 +54857,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -57218,7 +54866,6 @@ "supports_reasoning": true }, "fireworks_ai/accounts/fireworks/models/qwen3-4b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 2e-07, @@ -57227,7 +54874,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-4b-instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2e-07, @@ -57236,7 +54882,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-8b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 2e-07, @@ -57246,7 +54891,6 @@ "supports_reasoning": true }, "fireworks_ai/accounts/fireworks/models/qwen3-coder-30b-a3b-instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -57255,7 +54899,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-coder-480b-instruct-bf16": { - "max_tokens": 4096, "max_input_tokens": 262144, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57264,7 +54907,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-embedding-0p6b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 0.0, @@ -57273,7 +54915,6 @@ "mode": "embedding" }, "fireworks_ai/accounts/fireworks/models/qwen3-embedding-4b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 0.0, @@ -57282,7 +54923,6 @@ "mode": "embedding" }, "fireworks_ai/accounts/fireworks/models/qwen3-next-80b-a3b-instruct": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57291,7 +54931,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-next-80b-a3b-thinking": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57300,7 +54939,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-reranker-0p6b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 0.0, @@ -57309,7 +54947,6 @@ "mode": "rerank" }, "fireworks_ai/accounts/fireworks/models/qwen3-reranker-4b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 0.0, @@ -57318,7 +54955,6 @@ "mode": "rerank" }, "fireworks_ai/accounts/fireworks/models/qwen3-reranker-8b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 2e-07, @@ -57328,7 +54964,6 @@ "source": "https://api.fireworks.ai/v1/serverless/models" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-235b-a22b-instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.2e-07, @@ -57337,7 +54972,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-235b-a22b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.2e-07, @@ -57346,7 +54980,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-30b-a3b-instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -57355,7 +54988,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-30b-a3b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -57364,7 +54996,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-32b-instruct": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57373,7 +55004,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-8b-instruct": { - "max_tokens": 4096, "max_input_tokens": 262144, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -57387,7 +55017,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -57398,7 +55027,6 @@ "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/qwq-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -57407,7 +55035,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/rolm-ocr": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -57416,7 +55043,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/snorkel-mistral-7b-pairrm-dpo": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57425,7 +55051,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/stable-diffusion-xl-1024-v1-0": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -57434,7 +55059,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/stablecode-3b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -57443,7 +55067,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder-16b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -57452,7 +55075,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder-7b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -57461,7 +55083,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder2-15b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -57470,7 +55091,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder2-3b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 1e-07, @@ -57479,7 +55099,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder2-7b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -57488,7 +55107,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/toppy-m-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57497,7 +55115,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/yi-34b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57506,7 +55123,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/yi-34b-200k-capybara": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 9e-07, @@ -57515,7 +55131,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/yi-34b-chat": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57524,7 +55139,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/yi-6b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -57533,7 +55147,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/zephyr-7b-beta": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57547,7 +55160,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -57564,7 +55176,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -57581,7 +55192,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -57596,7 +55206,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-06, "supports_function_calling": true, @@ -57608,7 +55217,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -57620,7 +55228,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.25e-06, "supports_function_calling": true @@ -57636,7 +55243,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -57647,7 +55253,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true @@ -57663,7 +55268,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 5e-07, "supports_function_calling": true, @@ -57675,7 +55279,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.5e-06, "supports_reasoning": true, @@ -57688,7 +55291,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.5e-07, "supports_function_calling": true, @@ -57699,7 +55301,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-07, "supports_vision": true, @@ -57717,7 +55318,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 9e-07, "supports_function_calling": true @@ -57729,7 +55329,6 @@ "output_cost_per_token": 4e-07, "max_input_tokens": 163840, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57746,7 +55345,6 @@ "output_cost_per_token": 1.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57762,7 +55360,6 @@ "output_cost_per_token": 2.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57779,7 +55376,6 @@ "output_cost_per_token": 3.3e-07, "max_input_tokens": 262144, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57796,7 +55392,6 @@ "output_cost_per_token": 1.38e-07, "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_vision": true, "supports_system_messages": true }, @@ -57807,7 +55402,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 262144, "max_output_tokens": 100352, - "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57824,7 +55418,6 @@ "output_cost_per_token": 1.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57841,7 +55434,6 @@ "output_cost_per_token": 2e-08, "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_vision": true, "supports_system_messages": true }, @@ -57852,7 +55444,6 @@ "output_cost_per_token": 4.1e-07, "max_input_tokens": 163840, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57867,7 +55458,6 @@ "output_cost_per_token": 3.95e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_vision": true, "supports_system_messages": true, "supports_reasoning": true, @@ -57881,7 +55471,6 @@ "output_cost_per_token": 9e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57899,7 +55488,6 @@ "output_cost_per_token": 2.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57916,7 +55504,6 @@ "output_cost_per_token": 1.2e-06, "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57932,7 +55519,6 @@ "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57947,7 +55533,6 @@ "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57962,7 +55547,6 @@ "output_cost_per_token": 3e-08, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_vision": true, "supports_system_messages": true, @@ -57975,7 +55559,6 @@ "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57992,7 +55575,6 @@ "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58007,7 +55589,6 @@ "output_cost_per_token": 8.45e-06, "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58021,7 +55602,6 @@ "output_cost_per_token": 6e-07, "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_tool_choice": true, "supports_vision": true, "supports_system_messages": true, @@ -58034,7 +55614,6 @@ "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58051,7 +55630,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 262144, "max_output_tokens": 100352, - "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58065,7 +55643,6 @@ "output_cost_per_token": 1.55e-06, "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58079,7 +55656,6 @@ "output_cost_per_token": 2.7e-07, "max_input_tokens": 160000, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58093,7 +55669,6 @@ "output_cost_per_token": 2.5e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58109,7 +55684,6 @@ "output_cost_per_token": 2.3e-06, "max_input_tokens": 131072, "max_output_tokens": 100352, - "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58123,7 +55697,6 @@ "output_cost_per_token": 1.12e-06, "max_input_tokens": 163840, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58139,7 +55712,6 @@ "output_cost_per_token": 2.2e-06, "max_input_tokens": 131072, "max_output_tokens": 98304, - "max_tokens": 98304, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58155,7 +55727,6 @@ "output_cost_per_token": 3e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58169,7 +55740,6 @@ "output_cost_per_token": 5e-08, "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_system_messages": true, "supports_response_schema": true }, @@ -58180,7 +55750,6 @@ "output_cost_per_token": 1e-07, "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_vision": true, "supports_system_messages": true, @@ -58193,7 +55762,6 @@ "output_cost_per_token": 1.8e-06, "max_input_tokens": 65536, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58211,7 +55779,6 @@ "output_cost_per_token": 1.5e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_tool_choice": true, "supports_vision": false, "supports_system_messages": true, @@ -58225,7 +55792,6 @@ "output_cost_per_token": 5.8e-07, "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58239,7 +55805,6 @@ "output_cost_per_token": 1.5e-07, "max_input_tokens": 32768, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, @@ -58252,7 +55817,6 @@ "output_cost_per_token": 4e-07, "max_input_tokens": 12288, "max_output_tokens": 12288, - "max_tokens": 12288, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58266,7 +55830,6 @@ "output_cost_per_token": 4e-07, "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58280,7 +55843,6 @@ "output_cost_per_token": 1.7e-07, "max_input_tokens": 60288, "max_output_tokens": 16000, - "max_tokens": 16000, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58292,7 +55854,6 @@ "output_cost_per_token": 2.2e-06, "max_input_tokens": 1000000, "max_output_tokens": 40000, - "max_tokens": 40000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58307,7 +55868,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 163840, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58324,7 +55884,6 @@ "output_cost_per_token": 3e-07, "max_input_tokens": 64000, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, @@ -58337,7 +55896,6 @@ "output_cost_per_token": 4e-08, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_system_messages": true, "supports_response_schema": true }, @@ -58348,7 +55906,6 @@ "output_cost_per_token": 6.2e-07, "max_input_tokens": 65535, "max_output_tokens": 8000, - "max_tokens": 8000, "supports_system_messages": true, "supports_response_schema": true }, @@ -58359,7 +55916,6 @@ "output_cost_per_token": 9e-08, "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_system_messages": true, "supports_reasoning": true }, @@ -58370,7 +55926,6 @@ "output_cost_per_token": 8e-07, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, @@ -58383,7 +55938,6 @@ "output_cost_per_token": 7.4e-07, "max_input_tokens": 8192, "max_output_tokens": 8000, - "max_tokens": 8000, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58395,7 +55949,6 @@ "output_cost_per_token": 8e-07, "max_input_tokens": 40960, "max_output_tokens": 20000, - "max_tokens": 20000, "supports_system_messages": true, "supports_reasoning": true, "supports_response_schema": true @@ -58407,7 +55960,6 @@ "output_cost_per_token": 8.5e-07, "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_vision": true, "supports_system_messages": true, "supports_response_schema": true @@ -58419,7 +55971,6 @@ "output_cost_per_token": 5.9e-07, "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_vision": true, "supports_system_messages": true }, @@ -58430,7 +55981,6 @@ "output_cost_per_token": 1.4e-07, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58442,7 +55992,6 @@ "output_cost_per_token": 8e-07, "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_vision": true, "supports_system_messages": true }, @@ -58453,7 +56002,6 @@ "output_cost_per_token": 1.48e-06, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58466,7 +56014,6 @@ "output_cost_per_token": 2.8e-07, "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_system_messages": true, "supports_reasoning": true }, @@ -58477,7 +56024,6 @@ "output_cost_per_token": 5e-08, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58489,7 +56035,6 @@ "output_cost_per_token": 7e-08, "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_system_messages": true }, "novita/baidu/ernie-4.5-vl-424b-a47b": { @@ -58499,7 +56044,6 @@ "output_cost_per_token": 1.25e-06, "max_input_tokens": 123000, "max_output_tokens": 16000, - "max_tokens": 16000, "supports_vision": true, "supports_system_messages": true, "supports_reasoning": true @@ -58511,7 +56055,6 @@ "output_cost_per_token": 1.1e-06, "max_input_tokens": 123000, "max_output_tokens": 12000, - "max_tokens": 12000, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58523,7 +56066,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 160000, "max_output_tokens": 160000, - "max_tokens": 160000, "supports_system_messages": true }, "novita/qwen/qwen3-32b-fp8": { @@ -58533,7 +56075,6 @@ "output_cost_per_token": 4.5e-07, "max_input_tokens": 40960, "max_output_tokens": 20000, - "max_tokens": 20000, "supports_system_messages": true, "supports_reasoning": true }, @@ -58544,7 +56085,6 @@ "output_cost_per_token": 4.5e-07, "max_input_tokens": 40960, "max_output_tokens": 20000, - "max_tokens": 20000, "supports_system_messages": true, "supports_reasoning": true, "supports_function_calling": true, @@ -58557,7 +56097,6 @@ "output_cost_per_token": 2e-07, "max_input_tokens": 98304, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_vision": true, "supports_system_messages": true }, @@ -58568,7 +56107,6 @@ "output_cost_per_token": 1.3e-06, "max_input_tokens": 64000, "max_output_tokens": 16000, - "max_tokens": 16000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58581,7 +56119,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 64000, "max_output_tokens": 16000, - "max_tokens": 16000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58596,7 +56133,6 @@ "output_cost_per_token": 5e-08, "max_input_tokens": 8192, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58609,7 +56145,6 @@ "output_cost_per_token": 9e-08, "max_input_tokens": 4096, "max_output_tokens": 3200, - "max_tokens": 3200, "supports_system_messages": true }, "novita/baidu/ernie-4.5-vl-28b-a3b-thinking": { @@ -58619,7 +56154,6 @@ "output_cost_per_token": 3.9e-07, "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58635,7 +56169,6 @@ "output_cost_per_token": 5e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58650,7 +56183,6 @@ "output_cost_per_token": 8.5e-07, "max_input_tokens": 131072, "max_output_tokens": 98304, - "max_tokens": 98304, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58666,7 +56198,6 @@ "output_cost_per_token": 7e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58681,7 +56212,6 @@ "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58696,7 +56226,6 @@ "output_cost_per_token": 9.7e-07, "max_input_tokens": 65536, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58713,7 +56242,6 @@ "output_cost_per_token": 9.7e-07, "max_input_tokens": 65536, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58730,7 +56258,6 @@ "output_cost_per_token": 7.5e-07, "max_input_tokens": 16384, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_system_messages": true }, "novita/baidu/ernie-4.5-vl-28b-a3b": { @@ -58740,7 +56267,6 @@ "output_cost_per_token": 5.6e-07, "max_input_tokens": 30000, "max_output_tokens": 8000, - "max_tokens": 8000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58755,7 +56281,6 @@ "output_cost_per_token": 2.8e-07, "max_input_tokens": 120000, "max_output_tokens": 8000, - "max_tokens": 8000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58768,7 +56293,6 @@ "output_cost_per_token": 1.38e-07, "max_input_tokens": 128000, "max_output_tokens": 20000, - "max_tokens": 20000, "supports_system_messages": true, "supports_reasoning": true }, @@ -58779,7 +56303,6 @@ "output_cost_per_token": 3e-08, "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_system_messages": true, "supports_reasoning": true, "supports_function_calling": true, @@ -58792,7 +56315,6 @@ "output_cost_per_token": 7e-08, "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58806,7 +56328,6 @@ "output_cost_per_token": 5e-08, "max_input_tokens": 32768, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58819,7 +56340,6 @@ "output_cost_per_token": 1.48e-06, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58831,8 +56351,7 @@ "input_cost_per_token": 7e-08, "output_cost_per_token": 0, "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768 + "max_output_tokens": 32768 }, "novita/qwen/qwen3-embedding-8b": { "litellm_provider": "novita", @@ -58840,8 +56359,7 @@ "input_cost_per_token": 7e-08, "output_cost_per_token": 0, "max_input_tokens": 32768, - "max_output_tokens": 4096, - "max_tokens": 4096 + "max_output_tokens": 4096 }, "novita/baai/bge-m3": { "litellm_provider": "novita", @@ -58849,8 +56367,7 @@ "input_cost_per_token": 1e-08, "output_cost_per_token": 1e-08, "max_input_tokens": 8192, - "max_output_tokens": 96000, - "max_tokens": 96000 + "max_output_tokens": 96000 }, "novita/qwen/qwen3-reranker-8b": { "litellm_provider": "novita", @@ -58858,8 +56375,7 @@ "input_cost_per_token": 5e-08, "output_cost_per_token": 5e-08, "max_input_tokens": 32768, - "max_output_tokens": 4096, - "max_tokens": 4096 + "max_output_tokens": 4096 }, "novita/baai/bge-reranker-v2-m3": { "litellm_provider": "novita", @@ -58867,11 +56383,9 @@ "input_cost_per_token": 1e-08, "output_cost_per_token": 1e-08, "max_input_tokens": 8000, - "max_output_tokens": 8000, - "max_tokens": 8000 + "max_output_tokens": 8000 }, "llamagate/llama-3.1-8b": { - "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 3e-08, @@ -58882,7 +56396,6 @@ "supports_response_schema": true }, "llamagate/llama-3.2-3b": { - "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 4e-08, @@ -58893,7 +56406,6 @@ "supports_response_schema": true }, "llamagate/mistral-7b-v0.3": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -58904,7 +56416,6 @@ "supports_response_schema": true }, "llamagate/qwen3-8b": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 4e-08, @@ -58915,7 +56426,6 @@ "supports_response_schema": true }, "llamagate/dolphin3-8b": { - "max_tokens": 8192, "max_input_tokens": 128000, "max_output_tokens": 8192, "input_cost_per_token": 8e-08, @@ -58926,7 +56436,6 @@ "supports_response_schema": true }, "llamagate/deepseek-r1-8b": { - "max_tokens": 16384, "max_input_tokens": 65536, "max_output_tokens": 16384, "input_cost_per_token": 1e-07, @@ -58938,7 +56447,6 @@ "supports_reasoning": true }, "llamagate/deepseek-r1-7b-qwen": { - "max_tokens": 16384, "max_input_tokens": 131072, "max_output_tokens": 16384, "input_cost_per_token": 8e-08, @@ -58950,7 +56458,6 @@ "supports_reasoning": true }, "llamagate/openthinker-7b": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 8e-08, @@ -58962,7 +56469,6 @@ "supports_reasoning": true }, "llamagate/qwen2.5-coder-7b": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 6e-08, @@ -58973,7 +56479,6 @@ "supports_response_schema": true }, "llamagate/deepseek-coder-6.7b": { - "max_tokens": 4096, "max_input_tokens": 16384, "max_output_tokens": 4096, "input_cost_per_token": 6e-08, @@ -58984,7 +56489,6 @@ "supports_response_schema": true }, "llamagate/codellama-7b": { - "max_tokens": 4096, "max_input_tokens": 16384, "max_output_tokens": 4096, "input_cost_per_token": 6e-08, @@ -58995,7 +56499,6 @@ "supports_response_schema": true }, "llamagate/qwen3-vl-8b": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 1.5e-07, @@ -59007,7 +56510,6 @@ "supports_vision": true }, "llamagate/llava-7b": { - "max_tokens": 2048, "max_input_tokens": 4096, "max_output_tokens": 2048, "input_cost_per_token": 1e-07, @@ -59018,7 +56520,6 @@ "supports_vision": true }, "llamagate/gemma3-4b": { - "max_tokens": 8192, "max_input_tokens": 128000, "max_output_tokens": 8192, "input_cost_per_token": 3e-08, @@ -59030,7 +56531,6 @@ "supports_vision": true }, "llamagate/nomic-embed-text": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 2e-08, "output_cost_per_token": 0, @@ -59038,7 +56538,6 @@ "mode": "embedding" }, "llamagate/qwen3-embedding-8b": { - "max_tokens": 40960, "max_input_tokens": 40960, "input_cost_per_token": 2e-08, "output_cost_per_token": 0, @@ -59046,7 +56545,6 @@ "mode": "embedding" }, "libertai/hermes-3-8b-tee": { - "max_tokens": 16000, "max_input_tokens": 16000, "max_output_tokens": 16000, "input_cost_per_token": 1.5e-07, @@ -59060,7 +56558,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/gemma-4-31b-it": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59074,7 +56571,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/gemma-4-31b-it-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59089,7 +56585,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.6-27b": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59103,7 +56598,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.6-27b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59118,7 +56612,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.6-35b-a3b": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59132,7 +56625,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.6-35b-a3b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59147,7 +56639,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.5-122b-a10b": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.5e-07, @@ -59161,7 +56652,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.5-122b-a10b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.5e-07, @@ -59176,7 +56666,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/deepseek-v4-flash": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 2.5e-07, @@ -59190,7 +56679,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/deepseek-v4-flash-thinking": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 2.5e-07, @@ -59205,7 +56693,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/bge-m3": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -59221,7 +56708,6 @@ "litellm_provider": "sarvam", "max_input_tokens": 8192, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 0, "supports_reasoning": true @@ -59297,7 +56783,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -59325,7 +56810,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -59357,7 +56841,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -59467,7 +56950,6 @@ "litellm_provider": "gemini", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_token": 0.0, @@ -59488,7 +56970,6 @@ "litellm_provider": "gemini", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_token": 0.0, @@ -59511,7 +56992,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59537,7 +57017,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59564,7 +57043,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59599,7 +57077,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -59634,7 +57111,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -59668,7 +57144,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -59701,7 +57176,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59729,7 +57203,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59758,7 +57231,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59796,7 +57268,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -59833,7 +57304,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, @@ -59860,7 +57330,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 9e-06, "output_cost_per_token": 9e-06, @@ -59885,7 +57354,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 6e-06, "output_cost_per_token": 6e-06, @@ -59905,7 +57373,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, @@ -59925,7 +57392,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -59984,7 +57450,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -60044,7 +57509,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -60106,7 +57570,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -60167,7 +57630,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -60218,7 +57680,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -60257,7 +57718,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -60299,7 +57759,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1.05e-06, @@ -60325,7 +57784,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token_batches": 1.5e-07, "output_cost_per_token_priority": 5.25e-07, @@ -60351,7 +57809,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1.05e-06, @@ -60375,7 +57832,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 1e-07, "output_cost_per_token_priority": 3.5e-07, @@ -60405,7 +57861,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -60449,7 +57904,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -60484,7 +57938,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-cyber.html", "use_openai_responses_path": true, @@ -60520,7 +57973,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -60560,7 +58012,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -60601,7 +58052,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html", "supported_modalities": [ @@ -60638,7 +58088,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html", "supported_modalities": [ @@ -60675,7 +58124,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html", "supported_modalities": [ @@ -60712,7 +58160,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html", "supported_modalities": [ @@ -60749,7 +58196,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html", "supported_modalities": [ @@ -60783,7 +58229,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html", "supported_modalities": [ @@ -60815,7 +58260,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html", "supported_modalities": [ @@ -60847,7 +58291,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html", "supported_modalities": [ @@ -60879,7 +58322,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html", "supported_modalities": [ @@ -60914,7 +58356,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html", "supported_modalities": [ @@ -60955,7 +58396,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token_above_272k_tokens_ultrafast": 0.000495, "output_cost_per_token_ultrafast": 0.00033, @@ -60996,7 +58436,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -61035,7 +58474,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -61082,7 +58520,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_ultrafast": 0.000495, "output_cost_per_token_ultrafast": 0.00033, @@ -61123,7 +58560,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61162,7 +58598,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61207,7 +58642,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_ultrafast": 0.00045, "output_cost_per_token_ultrafast": 0.0003, @@ -61247,7 +58681,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61282,7 +58715,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61320,7 +58752,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61355,7 +58786,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61395,7 +58825,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -61435,7 +58864,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -61470,7 +58898,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token_batches": 2e-07, "output_cost_per_token_priority": 7e-07, @@ -61496,7 +58923,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token_batches": 2e-07, "output_cost_per_token_priority": 7e-07, @@ -61522,7 +58948,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_batches": 4e-08, "output_cost_per_token_priority": 1.4e-07, @@ -61552,7 +58977,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1048576, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_flex": 1.25e-06, @@ -61580,7 +59004,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token_flex": 3.3e-06, "output_cost_per_token_priority": 1.155e-05, @@ -61605,7 +59028,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -61636,7 +59058,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -61674,7 +59095,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -61722,7 +59142,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token_flex": 3.3e-06, "output_cost_per_token_priority": 1.155e-05, @@ -61750,7 +59169,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token_flex": 3e-06, "output_cost_per_token_priority": 1.05e-05, @@ -61765,7 +59183,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, @@ -61805,7 +59222,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4.3125e-06, "source": "https://www.volcengine.com/docs/82379/1544106", @@ -61821,7 +59237,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 1024000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4.3125e-06, "source": "https://www.volcengine.com/docs/82379/1544106", @@ -61839,7 +59254,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 1024000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3.88126e-07, "source": "https://www.volcengine.com/docs/82379/1544106", @@ -61856,7 +59270,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.15625e-06, "source": "https://www.volcengine.com/docs/82379/1544106", @@ -61870,7 +59283,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, @@ -61908,7 +59320,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, @@ -61946,7 +59357,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, @@ -62041,7 +59451,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, @@ -62059,7 +59468,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, @@ -62080,7 +59488,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -62110,7 +59517,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -62132,7 +59538,6 @@ } }, "snowflake/claude-sonnet-4-5": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 3e-06, @@ -62150,7 +59555,6 @@ "snowflake/claude-sonnet-4-6": { "supports_adaptive_thinking": true, "supports_legacy_thinking": true, - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 3e-06, @@ -62166,7 +59570,6 @@ "prompt_cache_min_tokens": 1024 }, "snowflake/claude-4-sonnet": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 3e-06, @@ -62182,7 +59585,6 @@ "prompt_cache_min_tokens": 1024 }, "snowflake/claude-4-opus": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 5e-06, @@ -62199,7 +59601,6 @@ "prompt_cache_min_tokens": 1024 }, "snowflake/claude-haiku-4-5": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 1e-06, @@ -62215,7 +59616,6 @@ "prompt_cache_min_tokens": 4096 }, "snowflake/claude-3-7-sonnet": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 3e-06, @@ -62231,7 +59631,6 @@ "supports_response_schema": true }, "snowflake/openai-gpt-4.1": { - "max_tokens": 16384, "max_input_tokens": 300000, "max_output_tokens": 16384, "input_cost_per_token": 2e-06, @@ -62246,7 +59645,6 @@ "supports_response_schema": true }, "snowflake/openai-gpt-5": { - "max_tokens": 16384, "max_input_tokens": 300000, "max_output_tokens": 16384, "input_cost_per_token": 1.25e-06, @@ -62262,7 +59660,6 @@ "supports_response_schema": true }, "snowflake/openai-gpt-5-mini": { - "max_tokens": 16384, "max_input_tokens": 1000000, "max_output_tokens": 16384, "input_cost_per_token": 3e-07, @@ -62274,7 +59671,6 @@ "supports_response_schema": true }, "snowflake/openai-gpt-5-nano": { - "max_tokens": 16384, "max_input_tokens": 5000000, "max_output_tokens": 16384, "input_cost_per_token": 1.5e-07, @@ -62286,7 +59682,6 @@ "supports_response_schema": true }, "snowflake/llama4-maverick": { - "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.4e-07, @@ -62297,7 +59692,6 @@ "supports_system_messages": true }, "snowflake/snowflake-arctic-embed-l-v2.0": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 7e-08, "output_cost_per_token": 0.0, @@ -62305,7 +59699,6 @@ "mode": "embedding" }, "snowflake/snowflake-arctic-embed-m-v2.0": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 7e-08, "output_cost_per_token": 0.0, @@ -62315,7 +59708,7 @@ "soniox/stt-async-v4": { "litellm_provider": "soniox", "max_output_tokens": 8000, - "max_tokens": 8000, + "max_input_tokens": 8000, "input_cost_per_second": 0.0, "output_cost_per_second": 2.77778e-05, "mode": "audio_transcription", @@ -62328,7 +59721,7 @@ "soniox/stt-async-v5": { "litellm_provider": "soniox", "max_output_tokens": 8000, - "max_tokens": 8000, + "max_input_tokens": 8000, "input_cost_per_second": 0.0, "output_cost_per_second": 2.77778e-05, "mode": "audio_transcription", @@ -62505,7 +59898,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62559,7 +59951,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62613,7 +60004,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62667,7 +60057,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 2.2e-08, @@ -62721,7 +60110,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62775,7 +60163,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62829,7 +60216,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62883,7 +60269,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 2.2e-08, @@ -62937,7 +60322,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.48e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -62963,7 +60347,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -62989,7 +60372,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 2.2e-08, @@ -63044,7 +60426,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -63099,7 +60480,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63124,7 +60504,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63149,7 +60528,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63173,7 +60551,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63197,7 +60574,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63221,7 +60597,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63245,7 +60620,6 @@ "litellm_provider": "tencent", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63269,7 +60643,6 @@ "litellm_provider": "tencent", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63336,7 +60709,6 @@ "source": "https://docs.devin.ai/desktop/models" }, "pinstripes/ps/glm-4.5-air": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.25e-07, @@ -63349,7 +60721,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3.6-35b-a3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.4e-07, @@ -63362,7 +60733,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3-30b-a3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-08, @@ -63375,7 +60745,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3-coder-30b-a3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-07, @@ -63388,7 +60757,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/deepseek-v4-flash": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 1e-07, @@ -63401,7 +60769,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/minimax-m2.7": { - "max_tokens": 1000192, "max_input_tokens": 1000192, "max_output_tokens": 1000192, "input_cost_per_token": 2.55e-07, @@ -63418,7 +60785,6 @@ "litellm_provider": "darkbloom", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.65e-07, "source": "https://www.darkbloom.dev/", @@ -63435,7 +60801,6 @@ "litellm_provider": "darkbloom", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7e-08, "source": "https://www.darkbloom.dev/", @@ -63455,7 +60820,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -63482,7 +60846,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -63511,7 +60874,6 @@ "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://api.x.ai/v1/language-models", @@ -63584,7 +60946,6 @@ "litellm_provider": "openai", "max_input_tokens": 16000, "max_output_tokens": 2000, - "max_tokens": 2000, "mode": "realtime", "source": "https://developers.openai.com/api/docs/pricing", "supported_modalities": [ @@ -63609,7 +60970,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -63653,7 +61013,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -63696,7 +61055,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "prompt_cache_min_tokens": 512, @@ -63732,7 +61090,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 5e-06, "search_context_cost_per_query": { @@ -63767,7 +61124,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6e-07, "reasoning_effort_levels": [ @@ -63788,7 +61144,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/models/model-cards/leanstral-1-5", @@ -63878,7 +61233,6 @@ "mode": "chat", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, @@ -63977,7 +61331,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 2.1e-05, "output_cost_per_token": 2.1e-05, @@ -64006,7 +61359,6 @@ "litellm_provider": "gemini", "max_input_tokens": 98304, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "audio_transcription", "output_cost_per_token": 1.2e-05, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -64031,7 +61383,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "audio_transcription", "output_cost_per_token": 2.1e-05, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -64112,7 +61463,6 @@ "input_cost_per_token": 8e-09, "litellm_provider": "perplexity", "max_input_tokens": 32768, - "max_tokens": 32768, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64122,7 +61472,6 @@ "input_cost_per_token": 5e-08, "litellm_provider": "perplexity", "max_input_tokens": 32768, - "max_tokens": 32768, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2560, @@ -64132,7 +61481,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64142,7 +61490,6 @@ "input_cost_per_token": 6e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64152,7 +61499,6 @@ "input_cost_per_token": 2e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64162,7 +61508,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64172,7 +61517,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 120000, - "max_tokens": 120000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64182,7 +61526,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64198,7 +61541,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_priority": 8.25e-07, @@ -64217,7 +61559,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.5e-06, @@ -64237,7 +61578,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.8e-06, "output_cost_per_token_priority": 2.25e-06, @@ -64257,7 +61597,7 @@ "input_cost_per_token_priority": 2.75e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_priority": 8.25e-07, @@ -64275,7 +61615,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_priority": 1.875e-05, @@ -64300,7 +61639,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_priority": 8.25e-07, @@ -64319,7 +61657,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.5e-06, @@ -64339,7 +61676,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.8e-06, "output_cost_per_token_priority": 2.25e-06, @@ -64359,7 +61695,7 @@ "input_cost_per_token_priority": 2.75e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_priority": 8.25e-07, @@ -64376,7 +61712,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -64393,7 +61728,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -64411,7 +61745,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_priority": 1.875e-05, @@ -64433,7 +61766,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, "reasoning_effort_levels": [ @@ -64456,7 +61788,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, "output_cost_per_token_priority": 2.8125e-05, @@ -64498,7 +61829,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_priority": 2.25e-06, @@ -64518,7 +61848,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_priority": 2.5e-07, @@ -64538,7 +61867,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.4e-06, "output_cost_per_token_priority": 3e-06, @@ -64558,7 +61886,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_priority": 2.25e-06, @@ -64593,7 +61920,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_priority": 2.5e-07, @@ -64613,7 +61939,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.4e-06, "output_cost_per_token_priority": 3e-06, @@ -64648,7 +61973,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -64665,7 +61989,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -64681,7 +62004,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, "reasoning_effort_levels": [ @@ -64704,7 +62026,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, "output_cost_per_token_priority": 2.8125e-05, @@ -64726,7 +62047,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -64743,7 +62063,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 3.96e-06, "source": "https://novita.ai/pricing", @@ -64760,7 +62079,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://novita.ai/pricing", @@ -64777,7 +62095,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 5.8e-07, "source": "https://novita.ai/pricing", @@ -64794,7 +62111,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://novita.ai/pricing", @@ -64811,7 +62127,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://novita.ai/pricing", @@ -64828,7 +62143,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -64845,7 +62159,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://novita.ai/pricing", @@ -64862,7 +62175,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.5e-06, "source": "https://novita.ai/pricing", @@ -64878,7 +62190,6 @@ "litellm_provider": "novita", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://novita.ai/pricing", @@ -64895,7 +62206,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://novita.ai/pricing", @@ -64912,7 +62222,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 3.2000000000000003e-06, "source": "https://novita.ai/pricing", @@ -64929,7 +62238,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.8e-07, "source": "https://novita.ai/pricing", @@ -64945,7 +62253,6 @@ "litellm_provider": "novita", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://novita.ai/pricing", @@ -64962,7 +62269,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.8e-07, "source": "https://novita.ai/pricing", @@ -64978,7 +62284,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.6e-06, "source": "https://novita.ai/pricing", @@ -64994,7 +62299,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.15e-06, "source": "https://novita.ai/pricing", @@ -65010,7 +62314,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.0000000000000002e-07, "source": "https://novita.ai/pricing", @@ -65026,7 +62329,6 @@ "litellm_provider": "novita", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.13e-06, "source": "https://novita.ai/pricing", @@ -65042,7 +62344,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.3600000000000004e-07, "source": "https://novita.ai/pricing", @@ -65059,7 +62360,6 @@ "litellm_provider": "novita", "max_input_tokens": 1000000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.75e-06, "source": "https://novita.ai/pricing", @@ -65076,7 +62376,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.044e-06, "source": "https://novita.ai/pricing", @@ -65092,7 +62391,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.6000000000000003e-06, "source": "https://novita.ai/pricing", @@ -65108,7 +62406,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.4e-06, "source": "https://novita.ai/pricing", @@ -65125,7 +62422,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://novita.ai/pricing", @@ -65142,7 +62438,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65159,7 +62454,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65175,7 +62469,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.0000000000000003e-07, "source": "https://novita.ai/pricing", @@ -65190,7 +62483,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.0000000000000003e-07, "source": "https://novita.ai/pricing", @@ -65206,7 +62498,6 @@ "litellm_provider": "novita", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65223,7 +62514,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://novita.ai/pricing", @@ -65240,7 +62530,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131100, - "max_tokens": 131100, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://novita.ai/pricing", @@ -65256,7 +62545,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://novita.ai/pricing", @@ -65271,7 +62559,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.2000000000000003e-06, "source": "https://novita.ai/pricing", @@ -65286,7 +62573,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://novita.ai/pricing", @@ -65301,7 +62587,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.6000000000000003e-06, "source": "https://novita.ai/pricing", @@ -65317,7 +62602,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131100, - "max_tokens": 131100, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://novita.ai/pricing", @@ -65334,7 +62618,6 @@ "litellm_provider": "novita", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.2000000000000003e-06, "source": "https://novita.ai/pricing", @@ -65350,7 +62633,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://novita.ai/pricing", @@ -65364,7 +62646,6 @@ "litellm_provider": "novita", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-08, "source": "https://novita.ai/pricing", @@ -65376,7 +62657,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://novita.ai/pricing", @@ -65393,7 +62673,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65410,7 +62689,6 @@ "litellm_provider": "novita", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.0000000000000003e-07, "source": "https://novita.ai/pricing", @@ -65426,7 +62704,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.4850000000000002e-06, "source": "https://novita.ai/pricing", @@ -65441,7 +62718,6 @@ "litellm_provider": "novita", "max_input_tokens": 64000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 8.900000000000001e-07, "source": "https://api.novita.ai/v3/openai/models", @@ -65454,7 +62730,6 @@ "litellm_provider": "novita", "max_input_tokens": 64000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65468,7 +62743,6 @@ "litellm_provider": "novita", "max_input_tokens": 64000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 8.900000000000001e-07, "source": "https://api.novita.ai/v3/openai/models", @@ -65481,7 +62755,6 @@ "litellm_provider": "novita", "max_input_tokens": 64000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65495,7 +62768,6 @@ "litellm_provider": "novita", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.66e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65509,7 +62781,6 @@ "litellm_provider": "novita", "max_input_tokens": 131000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2e-08, "source": "https://api.novita.ai/v3/openai/models", @@ -65518,7 +62789,7 @@ }, "wandb/deepseek-ai/DeepSeek-V4-Flash": { "supports_reasoning": true, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1049000, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, @@ -65532,7 +62803,7 @@ }, "wandb/deepseek-ai/DeepSeek-V4-Flash-0731": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 1.3e-07, "output_cost_per_token": 2.8e-07, @@ -65545,7 +62816,7 @@ }, "wandb/deepseek-ai/DeepSeek-V4-Pro": { "supports_reasoning": true, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1049000, "input_cost_per_token": 1.15e-06, "output_cost_per_token": 2.55e-06, @@ -65559,7 +62830,7 @@ }, "wandb/google/gemma-4-31B-it": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 1e-07, "output_cost_per_token": 3.4e-07, @@ -65570,7 +62841,7 @@ }, "wandb/ibm-granite/granite-4.1-8b": { "deprecation_date": "2026-10-05", - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131000, "input_cost_per_token": 5e-08, "output_cost_per_token": 1e-07, @@ -65581,7 +62852,7 @@ }, "wandb/JetBrains/Mellum2-12B-A2.5B-Instruct": { "deprecation_date": "2026-10-05", - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131000, "input_cost_per_token": 5e-08, "output_cost_per_token": 1e-07, @@ -65592,7 +62863,7 @@ }, "wandb/meta-llama/Llama-3.1-70B-Instruct": { "deprecation_date": "2026-10-05", - "max_tokens": 128000, + "max_output_tokens": 128000, "max_input_tokens": 131000, "input_cost_per_token": 8e-07, "output_cost_per_token": 8e-07, @@ -65603,7 +62874,7 @@ }, "wandb/MiniMaxAI/MiniMax-M3": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 2.3e-07, "output_cost_per_token": 9.6e-07, @@ -65616,7 +62887,7 @@ }, "wandb/moonshotai/Kimi-K2.7-Code": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 7.1e-07, "output_cost_per_token": 3.5e-06, @@ -65629,7 +62900,7 @@ }, "wandb/moonshotai/Kimi-K2.6": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 6.5e-07, "output_cost_per_token": 3.41e-06, @@ -65642,7 +62913,7 @@ }, "wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 7e-08, "output_cost_per_token": 2e-07, @@ -65655,7 +62926,7 @@ }, "wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 5e-07, "output_cost_per_token": 2.15e-06, @@ -65668,7 +62939,7 @@ }, "wandb/OpenPipe/Qwen3-14B-Instruct": { "deprecation_date": "2026-10-05", - "max_tokens": 32768, + "max_output_tokens": 32768, "max_input_tokens": 32800, "input_cost_per_token": 5e-08, "output_cost_per_token": 2.2e-07, @@ -65679,7 +62950,7 @@ }, "wandb/Qwen/Qwen3.8-27B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 4e-07, "output_cost_per_token": 3e-06, @@ -65692,7 +62963,7 @@ }, "wandb/Qwen/Qwen3.6-35B-A3B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.25e-06, @@ -65703,7 +62974,7 @@ }, "wandb/Qwen/Qwen3.6-27B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 6e-07, "output_cost_per_token": 3.6e-06, @@ -65718,7 +62989,7 @@ "wandb/Qwen/Qwen3.5-35B-A3B": { "deprecation_date": "2026-10-05", "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.25e-06, @@ -65729,7 +63000,7 @@ }, "wandb/Qwen/Qwen3-30B-A3B-Instruct-2507": { "deprecation_date": "2026-10-05", - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 1e-07, "output_cost_per_token": 3e-07, @@ -65762,7 +63033,7 @@ }, "wandb/zai-org/GLM-5.2": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 1049000, "input_cost_per_token": 7.6e-07, "output_cost_per_token": 2.42e-06, @@ -65774,7 +63045,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "deepinfra/openai/gpt-oss-120b-Turbo": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07, @@ -65788,7 +63059,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/MiniMaxAI/MiniMax-M2.7": { - "max_tokens": 196608, + "max_output_tokens": 196608, "max_input_tokens": 196608, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1e-06, @@ -65804,7 +63075,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.8-27B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4e-07, "output_cost_per_token": 3e-06, @@ -65820,7 +63091,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-31B-it-Ultra": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 2.7e-07, "output_cost_per_token": 7.6e-07, @@ -65834,7 +63105,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/moonshotai/Kimi-K2.5": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4.5e-07, "output_cost_per_token": 2.25e-06, @@ -65850,7 +63121,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.7-Flash": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 6e-08, "output_cost_per_token": 4e-07, @@ -65866,7 +63137,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.6": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 5e-07, "output_cost_per_token": 2e-06, @@ -65885,7 +63156,7 @@ "input_cost_per_token": 5e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-05, "prompt_cache_min_tokens": 1024, @@ -65899,7 +63170,7 @@ "supports_vision": true }, "deepinfra/anthropic/claude-sonnet-4-6": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, @@ -65915,7 +63186,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemini-3.5-flash": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 1.5e-06, "output_cost_per_token": 9e-06, @@ -65930,7 +63201,7 @@ "supports_audio_input": true }, "deepinfra/XiaomiMiMo/MiMo-V2.5": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4e-07, "output_cost_per_token": 2e-06, @@ -65946,7 +63217,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Max": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 1.2e-06, "output_cost_per_token": 6e-06, @@ -65961,7 +63232,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-31B-it-turbo": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 9e-08, "output_cost_per_token": 3.4e-07, @@ -65976,7 +63247,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/thinkingmachines/Inkling-Small": { - "max_tokens": 524288, + "max_output_tokens": 524288, "max_input_tokens": 524288, "input_cost_per_token": 4.5e-07, "output_cost_per_token": 1.2e-06, @@ -65992,7 +63263,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-models/Muse-Glimmer-30B": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, @@ -66008,7 +63279,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Max-Thinking": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 1.2e-06, "output_cost_per_token": 6e-06, @@ -66023,7 +63294,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-VL-235B-A22B-Instruct": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2e-07, "output_cost_per_token": 8.8e-07, @@ -66038,7 +63309,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-VL-30B-A3B-Instruct": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07, @@ -66051,7 +63322,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.5-27B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.6e-07, "output_cost_per_token": 2.6e-06, @@ -66065,7 +63336,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.6-35B-A3B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1e-07, "output_cost_per_token": 9.5e-07, @@ -66079,7 +63350,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/nvidia/Nemotron-Content-Safety-3.5": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 2e-07, "output_cost_per_token": 2e-07, @@ -66092,7 +63363,7 @@ "input_cost_per_token": 5e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-05, "prompt_cache_min_tokens": 512, @@ -66106,7 +63377,7 @@ "supports_vision": true }, "deepinfra/thinkingmachines/Inkling": { - "max_tokens": 524288, + "max_output_tokens": 524288, "max_input_tokens": 524288, "input_cost_per_token": 9.5e-07, "output_cost_per_token": 4.05e-06, @@ -66122,7 +63393,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/moonshotai/Kimi-K2.6": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7.5e-07, "output_cost_per_token": 3.5e-06, @@ -66138,7 +63409,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4-Pro-0813": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1.3e-06, "output_cost_per_token": 2.6e-06, @@ -66153,7 +63424,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.7-Max": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 2.5e-06, "output_cost_per_token": 7.5e-06, @@ -66168,7 +63439,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/ByteDance/Seed-2.0-mini": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 1e-07, "output_cost_per_token": 4e-07, @@ -66184,7 +63455,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.8-2.4T-A95B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, @@ -66200,7 +63471,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/MiniMaxAI/MiniMax-M3": { - "max_tokens": 524288, + "max_output_tokens": 524288, "max_input_tokens": 524288, "input_cost_per_token": 2.8e-07, "output_cost_per_token": 1.1e-06, @@ -66216,7 +63487,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemini-3.1-flash-lite": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.5e-06, @@ -66231,7 +63502,7 @@ "supports_audio_input": true }, "deepinfra/google/gemini-3.7-flash": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 7.5e-07, "output_cost_per_token": 3.75e-06, @@ -66247,7 +63518,7 @@ "supports_audio_input": true }, "deepinfra/inclusionAI/Ling-3.0-flash": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 6e-08, "output_cost_per_token": 1.8e-07, @@ -66263,7 +63534,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/stepfun-ai/Step-3.7-Flash": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2e-07, "output_cost_per_token": 1.15e-06, @@ -66277,7 +63548,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.5-35B-A3B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 1e-06, @@ -66293,7 +63564,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/ByteDance/Seed-1.8": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 2e-06, @@ -66309,7 +63580,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/tencent/Hy3": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 5.8e-07, @@ -66325,7 +63596,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/ByteDance/Seed-2.0-code": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 5e-07, "output_cost_per_token": 3e-06, @@ -66341,7 +63612,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/ByteDance/Seed-2.0-pro": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 5e-07, "output_cost_per_token": 3e-06, @@ -66357,7 +63628,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-5": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 6e-07, "output_cost_per_token": 2.08e-06, @@ -66373,7 +63644,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/nvidia/Nemotron-3-Nano-30B-A3B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 5e-08, "output_cost_per_token": 2e-07, @@ -66388,7 +63659,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/moonshotai/Kimi-K2.7-Code": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 6.8e-07, "output_cost_per_token": 3.4e-06, @@ -66407,7 +63678,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1e-05, "prompt_cache_min_tokens": 1024, @@ -66421,7 +63692,7 @@ "supports_vision": true }, "deepinfra/Qwen/Qwen3.5-397B-A17B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4.5e-07, "output_cost_per_token": 3e-06, @@ -66437,7 +63708,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 8e-08, "output_cost_per_token": 1.8e-07, @@ -66453,7 +63724,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-E4B-it": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 2e-08, "output_cost_per_token": 1e-07, @@ -66466,7 +63737,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3.2": { - "max_tokens": 163840, + "max_output_tokens": 163840, "max_input_tokens": 163840, "input_cost_per_token": 2.6e-07, "output_cost_per_token": 3.8e-07, @@ -66481,7 +63752,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.8-Max": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 1.65e-06, "output_cost_per_token": 4.951e-06, @@ -66499,7 +63770,7 @@ "input_cost_per_token": 1e-05, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 5e-05, "prompt_cache_min_tokens": 512, @@ -66514,7 +63785,7 @@ "thinking_always_on": true }, "deepinfra/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 5e-07, "output_cost_per_token": 2.2e-06, @@ -66530,7 +63801,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.5-122B-A10B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.9e-07, "output_cost_per_token": 2.4e-06, @@ -66544,7 +63815,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-5.1": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 1.05e-06, "output_cost_per_token": 3.5e-06, @@ -66560,7 +63831,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4-Pro": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1.3e-06, "output_cost_per_token": 2.6e-06, @@ -66575,7 +63846,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 8.5e-08, "output_cost_per_token": 4e-07, @@ -66589,7 +63860,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-5.2": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 7.5e-07, "output_cost_per_token": 2.4e-06, @@ -66605,7 +63876,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/moonshotai/Kimi-K3": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 2.85e-06, "output_cost_per_token": 1.425e-05, @@ -66623,7 +63894,7 @@ "input_cost_per_token": 5e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-05, "prompt_cache_min_tokens": 2048, @@ -66637,7 +63908,7 @@ "supports_vision": true }, "deepinfra/Qwen/Qwen3.6-27B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 3.2e-07, "output_cost_per_token": 3.2e-06, @@ -66651,7 +63922,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-26B-A4B-it": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7e-08, "output_cost_per_token": 3.4e-07, @@ -66665,7 +63936,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemini-3.1-pro": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 2e-06, "output_cost_per_token": 1.2e-05, @@ -66680,7 +63951,7 @@ "supports_audio_input": true }, "deepinfra/XiaomiMiMo/MiMo-V2.5-Pro": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1e-06, "output_cost_per_token": 3e-06, @@ -66696,7 +63967,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/anthropic/claude-haiku-4-5": { - "max_tokens": 200000, + "max_output_tokens": 200000, "max_input_tokens": 200000, "input_cost_per_token": 1e-06, "output_cost_per_token": 5e-06, @@ -66711,7 +63982,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4-Flash": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 9e-08, "output_cost_per_token": 1.8e-07, @@ -66727,7 +63998,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4.1-Flash": { - "max_tokens": 131072, "max_input_tokens": 1048576, "max_output_tokens": 131072, "input_cost_per_token": 1.4e-07, @@ -66744,7 +64014,6 @@ "source": "https://deepinfra.com/deepseek-ai/DeepSeek-V4.1-Flash" }, "deepinfra/deepseek-ai/DeepSeek-V4.1-Flash-1017": { - "max_tokens": 131072, "max_input_tokens": 1048576, "max_output_tokens": 131072, "input_cost_per_token": 1.4e-07, @@ -66761,7 +64030,7 @@ "source": "https://deepinfra.com/deepseek-ai/DeepSeek-V4.1-Flash-1017" }, "deepinfra/openai/gpt-oss-120b-Ultra": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 2e-07, "output_cost_per_token": 9.5e-07, @@ -66775,7 +64044,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.5-9B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1e-07, "output_cost_per_token": 1.5e-07, @@ -66788,7 +64057,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/MiniMaxAI/MiniMax-M2.7-Turbo": { - "max_tokens": 196608, + "max_output_tokens": 196608, "max_input_tokens": 196608, "input_cost_per_token": 3.8e-07, "output_cost_per_token": 1.7e-06, @@ -66804,7 +64073,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.7": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 4e-07, "output_cost_per_token": 1.75e-06, @@ -66820,7 +64089,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-31B-it": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.3e-07, "output_cost_per_token": 3.8e-07, @@ -66839,7 +64108,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -66874,7 +64142,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -66902,7 +64169,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -66930,7 +64196,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67206,7 +64471,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67233,7 +64497,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67260,7 +64523,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67291,7 +64553,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67319,7 +64580,6 @@ "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://console.groq.com/docs/model/qwen/qwen3.8-27b", @@ -67335,7 +64595,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -67356,7 +64615,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -67377,7 +64635,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -67398,7 +64655,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6e-07, "reasoning_effort_levels": [ @@ -67419,7 +64675,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", @@ -67434,7 +64689,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", @@ -67449,7 +64703,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://mistral.ai/news/devstral-2-vibe-cli", @@ -67550,7 +64803,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/models/model-cards/leanstral-1-5", @@ -67566,7 +64818,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_priority": 5.5e-06, @@ -67585,7 +64836,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.6e-06, "output_cost_per_token_priority": 8.25e-06, @@ -67604,7 +64854,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_priority": 5.5e-06, @@ -67623,7 +64872,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.6e-06, "output_cost_per_token_priority": 8.25e-06, @@ -67639,7 +64887,7 @@ "input_cost_per_token": 2.1e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -67681,7 +64929,7 @@ "input_cost_per_token": 2.1e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -67698,7 +64946,7 @@ "input_cost_per_token_priority": 1.875e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_priority": 6.25e-07, @@ -67716,7 +64964,7 @@ "input_cost_per_token_priority": 2.8125e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 7.5e-07, "output_cost_per_token_priority": 9.375e-07, @@ -67733,7 +64981,7 @@ "input_cost_per_token_priority": 1.875e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_priority": 6.25e-07, @@ -67751,7 +64999,7 @@ "input_cost_per_token_priority": 2.8125e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 7.5e-07, "output_cost_per_token_priority": 9.375e-07, @@ -67769,7 +65017,7 @@ "input_cost_per_token_priority": 1e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.05e-06, "output_cost_per_token_priority": 4.05e-06, @@ -67785,7 +65033,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 40960, "max_output_tokens": 40960, - "max_tokens": 40960, "mode": "embedding", "source": "https://api.fireworks.ai/v1/serverless/models" }, @@ -67808,7 +65055,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://api.together.ai/v1/models" @@ -67868,7 +65115,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 131072, "max_output_tokens": 40960, - "max_tokens": 40960, "mode": "chat", "output_cost_per_token": 1.49e-06, "source": "https://api.cerebras.ai/public/v1/models/gemma-4-31b", @@ -67894,7 +65140,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", @@ -67908,7 +65153,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", @@ -67924,7 +65168,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -67947,7 +65190,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": true, @@ -67989,7 +65231,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68004,7 +65245,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.1e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68019,7 +65259,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68034,7 +65273,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.25e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68050,7 +65288,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68067,7 +65304,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68080,7 +65316,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68155,7 +65390,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.88e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -68173,7 +65407,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", @@ -68189,7 +65422,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", @@ -68204,7 +65436,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -68222,7 +65453,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", @@ -68242,7 +65472,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", @@ -68267,7 +65496,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, @@ -68306,7 +65534,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 1024, @@ -68344,7 +65571,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, @@ -68381,7 +65607,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.4e-05, "prompt_cache_min_tokens": 512, @@ -68419,7 +65644,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_adaptive_thinking": true, @@ -68454,7 +65678,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.88e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -68472,7 +65695,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", @@ -68488,7 +65710,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", @@ -68503,7 +65724,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -68521,7 +65741,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", @@ -68541,7 +65760,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", @@ -68566,7 +65784,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, @@ -68600,7 +65817,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 1024, @@ -68633,7 +65849,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, @@ -68665,7 +65880,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.4e-05, "prompt_cache_min_tokens": 512, @@ -68698,7 +65912,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_adaptive_thinking": true, @@ -68727,7 +65940,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68763,7 +65975,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68799,7 +66010,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68830,7 +66040,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1048576, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -68854,7 +66063,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -68873,7 +66081,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68892,7 +66099,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68911,7 +66117,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68931,7 +66136,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ @@ -68951,7 +66155,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ @@ -68974,7 +66177,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.4e-05, "prompt_cache_min_tokens": 512, @@ -69010,7 +66212,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "supports_forced_tool_use": false, @@ -69039,7 +66240,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -69072,7 +66272,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -69092,7 +66291,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ @@ -69112,7 +66310,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ @@ -69136,7 +66333,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 8.4e-07, "output_cost_per_token_priority": 2.94e-06, @@ -69158,7 +66354,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_priority": 4.38e-06, @@ -69180,7 +66375,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 4.4e-07, "output_cost_per_token_priority": 1.54e-06, @@ -69202,7 +66396,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 32000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1.05e-06, @@ -69225,7 +66418,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1.05e-06, @@ -69247,7 +66439,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 9e-07, "output_cost_per_token_priority": 3.15e-06, @@ -69268,7 +66459,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 6e-07, "output_cost_per_token_priority": 2.1e-06, @@ -69290,7 +66480,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 1.33e-06, "output_cost_per_token_priority": 4.66e-06, @@ -69309,7 +66498,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.375e-05, "supported_endpoints": [ @@ -69343,7 +66531,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.05e-06, "supports_prompt_caching": true, @@ -69356,7 +66543,6 @@ "input_cost_per_token": 1.63e-07, "litellm_provider": "azure", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -69364,7 +66550,6 @@ "input_cost_per_token": 2.5e-08, "litellm_provider": "azure", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -69395,7 +66580,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_image": 0.04, "output_cost_per_token": 0, @@ -69420,7 +66604,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -69445,7 +66628,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 0, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -69470,7 +66652,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 0, "source": "https://ai.google.dev/gemini-api/docs/models/lyria-realtime-exp", @@ -69843,7 +67024,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -69867,7 +67047,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://inference.baseten.co/v1/models", @@ -69892,7 +67071,7 @@ "output_cost_per_token": 1.5e-07, "source": "https://api.together.ai/v1/models", "max_input_tokens": 128000, - "max_tokens": 128000 + "max_output_tokens": 128000 }, "vertex_ai/gemini-2.5-flash-native-audio": { "deprecation_date": "2026-12-13", @@ -69953,7 +67132,7 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai", "max_output_tokens": 57920, - "max_tokens": 57920, + "max_input_tokens": 57920, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -70016,7 +67195,7 @@ "output_cost_per_token": 3.5e-06, "source": "https://api.together.ai/v1/models", "max_input_tokens": 4096, - "max_tokens": 4096 + "max_output_tokens": 4096 }, "together_ai/meta-llama/Llama-3.2-1B-Instruct": { "input_cost_per_token": 6e-08, @@ -70025,7 +67204,7 @@ "output_cost_per_token": 6e-08, "source": "https://api.together.ai/v1/models", "max_input_tokens": 131072, - "max_tokens": 131072 + "max_output_tokens": 131072 }, "together_ai/meta-llama/Llama-3.2-3B-Instruct": { "input_cost_per_token": 6e-08, @@ -70034,7 +67213,7 @@ "output_cost_per_token": 6e-08, "source": "https://api.together.ai/v1/models", "max_input_tokens": 131072, - "max_tokens": 131072 + "max_output_tokens": 131072 }, "together_ai/Qwen/Qwen2-1.5B-Instruct": { "input_cost_per_token": 2e-08, @@ -70043,7 +67222,7 @@ "output_cost_per_token": 2e-08, "source": "https://api.together.ai/v1/models", "max_input_tokens": 32768, - "max_tokens": 32768 + "max_output_tokens": 32768 }, "together_ai/Qwen/Qwen2.5-14B-Instruct": { "input_cost_per_token": 8e-07, @@ -70052,7 +67231,7 @@ "output_cost_per_token": 8e-07, "source": "https://api.together.ai/v1/models", "max_input_tokens": 32768, - "max_tokens": 32768 + "max_output_tokens": 32768 }, "together_ai/Qwen/Qwen2.5-72B-Instruct": { "input_cost_per_token": 1.2e-06, @@ -70061,14 +67240,13 @@ "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", "max_input_tokens": 32768, - "max_tokens": 32768 + "max_output_tokens": 32768 }, "together_ai/Salesforce/Llama-Rank-V1": { "deprecation_date": "2026-02-06", "input_cost_per_token": 1e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://api.together.ai/v1/models" @@ -70077,7 +67255,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 16384, - "max_tokens": 16384, + "max_output_tokens": 16384, "mode": "completion", "output_cost_per_token": 2e-07, "source": "https://api.together.xyz/v1/models" @@ -70095,7 +67273,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://api.together.ai/v1/models" @@ -70105,7 +67283,7 @@ "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" @@ -70115,7 +67293,7 @@ "input_cost_per_token": 9e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://api.together.ai/v1/models" @@ -70125,7 +67303,7 @@ "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" @@ -70135,7 +67313,7 @@ "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" @@ -70145,7 +67323,7 @@ "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "max_input_tokens": 16384, - "max_tokens": 16384, + "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://api.together.ai/v1/models" @@ -70155,7 +67333,7 @@ "input_cost_per_token": 1.95e-06, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://api.together.ai/v1/models" @@ -70165,7 +67343,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://api.together.ai/v1/models" @@ -70175,7 +67353,7 @@ "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" @@ -70185,7 +67363,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://api.together.ai/v1/models" @@ -70195,7 +67373,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://api.together.ai/v1/models" @@ -70205,7 +67383,7 @@ "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://api.together.ai/v1/models" @@ -70215,7 +67393,7 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://api.together.ai/v1/models" @@ -70226,7 +67404,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.6e-06, "source": "https://api.together.ai/v1/models" @@ -70236,7 +67414,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://api.together.ai/v1/models" @@ -70246,7 +67424,7 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.8e-07, "source": "https://api.together.ai/v1/models" @@ -70256,7 +67434,7 @@ "input_cost_per_token": 1.6e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://api.together.ai/v1/models" @@ -70266,7 +67444,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.7e-06, "source": "https://api.together.ai/v1/models" @@ -70276,7 +67454,7 @@ "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "max_input_tokens": 16384, - "max_tokens": 16384, + "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://api.together.ai/v1/models" @@ -70286,7 +67464,7 @@ "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://api.together.ai/v1/models" @@ -70296,7 +67474,7 @@ "input_cost_per_token": 3.9e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 9.7e-07, "source": "https://api.together.ai/v1/models" @@ -70306,7 +67484,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://api.together.ai/v1/models" @@ -70316,7 +67494,7 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5.9e-07, "source": "https://api.together.ai/v1/models" @@ -70326,7 +67504,7 @@ "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://api.together.ai/v1/models" @@ -70336,7 +67514,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://api.together.ai/v1/models" @@ -70346,7 +67524,7 @@ "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://api.together.ai/v1/models" @@ -70356,7 +67534,7 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.8e-07, "source": "https://api.together.ai/v1/models" @@ -70366,7 +67544,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://api.together.ai/v1/models" @@ -70376,7 +67554,7 @@ "input_cost_per_token": 1e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://api.together.ai/v1/models" @@ -70386,7 +67564,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://api.together.ai/v1/models" @@ -70397,7 +67575,7 @@ "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.5e-06, "source": "https://api.together.ai/v1/models" @@ -70408,7 +67586,7 @@ "input_cost_per_token": 9.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.together.ai/v1/models" @@ -70418,7 +67596,7 @@ "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://api.together.ai/v1/models" @@ -70429,7 +67607,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 512288, - "max_tokens": 512288, + "max_output_tokens": 512288, "mode": "chat", "output_cost_per_token": 3.6e-06, "source": "https://api.together.ai/v1/models" @@ -70439,7 +67617,7 @@ "input_cost_per_token": 5e-08, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://api.together.ai/v1/models" @@ -70449,7 +67627,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.1e-06, "source": "https://api.together.ai/v1/models" @@ -70459,7 +67637,7 @@ "input_cost_per_token": 4.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 202752, - "max_tokens": 202752, + "max_output_tokens": 202752, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://api.together.ai/v1/models" @@ -70469,7 +67647,7 @@ "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", "max_input_tokens": 202752, - "max_tokens": 202752, + "max_output_tokens": 202752, "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://api.together.ai/v1/models" @@ -70480,7 +67658,7 @@ "input_cost_per_token": 1.4e-06, "litellm_provider": "together_ai", "max_input_tokens": 202752, - "max_tokens": 202752, + "max_output_tokens": 202752, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://api.together.ai/v1/models" @@ -70547,7 +67725,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_batches": 8.25e-06, @@ -70720,7 +67897,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_above_272k_tokens": 9e-05, @@ -70764,7 +67940,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_above_272k_tokens": 9e-07, @@ -70813,7 +67988,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -70869,7 +68043,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -70883,7 +68056,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -70906,7 +68078,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -70920,7 +68091,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -70955,7 +68125,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -70977,7 +68146,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -71053,7 +68221,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_batches": 8.25e-06, @@ -71266,7 +68433,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, @@ -71314,7 +68480,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "audio_transcription", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ @@ -71355,7 +68520,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ @@ -71373,7 +68537,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 512000, "max_output_tokens": 65500, - "max_tokens": 65500, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71386,7 +68549,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71399,7 +68561,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 200000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.2e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71414,7 +68575,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71435,7 +68595,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71453,7 +68612,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71473,7 +68631,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71494,7 +68651,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71513,7 +68669,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.2e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71528,7 +68683,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.61333e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71557,7 +68711,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.6e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71573,7 +68726,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71587,7 +68739,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71602,7 +68753,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.84e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71617,7 +68767,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.38e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71632,7 +68781,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.411e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71648,7 +68796,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.4246e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71665,7 +68812,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.0136e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71682,7 +68828,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.411e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71699,7 +68844,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.32375e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71716,7 +68860,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 119000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.5353e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71729,7 +68872,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71746,7 +68888,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71763,7 +68904,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71780,7 +68920,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71797,7 +68936,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.499999e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71815,7 +68953,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.75e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71832,7 +68969,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 131100, - "max_tokens": 131100, "mode": "chat", "output_cost_per_token": 3.9998e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71844,7 +68980,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 131100, - "max_tokens": 131100, "mode": "chat", "output_cost_per_token": 3.9998e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71857,7 +68992,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 7.889e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71872,7 +69006,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.9438e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71887,7 +69020,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.9438e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71903,7 +69035,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 200000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.09848e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71917,7 +69048,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71933,7 +69063,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71950,7 +69079,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71967,7 +69095,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71984,7 +69111,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72001,7 +69127,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72017,7 +69142,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 0.00018, "source": "https://aihubmix.com/api/v1/models", @@ -72034,7 +69158,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72051,7 +69174,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72068,7 +69190,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72084,7 +69205,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72129,7 +69249,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72145,7 +69264,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72161,7 +69279,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.248e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72177,7 +69294,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.535e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72193,7 +69309,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.9995e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72209,7 +69324,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.999e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72225,7 +69339,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72241,7 +69354,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.0984e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72254,7 +69366,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72290,7 +69401,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 204800, "max_output_tokens": 204800, - "max_tokens": 204800, "mode": "chat", "output_cost_per_token": 1.1832e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72304,7 +69414,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.152e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72329,7 +69438,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 5.48e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72341,7 +69449,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 9.008e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72356,7 +69463,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 9.864e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72371,7 +69477,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.532e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72386,7 +69491,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.524e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72403,7 +69507,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.608e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72420,7 +69523,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.128e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72437,7 +69539,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72454,7 +69555,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.80025e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72472,7 +69572,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5.07e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72657,7 +69756,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "wandb", "max_input_tokens": 1049000, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.5e-07, "source": "https://wandb.ai/site/pricing/tokens/", @@ -72670,7 +69769,7 @@ "input_cost_per_token": 1e-07, "litellm_provider": "wandb", "max_input_tokens": 262000, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://wandb.ai/site/pricing/tokens/", @@ -72711,7 +69810,6 @@ "token_count_tokenizer": "kimi_k3", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_flex": 8.25e-06, @@ -72743,7 +69841,6 @@ "token_count_tokenizer": "kimi_k3", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_flex": 7.5e-06, @@ -72775,7 +69872,6 @@ "token_count_tokenizer": "kimi_k3", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_flex": 8.25e-06, @@ -72797,7 +69893,6 @@ "litellm_provider": "xiaomi_mimo", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.7e-07, "source": "https://platform.xiaomimimo.com/static/docs/price/pay-as-you-go.md", @@ -72820,7 +69915,6 @@ "litellm_provider": "xiaomi_mimo", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://platform.xiaomimimo.com/static/docs/price/pay-as-you-go.md", @@ -72845,7 +69939,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -72871,7 +69964,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72893,7 +69985,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72915,7 +70006,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72937,7 +70027,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72958,7 +70047,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72980,7 +70068,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73001,7 +70088,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73023,7 +70109,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73045,7 +70130,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73066,7 +70150,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73088,7 +70171,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73110,7 +70192,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73131,7 +70212,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73153,7 +70233,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73178,7 +70257,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73203,7 +70281,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73228,7 +70305,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73249,7 +70325,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73271,7 +70346,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -73289,7 +70363,6 @@ "litellm_provider": "baseten", "max_input_tokens": 262000, "max_output_tokens": 262000, - "max_tokens": 262000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://inference.baseten.co/v1/models", @@ -73307,7 +70380,6 @@ "litellm_provider": "baseten", "max_input_tokens": 262000, "max_output_tokens": 262000, - "max_tokens": 262000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://inference.baseten.co/v1/models", @@ -73324,7 +70396,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://inference.baseten.co/v1/models", @@ -73341,7 +70412,6 @@ "litellm_provider": "baseten", "max_input_tokens": 202800, "max_output_tokens": 202800, - "max_tokens": 202800, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -73359,7 +70429,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4.05e-06, "source": "https://inference.baseten.co/v1/models", @@ -73377,7 +70446,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -73394,7 +70462,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -73411,7 +70478,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inference.baseten.co/v1/models", @@ -73428,7 +70494,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.6e-07, "source": "https://inference.baseten.co/v1/models", @@ -73446,7 +70511,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.48e-06, "source": "https://inference.baseten.co/v1/models", @@ -73463,7 +70527,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.96e-06, "source": "https://inference.baseten.co/v1/models", @@ -73480,7 +70543,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://inference.baseten.co/v1/models", @@ -73500,7 +70562,6 @@ "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, @@ -73521,7 +70582,6 @@ "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, @@ -73542,7 +70602,6 @@ "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, @@ -73563,7 +70622,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.2e-06, @@ -73581,7 +70639,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 196608, "max_output_tokens": 196608, - "max_tokens": 196608, "mode": "chat", "output_cost_per_token_priority": 1.2e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -73647,7 +70704,6 @@ "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -73665,7 +70721,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -73684,7 +70739,6 @@ "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#veo", @@ -73700,7 +70754,6 @@ "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -73717,7 +70770,6 @@ "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.5, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#veo", @@ -73764,7 +70816,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73797,7 +70848,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73830,7 +70880,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73863,7 +70912,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73901,7 +70949,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73934,7 +70981,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73970,7 +71016,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74009,7 +71054,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74049,7 +71093,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74087,7 +71130,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74126,7 +71168,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74159,7 +71200,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "supports_mid_conversation_system": true, @@ -74197,7 +71237,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "thinking_always_on": true, "supports_function_calling": true, @@ -74217,7 +71256,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "thinking_always_on": true, "supports_function_calling": true, @@ -74237,7 +71275,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "thinking_always_on": true, "supports_function_calling": true, @@ -74259,7 +71296,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-r1.html", @@ -74272,7 +71308,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -74285,7 +71320,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.5e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/", @@ -74321,7 +71355,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -74359,7 +71392,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "supports_mid_conversation_system": true, @@ -74405,7 +71437,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -74447,7 +71478,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -74484,7 +71514,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "supports_forced_tool_use": false, @@ -74521,7 +71550,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "supports_forced_tool_use": false, @@ -74562,7 +71590,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -74609,7 +71636,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -74651,7 +71677,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -74783,7 +71808,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "supports_forced_tool_use": false, @@ -74829,7 +71853,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -74874,7 +71897,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -74914,7 +71936,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -74947,7 +71968,6 @@ "litellm_provider": "prism", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 6.3e-07, "source": "https://prisminference.com/pricing", @@ -74972,7 +71992,6 @@ "litellm_provider": "prism", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.1e-07, "source": "https://prisminference.com/pricing", @@ -75005,7 +72024,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 524288, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_flex": 3e-06, @@ -75031,7 +72049,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 524288, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6.6e-06, "output_cost_per_token_flex": 3.3e-06, @@ -75049,7 +72066,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -75104,7 +72120,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -75172,7 +72187,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -75218,7 +72232,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "responses", "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, @@ -75266,7 +72279,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, @@ -75301,7 +72313,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 9e-06, "output_cost_per_token": 9e-06, @@ -75315,7 +72326,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 6e-06, "output_cost_per_token": 6e-06, @@ -75332,7 +72342,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 524288, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -75350,7 +72359,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", @@ -75383,7 +72391,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.28e-06, "output_cost_per_token_batches": 2.64e-06, @@ -75416,7 +72423,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.808e-06, "output_cost_per_token_batches": 2.904e-06, @@ -75439,7 +72445,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2.75e-06, @@ -75454,7 +72459,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/model-router/", @@ -75474,7 +72478,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -75519,7 +72522,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -75565,7 +72567,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -75609,7 +72610,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -75637,7 +72637,6 @@ "token_count_tokenizer": "kimi_k3", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_flex": 8.25e-06, @@ -75670,7 +72669,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, @@ -75724,7 +72722,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -75771,7 +72768,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_forced_tool_use": true, @@ -75816,7 +72812,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "search_context_cost_per_query": { @@ -75859,7 +72854,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "supports_mid_conversation_system": true, @@ -75910,7 +72904,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -75957,7 +72950,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "search_context_cost_per_query": { @@ -75999,7 +72991,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_forced_tool_use": true, @@ -76041,7 +73032,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_forced_tool_use": true, @@ -76087,7 +73077,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -76139,7 +73128,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "search_context_cost_per_query": { @@ -76186,7 +73174,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -76245,7 +73232,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_forced_tool_use": true, @@ -76296,7 +73282,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -76346,7 +73331,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, @@ -76391,7 +73375,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, @@ -76430,7 +73413,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 261120, "max_output_tokens": 98304, - "max_tokens": 98304, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", @@ -76441,7 +73423,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 261120, "max_output_tokens": 98304, - "max_tokens": 98304, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", @@ -76452,7 +73433,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 261120, "max_output_tokens": 98304, - "max_tokens": 98304, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", @@ -76488,7 +73468,6 @@ "input_cost_per_token": 8e-08, "litellm_provider": "azure_ai", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048.0, @@ -76508,7 +73487,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048.0, @@ -76577,7 +73555,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, @@ -76623,7 +73600,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, @@ -76673,7 +73649,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_reasoning_token": 1e-05, "output_cost_per_token": 1e-05, @@ -76732,7 +73707,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_reasoning_token": 1e-05, "output_cost_per_token": 1e-05, @@ -76790,7 +73764,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_reasoning_token": 2e-05, "output_cost_per_token": 2e-05, @@ -76838,7 +73811,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -76851,7 +73823,6 @@ "supports_vision": true }, "openinfer/@oi/GLM-5.3-Flash": { - "max_tokens": 393216, "max_input_tokens": 393216, "max_output_tokens": 393216, "input_cost_per_token": 1.95e-07, @@ -76863,7 +73834,6 @@ "source": "https://api.openinfer.ai/v1/models" }, "openinfer/@oi/Qwen3.5-27B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1.95e-07, @@ -76875,7 +73845,6 @@ "source": "https://api.openinfer.ai/v1/models" }, "openinfer/@oi/Qwen3.5-9B": { - "max_tokens": 100000, "max_input_tokens": 100000, "max_output_tokens": 100000, "input_cost_per_token": 5e-06, @@ -76954,7 +73923,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://api.together.ai/v1/models", @@ -76977,7 +73946,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.311e-06, "output_cost_per_token_batches": 1.6555e-06, @@ -76999,7 +73967,6 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -77023,7 +73990,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-05, "prompt_cache_min_tokens": 512.0, @@ -77053,7 +74019,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.25e-06, "source": "https://inference.baseten.co/v1/models", @@ -77071,7 +74036,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inference.baseten.co/v1/models", @@ -77087,7 +74051,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "wandb", "max_input_tokens": 1049000, - "max_tokens": 943718, + "max_output_tokens": 943718, "mode": "chat", "output_cost_per_token": 7.5e-07, "source": "https://openrouter.ai/api/v1/models/nvidia/nemotron-3.5-super-vl/endpoints", @@ -77103,7 +74067,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", @@ -77117,7 +74081,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -77131,7 +74094,7 @@ "input_cost_per_token": 2.5e-07, "litellm_provider": "nebius", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://tokenfactory.nebius.com/models/catalog/text2text/reflection%2Fbeam", @@ -77144,7 +74107,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -77161,7 +74123,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -77180,7 +74141,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.5e-06, @@ -77200,7 +74160,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -77222,7 +74181,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.25e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -77244,7 +74202,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -77266,7 +74223,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -77288,7 +74244,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -77307,7 +74262,7 @@ "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", @@ -77344,7 +74299,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 9e-07, @@ -77380,7 +74334,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.8e-06, "output_cost_per_token_priority": 2.25e-06, @@ -77429,7 +74382,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -77459,7 +74411,6 @@ "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 5e-07, - "max_tokens": 500000, "max_output_tokens": 500000, "max_input_tokens": 500000, "supports_function_calling": true, @@ -77497,7 +74448,6 @@ "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, "cache_creation_input_token_cost": 1.25e-07, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 922000, "supports_function_calling": true, @@ -77537,7 +74487,6 @@ "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 2e-08, "cache_creation_input_token_cost": 2.5e-07, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 922000, "supports_function_calling": true, @@ -77577,7 +74526,6 @@ "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, "cache_creation_input_token_cost": 1.25e-07, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77616,7 +74564,6 @@ "input_cost_per_token": 2e-06, "output_cost_per_token": 1e-05, "cache_read_input_token_cost": 1e-07, - "max_tokens": 65536, "max_output_tokens": 65536, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77633,7 +74580,6 @@ "input_cost_per_token": 1.5e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 3e-08, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77650,7 +74596,6 @@ "input_cost_per_token": 1.4e-06, "output_cost_per_token": 4.4e-06, "cache_read_input_token_cost": 2.6e-07, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77667,7 +74612,6 @@ "input_cost_per_token": 1.4e-06, "output_cost_per_token": 4.4e-06, "cache_read_input_token_cost": 2.6e-07, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77685,7 +74629,6 @@ "output_cost_per_token": 1e-05, "cache_read_input_token_cost": 2.0000000000000002e-07, "cache_creation_input_token_cost": 2e-06, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77703,7 +74646,6 @@ "output_cost_per_token": 1.5e-05, "cache_read_input_token_cost": 3e-07, "cache_creation_input_token_cost": 3e-06, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77720,7 +74662,6 @@ "input_cost_per_token": 9.499999999999999e-07, "output_cost_per_token": 4e-06, "cache_read_input_token_cost": 1.9e-07, - "max_tokens": 262144, "max_output_tokens": 262144, "max_input_tokens": 262144, "supports_function_calling": true, @@ -77737,7 +74678,6 @@ "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 6e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77754,7 +74694,6 @@ "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 6e-09, - "max_tokens": 384000, "max_output_tokens": 384000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77771,7 +74710,6 @@ "input_cost_per_token": 1.32e-06, "output_cost_per_token": 3.96e-06, "cache_read_input_token_cost": 4.4e-08, - "max_tokens": 384000, "max_output_tokens": 384000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77788,7 +74726,6 @@ "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, "cache_read_input_token_cost": 2.8e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77805,7 +74742,6 @@ "input_cost_per_token": 4.35e-07, "output_cost_per_token": 8.7e-07, "cache_read_input_token_cost": 3.625e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77822,7 +74758,6 @@ "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, "cache_read_input_token_cost": 2.8e-09, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77839,7 +74774,6 @@ "input_cost_per_token": 4.35e-07, "output_cost_per_token": 8.7e-07, "cache_read_input_token_cost": 3.625e-09, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77856,7 +74790,6 @@ "input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07, "cache_read_input_token_cost": 1.5e-08, - "max_tokens": 512000, "max_output_tokens": 512000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77873,7 +74806,6 @@ "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 6e-08, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77890,7 +74822,6 @@ "input_cost_per_token": 1.0000000000000001e-07, "output_cost_per_token": 2.0000000000000002e-07, "cache_read_input_token_cost": 2e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77907,7 +74838,6 @@ "input_cost_per_token": 1.0000000000000001e-07, "output_cost_per_token": 2.0000000000000002e-07, "cache_read_input_token_cost": 2e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77925,7 +74855,6 @@ "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 2.5e-07, "cache_creation_input_token_cost": 2.5e-06, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77943,7 +74872,6 @@ "output_cost_per_token": 4.6999999999999995e-07, "cache_read_input_token_cost": 1.6e-08, "cache_creation_input_token_cost": 2.0000000000000002e-07, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77961,7 +74889,6 @@ "output_cost_per_token": 1.6000000000000001e-06, "cache_read_input_token_cost": 4e-08, "cache_creation_input_token_cost": 5e-07, - "max_tokens": 65536, "max_output_tokens": 65536, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -78000,7 +74927,6 @@ "input_cost_per_token": 8.34e-07, "output_cost_per_token": 2.501e-06, "cache_read_input_token_cost": 4.2000000000000006e-08, - "max_tokens": 64000, "max_output_tokens": 64000, "max_input_tokens": 1024000, "supports_function_calling": true, @@ -78017,7 +74943,6 @@ "input_cost_per_token": 1.4e-07, "output_cost_per_token": 5.8e-07, "cache_read_input_token_cost": 3.5e-08, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 192000, "supports_function_calling": true, @@ -78034,7 +74959,6 @@ "input_cost_per_token": 0, "output_cost_per_token": 0, "cache_read_input_token_cost": 0, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -78051,7 +74975,6 @@ "input_cost_per_token": 1e-06, "output_cost_per_token": 2.7e-06, "cache_read_input_token_cost": 1.0000000000000001e-07, - "max_tokens": 65536, "max_output_tokens": 65536, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -78068,7 +74991,6 @@ "input_cost_per_token": 0, "output_cost_per_token": 0, "cache_read_input_token_cost": 0, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -78089,7 +75011,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -78134,7 +75055,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -78178,7 +75098,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -78211,7 +75130,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, @@ -78261,7 +75179,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78298,7 +75215,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78341,7 +75257,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78385,7 +75300,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78430,7 +75344,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78475,7 +75388,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_batches": 1e-05, @@ -78520,7 +75432,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -78555,7 +75466,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -78598,7 +75508,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -78642,7 +75551,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -78712,7 +75620,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_above_272k_tokens": 7.5e-06, @@ -78761,7 +75668,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -78788,7 +75694,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_above_272k_tokens": 7.5e-06, @@ -78827,7 +75732,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-06, "output_cost_per_token_above_272k_tokens": 8.25e-06, @@ -78884,7 +75788,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 5e-06, "output_cost_per_token_above_272k_tokens": 7.5e-06, diff --git a/tests/unit/litellm_core_utils/test_fallback_generalizations.py b/tests/unit/litellm_core_utils/test_fallback_generalizations.py index 7807c438503..e59252e1571 100644 --- a/tests/unit/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/unit/litellm_core_utils/test_fallback_generalizations.py @@ -369,7 +369,7 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp "output_cost_per_token": 2e-6, "litellm_provider": "openai", "mode": "chat", - "max_tokens": 7, + "max_output_tokens": 7, "supports_reasoning": False, }, "acme-bare": { @@ -398,7 +398,7 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp "name": "acme-backfill", "pattern": r"^acme-", "fill_missing_for_providers": ["openai"], - "model_info": {"supports_reasoning": True, "max_tokens": 5}, + "model_info": {"supports_reasoning": True, "max_output_tokens": 5}, } ] ) @@ -406,10 +406,12 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp full = litellm.get_model_info("acme-full", custom_llm_provider="openai") assert full["supports_reasoning"] is False + assert full["max_output_tokens"] == 7 assert full["max_tokens"] == 7 bare = litellm.get_model_info("acme-bare", custom_llm_provider="openai") assert bare["supports_reasoning"] is True + assert bare["max_output_tokens"] == 5 assert bare["max_tokens"] == 5 assert bare["input_cost_per_token"] == 3e-6 assert bare["key"] == "acme-bare" @@ -525,7 +527,7 @@ def test_shipped_gemini_chat_baseline_loses_to_exact_entries(shipped_cost_map): model = "gemini-2.5-flash-lite" info = litellm.get_model_info(model, custom_llm_provider="gemini") entry = litellm.model_cost["gemini/gemini-2.5-flash-lite"] - assert info["max_tokens"] == entry["max_tokens"] + assert info["max_tokens"] == entry["max_output_tokens"] assert info["input_cost_per_token"] == entry["input_cost_per_token"] assert entry["input_cost_per_token"] > 0 diff --git a/tests/unit/llms/openai_like/test_openinfer_provider.py b/tests/unit/llms/openai_like/test_openinfer_provider.py index 7bcc489fcf8..6e6e957df0b 100644 --- a/tests/unit/llms/openai_like/test_openinfer_provider.py +++ b/tests/unit/llms/openai_like/test_openinfer_provider.py @@ -11,9 +11,7 @@ from litellm.proxy.auth.model_checks import get_provider_models _REPO_ROOT: Final = Path(__file__).resolve().parents[4] _PRIMARY_PRICES: Final = json.loads((_REPO_ROOT / "model_prices_and_context_window.json").read_text()) -_BACKUP_PRICES: Final = json.loads( - (_REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json").read_text() -) +_BACKUP_PRICES: Final = json.loads((_REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json").read_text()) _CATALOG_MODELS: Final = tuple( key for key, row in _PRIMARY_PRICES.items() if isinstance(row, dict) and row.get("litellm_provider") == "openinfer" ) @@ -31,7 +29,9 @@ _CHAT_RESPONSE: Final = { "object": "chat.completion", "created": 1_792_000_000, "model": "@oi/Qwen3.5-27B", - "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello from OpenInfer"}, "finish_reason": "stop"}], + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Hello from OpenInfer"}, "finish_reason": "stop"} + ], "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}, } @@ -228,7 +228,6 @@ def test_catalog_row_matches_its_backup_and_fits_its_context(model: str): assert row == _BACKUP_PRICES[model] assert row["mode"] == "chat" - assert row["max_tokens"] == row["max_output_tokens"] assert 0 < row["max_output_tokens"] <= row["max_input_tokens"] assert row["input_cost_per_token"] > 0 assert row["output_cost_per_token"] > 0 diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 45222b2c9ab..35658097880 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -50,7 +50,6 @@ class _CopilotEndpointRow(BaseModel): mode: str max_input_tokens: int max_output_tokens: int - max_tokens: int supported_endpoints: tuple[str, ...] @@ -96,18 +95,22 @@ def _anthropic_counterpart(key: str) -> str: @pytest.mark.parametrize(("path",), [(PRICES_PATH,), (BACKUP_PRICES_PATH,)], ids=("main", "backup")) -def test_github_copilot_rows_derive_mode_and_max_tokens_from_their_endpoints(path: Path) -> None: +def test_github_copilot_rows_derive_mode_from_their_endpoints(path: Path) -> None: rows: Final[Mapping[str, _CopilotEndpointRow]] = _copilot_endpoint_rows(path) assert rows, f"No GitHub Copilot endpoint rows found in {path}" - assert {key: (row.mode, row.max_tokens) for key, row in rows.items()} == { - key: ( - "chat" if "/v1/chat/completions" in row.supported_endpoints else "responses", - row.max_output_tokens, - ) - for key, row in rows.items() + assert {key: row.mode for key, row in rows.items()} == { + key: "chat" if "/v1/chat/completions" in row.supported_endpoints else "responses" for key, row in rows.items() } +def _rows_carrying_legacy_max_tokens(prices: Mapping[str, Mapping[str, object]]) -> tuple[str, ...]: + return tuple(key for key, row in prices.items() if "max_tokens" in row) + + +def test_no_row_carries_the_retired_max_tokens_field(prices: dict) -> None: + assert _rows_carrying_legacy_max_tokens(prices) == () + + def test_github_copilot_backup_rows_match_the_main_catalog() -> None: assert _github_copilot_catalog_rows(PRICES_PATH) == _github_copilot_catalog_rows(BACKUP_PRICES_PATH) @@ -601,7 +604,6 @@ def test_tencent_tokenhub_glm_rows_match_published_metadata(path: Path): or entry.get("litellm_provider") != "tencent" or entry.get("max_input_tokens") != 1000000 or entry.get("max_output_tokens") != 128000 - or entry.get("max_tokens") != 128000 or entry.get("input_cost_per_token_cache_hit") != entry.get("cache_read_input_token_cost") ) assert actual == TENCENT_TOKENHUB_GLM_ROWS @@ -619,7 +621,7 @@ def test_tencent_tokenhub_kimi_rows_match_published_metadata(path: Path): entry.get("input_cost_per_token"), entry.get("output_cost_per_token"), entry.get("cache_read_input_token_cost"), - entry.get("max_tokens"), + entry.get("max_output_tokens"), entry.get("supports_vision", False), entry.get("supports_video_input", False), ) @@ -793,7 +795,6 @@ def test_volcengine_doubao_seed_2_1_pro_260915_row_matches_published_metadata(pa assert entry["mode"] == "chat" assert entry["max_input_tokens"] == 1024000 assert entry["max_output_tokens"] == 256000 - assert entry["max_tokens"] == 256000 # Volcengine Ark pricing https://www.volcengine.com/docs/82379/1544106 checked 2026-10-20 assert entry["input_cost_per_token"] == pytest.approx(6.0 / 6.9565 / 1_000_000, rel=1e-4) assert entry["output_cost_per_token"] == pytest.approx(30.0 / 6.9565 / 1_000_000, rel=1e-4) diff --git a/tests/unit/test_together_ai_model_metadata.py b/tests/unit/test_together_ai_model_metadata.py index dccba3d66f9..3616c6216ca 100644 --- a/tests/unit/test_together_ai_model_metadata.py +++ b/tests/unit/test_together_ai_model_metadata.py @@ -75,18 +75,6 @@ def cost_map() -> CostMap: return COST_MAP_ADAPTER.validate_python(json.load(f)) -def test_together_chat_entries_never_carry_context_length_as_output_ceiling(cost_map: CostMap): - inflated = sorted( - model - for model, info in cost_map.items() - if info.get("litellm_provider") == "together_ai" - and info.get("mode") == "chat" - and "max_output_tokens" in info - and info["max_output_tokens"] == info.get("max_input_tokens") - ) - assert inflated == [] - - def _successor(info: dict[str, object]) -> str | None: metadata = info.get("metadata") if not isinstance(metadata, dict): @@ -95,8 +83,6 @@ def _successor(info: dict[str, object]) -> str | None: return successor if isinstance(successor, str) else None - - def test_together_backup_cost_map_in_sync(cost_map: CostMap): with open(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json") as f: backup = COST_MAP_ADAPTER.validate_python(json.load(f)) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index b9baef4056a..6c7f6a9d869 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -2260,6 +2260,45 @@ class TestProxyFunctionCalling: @pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map") +def test_get_model_info_derives_max_tokens_from_the_output_limit() -> None: + model: Final = "derived-max-tokens-chat-model" + litellm.register_model( + { + model: { + "litellm_provider": "openai", + "mode": "chat", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + }, + persist_across_reloads=False, + ) + info: Final = litellm.get_model_info(model, custom_llm_provider="openai") + assert info["max_tokens"] == 128000 + assert litellm.get_max_tokens(model) == 128000 + + +def test_get_model_info_derives_max_tokens_from_the_input_limit_when_the_row_has_no_output_limit() -> None: + model: Final = "derived-max-tokens-embedding-model" + litellm.register_model( + { + model: { + "litellm_provider": "openai", + "mode": "embedding", + "max_input_tokens": 8191, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + }, + persist_across_reloads=False, + ) + info: Final = litellm.get_model_info(model, custom_llm_provider="openai") + assert info["max_tokens"] == 8191 + assert info["max_output_tokens"] is None + + def test_register_model_price_is_used_for_completion_cost() -> None: model: Final = "migration-custom-pricing-model" price: Final = { @@ -8629,7 +8668,7 @@ def test_basic_trimming_no_max_tokens_specified(): trimmed_messages = trim_messages(messages, model="gpt-4") print("trimmed messages for gpt-4") print(trimmed_messages) - assert (get_token_count(messages=trimmed_messages, model="gpt-4")) <= litellm.model_cost["gpt-4"]["max_tokens"] + assert (get_token_count(messages=trimmed_messages, model="gpt-4")) <= litellm.model_cost["gpt-4"]["max_input_tokens"] def test_multiple_messages_trimming(): From 554b011787043a4cfc6a48d7d011bbfca05b6e22 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 26 Oct 2026 17:05:04 -0700 Subject: [PATCH 087/155] feat(rust): add typed database query foundation (#48214) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-rust-db.yml | 106 ++++++ litellm-rust/Cargo.lock | 5 + litellm-rust/Cargo.toml | 1 + ...64311d25428fd393bac4250891f148597e92e.json | 78 ++++ litellm-rust/crates/db/AGENTS.md | 25 ++ litellm-rust/crates/db/Cargo.toml | 7 + .../db/queries/end_user/find_end_user.sql | 10 + litellm-rust/crates/db/src/daily_spend.rs | 76 ++++ litellm-rust/crates/db/src/end_user.rs | 359 ++++++++++++++++++ litellm-rust/crates/db/src/error.rs | 79 ++++ litellm-rust/crates/db/src/lib.rs | 18 +- litellm-rust/crates/db/src/test_support.rs | 10 + 12 files changed, 773 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test-rust-db.yml create mode 100644 litellm-rust/crates/db/.sqlx/query-377d063cd0fc3a265d07d01f5aa64311d25428fd393bac4250891f148597e92e.json create mode 100644 litellm-rust/crates/db/AGENTS.md create mode 100644 litellm-rust/crates/db/queries/end_user/find_end_user.sql create mode 100644 litellm-rust/crates/db/src/daily_spend.rs create mode 100644 litellm-rust/crates/db/src/end_user.rs create mode 100644 litellm-rust/crates/db/src/error.rs create mode 100644 litellm-rust/crates/db/src/test_support.rs diff --git a/.github/workflows/test-rust-db.yml b/.github/workflows/test-rust-db.yml new file mode 100644 index 00000000000..bb88b593d5b --- /dev/null +++ b/.github/workflows/test-rust-db.yml @@ -0,0 +1,106 @@ +name: LiteLLM Rust DB + +on: + push: + paths: + - "litellm-rust/crates/db/**" + - "litellm-rust/Cargo.toml" + - "litellm-rust/Cargo.lock" + - "litellm-rust/clippy.toml" + - "litellm-proxy-extras/litellm_proxy_extras/migrations/**" + - "**/schema.prisma" + - ".cargo/**" + - "rust-toolchain.toml" + - "pyproject.toml" + - "uv.lock" + - ".github/actions/setup-uv-with-retries/**" + - ".github/workflows/test-rust-db.yml" + pull_request: + branches: + - main + - "litellm_**" + paths: + - "litellm-rust/crates/db/**" + - "litellm-rust/Cargo.toml" + - "litellm-rust/Cargo.lock" + - "litellm-rust/clippy.toml" + - "litellm-proxy-extras/litellm_proxy_extras/migrations/**" + - "**/schema.prisma" + - ".cargo/**" + - "rust-toolchain.toml" + - "pyproject.toml" + - "uv.lock" + - ".github/actions/setup-uv-with-retries/**" + - ".github/workflows/test-rust-db.yml" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + +jobs: + db-queries: + runs-on: ubuntu-latest + timeout-minutes: 30 + services: + postgres: + image: postgres:16.6@sha256:557fea37a744d5f4c8faab304b0a90858b53ab119735a88c131fd19dab802f36 + env: + POSTGRES_DB: litellm + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d litellm" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm + SQLX_OFFLINE: "false" + defaults: + run: + working-directory: litellm-rust + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - run: rustup toolchain install --no-self-update + + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: litellm-rust + cache-on-failure: true + + - name: Install Python dependencies + working-directory: . + run: uv sync --frozen --extra extra-proxy + + - name: Deploy Prisma migrations + working-directory: . + run: uv run --no-sync prisma migrate deploy --schema litellm-proxy-extras/litellm_proxy_extras/schema.prisma + + - name: Install SQLx CLI + run: cargo install sqlx-cli --version 0.9.0 --locked --no-default-features -F postgres + + - name: Check SQLx query metadata + working-directory: litellm-rust/crates/db + run: cargo sqlx prepare --check + + - name: Test database queries + run: cargo test -p litellm-db --locked -- --include-ignored diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index cb50324e1dd..d58ba9ecb5f 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3695,8 +3695,13 @@ dependencies = [ name = "litellm-db" version = "0.1.0" dependencies = [ + "chrono", + "macro_rules_attribute", + "rstest", "serde", "sqlx", + "thiserror 2.0.19", + "tokio", ] [[package]] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 9e5e0a96b73..fc65355c047 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -104,6 +104,7 @@ serde_with = { version = "=3.21.0", default-features = false, features = ["std", sha2 = "0.10" syn = { version = "2", default-features = false } sqlx = { version = "0.9.0", default-features = false, features = ["json", "macros", "postgres", "runtime-tokio", "chrono", "tls-rustls-ring-native-roots"] } +chrono = { version = "0.4.45", default-features = false, features = ["std"] } proc-macro2 = "1" quote = "1" subtle = "2" diff --git a/litellm-rust/crates/db/.sqlx/query-377d063cd0fc3a265d07d01f5aa64311d25428fd393bac4250891f148597e92e.json b/litellm-rust/crates/db/.sqlx/query-377d063cd0fc3a265d07d01f5aa64311d25428fd393bac4250891f148597e92e.json new file mode 100644 index 00000000000..734935466ff --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-377d063cd0fc3a265d07d01f5aa64311d25428fd393bac4250891f148597e92e.json @@ -0,0 +1,78 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT e.spend AS \"spend?\",\n e.blocked AS \"blocked?\",\n b.max_budget AS \"max_budget?\",\n b.rpm_limit AS \"rpm_limit?\",\n b.tpm_limit AS \"tpm_limit?\"\nFROM (SELECT 1) AS one\nLEFT JOIN \"LiteLLM_EndUserTable\" AS e ON e.user_id = $1\nLEFT JOIN \"LiteLLM_BudgetTable\" AS assigned ON assigned.budget_id = e.budget_id\nLEFT JOIN \"LiteLLM_BudgetTable\" AS key_budget ON key_budget.budget_id = $3\nLEFT JOIN \"LiteLLM_BudgetTable\" AS b ON b.budget_id = COALESCE(assigned.budget_id, key_budget.budget_id, $2)\n", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "spend?", + "type_info": "Float8", + "origin": { + "Table": { + "table": "\"LiteLLM_EndUserTable\"", + "name": "spend" + } + } + }, + { + "ordinal": 1, + "name": "blocked?", + "type_info": "Bool", + "origin": { + "Table": { + "table": "\"LiteLLM_EndUserTable\"", + "name": "blocked" + } + } + }, + { + "ordinal": 2, + "name": "max_budget?", + "type_info": "Float8", + "origin": { + "Table": { + "table": "\"LiteLLM_BudgetTable\"", + "name": "max_budget" + } + } + }, + { + "ordinal": 3, + "name": "rpm_limit?", + "type_info": "Int8", + "origin": { + "Table": { + "table": "\"LiteLLM_BudgetTable\"", + "name": "rpm_limit" + } + } + }, + { + "ordinal": 4, + "name": "tpm_limit?", + "type_info": "Int8", + "origin": { + "Table": { + "table": "\"LiteLLM_BudgetTable\"", + "name": "tpm_limit" + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true, + true, + true, + true, + true + ] + }, + "hash": "377d063cd0fc3a265d07d01f5aa64311d25428fd393bac4250891f148597e92e" +} diff --git a/litellm-rust/crates/db/AGENTS.md b/litellm-rust/crates/db/AGENTS.md new file mode 100644 index 00000000000..1e08317230e --- /dev/null +++ b/litellm-rust/crates/db/AGENTS.md @@ -0,0 +1,25 @@ +# Database crate + +Keep each static query in `queries/` and call it through a typed SQLx query function. Rust does not create or migrate the database + +- Own reusable row projections in `src/` and export those consumed by other crates or integration tests +- Apply `#[macro_rules_attribute::apply(crate::db_row)]` to row types for shared derives, without adding serialization or schema derives automatically +- Apply `#[macro_rules_attribute::apply(crate::db_input)]` to write inputs, adding `Default` only where supported +- Keep function tests inline in their owning query module and reuse the strict database fixture in `test_support` +- Transaction-local setup in `begin` is the approved exception to static query files +- Keep write inputs separate from stored rows and keep test-only assertions out of public types +- `FromRow` supports runtime reads, but production static reads still use `query_file_as!` for compile-time checking + +Database integration tests are ignored by default, but still compile in workspace checks. Run them explicitly against a local database already migrated by Prisma. Missing or invalid database configuration must fail the selected tests: + +```sh +DATABASE_URL=postgresql://... cargo test -p litellm-db --locked -- --include-ignored +``` + +After changing a query, refresh the checked query metadata from this directory: + +```sh +SQLX_OFFLINE=false cargo sqlx prepare +``` + +Tests use transactions and leave them uncommitted so fixture rows roll back when each test finishes diff --git a/litellm-rust/crates/db/Cargo.toml b/litellm-rust/crates/db/Cargo.toml index ca9d6f555b0..bb6d5655b60 100644 --- a/litellm-rust/crates/db/Cargo.toml +++ b/litellm-rust/crates/db/Cargo.toml @@ -6,5 +6,12 @@ license.workspace = true repository.workspace = true [dependencies] +chrono.workspace = true +macro_rules_attribute.workspace = true serde.workspace = true sqlx.workspace = true +thiserror.workspace = true + +[dev-dependencies] +rstest.workspace = true +tokio.workspace = true diff --git a/litellm-rust/crates/db/queries/end_user/find_end_user.sql b/litellm-rust/crates/db/queries/end_user/find_end_user.sql new file mode 100644 index 00000000000..c10ef8b877d --- /dev/null +++ b/litellm-rust/crates/db/queries/end_user/find_end_user.sql @@ -0,0 +1,10 @@ +SELECT e.spend AS "spend?", + e.blocked AS "blocked?", + b.max_budget AS "max_budget?", + b.rpm_limit AS "rpm_limit?", + b.tpm_limit AS "tpm_limit?" +FROM (SELECT 1) AS one +LEFT JOIN "LiteLLM_EndUserTable" AS e ON e.user_id = $1 +LEFT JOIN "LiteLLM_BudgetTable" AS assigned ON assigned.budget_id = e.budget_id +LEFT JOIN "LiteLLM_BudgetTable" AS key_budget ON key_budget.budget_id = $3 +LEFT JOIN "LiteLLM_BudgetTable" AS b ON b.budget_id = COALESCE(assigned.budget_id, key_budget.budget_id, $2) diff --git a/litellm-rust/crates/db/src/daily_spend.rs b/litellm-rust/crates/db/src/daily_spend.rs new file mode 100644 index 00000000000..9db07514aab --- /dev/null +++ b/litellm-rust/crates/db/src/daily_spend.rs @@ -0,0 +1,76 @@ +use sqlx::types::chrono::NaiveDateTime; + +#[macro_rules_attribute::apply(crate::db_row)] +pub struct DailyUserRow { + pub id: String, + pub user_id: Option, + pub api_key: String, + pub model: Option, + pub custom_llm_provider: Option, + pub endpoint: Option, + pub model_group: Option, + pub mcp_namespaced_tool_name: Option, + pub prompt_tokens: i64, + pub completion_tokens: i64, + pub api_requests: i64, + pub successful_requests: i64, + pub failed_requests: i64, + pub cache_read_input_tokens: i64, + pub cache_creation_input_tokens: i64, + pub total_response_time_ms: i64, + pub timed_requests: i64, + pub spend: f64, + pub date: String, + pub created_at: NaiveDateTime, + pub updated_at: NaiveDateTime, +} + +#[macro_rules_attribute::apply(crate::db_row)] +pub struct DailyTeamRow { + pub id: String, + pub team_id: Option, + pub api_key: String, + pub model: Option, + pub custom_llm_provider: Option, + pub endpoint: Option, + pub model_group: Option, + pub mcp_namespaced_tool_name: Option, + pub prompt_tokens: i64, + pub completion_tokens: i64, + pub api_requests: i64, + pub successful_requests: i64, + pub failed_requests: i64, + pub cache_read_input_tokens: i64, + pub cache_creation_input_tokens: i64, + pub total_response_time_ms: i64, + pub timed_requests: i64, + pub spend: f64, + pub date: String, + pub created_at: NaiveDateTime, + pub updated_at: NaiveDateTime, +} + +#[macro_rules_attribute::apply(crate::db_row)] +pub struct DailyEndUserRow { + pub id: String, + pub end_user_id: Option, + pub api_key: String, + pub model: Option, + pub custom_llm_provider: Option, + pub endpoint: Option, + pub model_group: Option, + pub mcp_namespaced_tool_name: Option, + pub prompt_tokens: i64, + pub completion_tokens: i64, + pub api_requests: i64, + pub successful_requests: i64, + pub failed_requests: i64, + pub cache_read_input_tokens: i64, + pub cache_creation_input_tokens: i64, + pub total_response_time_ms: i64, + pub timed_requests: i64, + pub spend: f64, + pub date: String, + pub created_at: NaiveDateTime, + pub updated_at: NaiveDateTime, +} diff --git a/litellm-rust/crates/db/src/end_user.rs b/litellm-rust/crates/db/src/end_user.rs new file mode 100644 index 00000000000..66661b51640 --- /dev/null +++ b/litellm-rust/crates/db/src/end_user.rs @@ -0,0 +1,359 @@ +use sqlx::PgExecutor; + +use crate::Error; + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct RateLimits { + pub rpm: Option, + pub tpm: Option, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct EndUser { + pub spend: Option, + pub blocked: bool, + pub max_budget: Option, + pub limits: RateLimits, +} + +#[macro_rules_attribute::apply(crate::db_row)] +struct EndUserRow { + spend: Option, + blocked: Option, + max_budget: Option, + rpm_limit: Option, + tpm_limit: Option, +} + +pub async fn find_end_user( + executor: impl PgExecutor<'_>, + end_user_id: &str, + default_budget_id: Option<&str>, + key_budget_id: Option<&str>, +) -> Result { + let row = sqlx::query_file_as!( + EndUserRow, + "queries/end_user/find_end_user.sql", + end_user_id, + default_budget_id, + key_budget_id, + ) + .fetch_one(executor) + .await?; + Ok(EndUser { + spend: row.spend, + blocked: row.blocked.unwrap_or(false), + max_budget: row.max_budget, + limits: RateLimits { + rpm: row.rpm_limit, + tpm: row.tpm_limit, + }, + }) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{EndUser, RateLimits, find_end_user}; + use crate::test_support::database_pool; + use rstest::rstest; + use sqlx::{PgPool, Postgres, Transaction}; + + #[derive(Clone, Copy)] + struct BudgetFixture { + budget_id: &'static str, + max_budget: f64, + rpm_limit: i64, + tpm_limit: i64, + } + + async fn insert_budget( + tx: &mut Transaction<'_, Postgres>, + budget: BudgetFixture, + ) -> Result<(), sqlx::Error> { + #[expect( + clippy::disallowed_methods, + reason = "Fixture inserts are outside the compile-checked query set" + )] + let result = sqlx::query( + r#" + INSERT INTO "LiteLLM_BudgetTable" + (budget_id, max_budget, rpm_limit, tpm_limit, created_by, updated_by) + VALUES ($1, $2, $3, $4, 'db-query-test', 'db-query-test') + "#, + ) + .bind(budget.budget_id) + .bind(budget.max_budget) + .bind(budget.rpm_limit) + .bind(budget.tpm_limit) + .execute(&mut **tx) + .await; + result.map(|_| ()) + } + + async fn insert_end_user( + tx: &mut Transaction<'_, Postgres>, + user_id: &str, + spend: f64, + blocked: bool, + budget_id: Option<&str>, + ) -> Result<(), sqlx::Error> { + #[expect( + clippy::disallowed_methods, + reason = "Fixture inserts are outside the compile-checked query set" + )] + let result = sqlx::query( + r#" + INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked, budget_id) + VALUES ($1, $2, $3, $4) + "#, + ) + .bind(user_id) + .bind(spend) + .bind(blocked) + .bind(budget_id) + .execute(&mut **tx) + .await; + result.map(|_| ()) + } + + #[rstest] + #[case::without_default_budget(None)] + #[case::with_default_budget(Some(BudgetFixture { + budget_id: "missing-user-default-budget", + max_budget: 11.5, + rpm_limit: 12, + tpm_limit: 1300, + }))] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn missing_end_user_maps_to_defaults( + #[case] default_budget: Option, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + if let Some(budget) = default_budget { + insert_budget(&mut tx, budget).await?; + } + + let actual = find_end_user( + &mut *tx, + "missing-end-user", + default_budget.map(|budget| budget.budget_id), + None, + ) + .await?; + + assert_eq!( + actual, + EndUser { + spend: None, + blocked: false, + max_budget: default_budget.map(|budget| budget.max_budget), + limits: RateLimits { + rpm: default_budget.map(|budget| budget.rpm_limit), + tpm: default_budget.map(|budget| budget.tpm_limit), + }, + } + ); + Ok(()) + } + + #[rstest] + #[case::row_budget_precedes_default( + "row-budget-priority-user", + Some(BudgetFixture { + budget_id: "row-budget-priority-row", + max_budget: 10.0, + rpm_limit: 101, + tpm_limit: 1001, + }), + BudgetFixture { + budget_id: "row-budget-priority-default", + max_budget: 20.0, + rpm_limit: 201, + tpm_limit: 2001, + }, + None, + Some(BudgetFixture { + budget_id: "row-budget-priority-row", + max_budget: 10.0, + rpm_limit: 101, + tpm_limit: 1001, + }), + false + )] + #[case::default_applies_without_row_budget( + "default-budget-user", + None, + BudgetFixture { + budget_id: "default-budget-fallback", + max_budget: 30.0, + rpm_limit: 301, + tpm_limit: 3001, + }, + None, + Some(BudgetFixture { + budget_id: "default-budget-fallback", + max_budget: 30.0, + rpm_limit: 301, + tpm_limit: 3001, + }), + false + )] + #[case::assigned_budget_precedes_key_when_all_ids_differ( + "override-priority-user", + Some(BudgetFixture { + budget_id: "override-priority-row", + max_budget: 40.0, + rpm_limit: 401, + tpm_limit: 4001, + }), + BudgetFixture { + budget_id: "override-priority-default", + max_budget: 50.0, + rpm_limit: 501, + tpm_limit: 5001, + }, + Some(BudgetFixture { + budget_id: "override-priority-override", + max_budget: 60.0, + rpm_limit: 601, + tpm_limit: 6001, + }), + Some(BudgetFixture { + budget_id: "override-priority-row", + max_budget: 40.0, + rpm_limit: 401, + tpm_limit: 4001, + }), + true + )] + #[case::missing_key_budget_keeps_assigned_budget( + "dangling-override-user", + Some(BudgetFixture { + budget_id: "dangling-override-row", + max_budget: 70.0, + rpm_limit: 701, + tpm_limit: 7001, + }), + BudgetFixture { + budget_id: "dangling-override-default", + max_budget: 80.0, + rpm_limit: 801, + tpm_limit: 8001, + }, + Some(BudgetFixture { + budget_id: "dangling-override-missing", + max_budget: 90.0, + rpm_limit: 901, + tpm_limit: 9001, + }), + Some(BudgetFixture { + budget_id: "dangling-override-row", + max_budget: 70.0, + rpm_limit: 701, + tpm_limit: 7001, + }), + false + )] + #[case::key_budget_precedes_proxy_default( + "key-default-user", + None, + BudgetFixture { + budget_id: "key-default-proxy", + max_budget: 80.0, + rpm_limit: 801, + tpm_limit: 8001, + }, + Some(BudgetFixture { + budget_id: "key-default-key", + max_budget: 90.0, + rpm_limit: 901, + tpm_limit: 9001, + }), + Some(BudgetFixture { + budget_id: "key-default-key", + max_budget: 90.0, + rpm_limit: 901, + tpm_limit: 9001, + }), + true + )] + #[case::missing_key_budget_falls_back_to_proxy_default( + "missing-key-default-user", + None, + BudgetFixture { + budget_id: "missing-key-default-proxy", + max_budget: 80.0, + rpm_limit: 801, + tpm_limit: 8001, + }, + Some(BudgetFixture { + budget_id: "missing-key-default-key", + max_budget: 90.0, + rpm_limit: 901, + tpm_limit: 9001, + }), + Some(BudgetFixture { + budget_id: "missing-key-default-proxy", + max_budget: 80.0, + rpm_limit: 801, + tpm_limit: 8001, + }), + false + )] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn budget_resolution_matches_python_precedence( + #[case] user_id: &'static str, + #[case] user_budget: Option, + #[case] default_budget: BudgetFixture, + #[case] override_budget: Option, + #[case] expected_budget: Option, + #[case] insert_override_budget: bool, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + insert_budget(&mut tx, default_budget).await?; + if let Some(budget) = user_budget { + insert_budget(&mut tx, budget).await?; + } + if let (Some(budget), true) = (override_budget, insert_override_budget) { + insert_budget(&mut tx, budget).await?; + } + insert_end_user( + &mut tx, + user_id, + 7.5, + true, + user_budget.map(|budget| budget.budget_id), + ) + .await?; + + let actual = find_end_user( + &mut *tx, + user_id, + Some(default_budget.budget_id), + override_budget.map(|budget| budget.budget_id), + ) + .await?; + + assert_eq!( + actual, + EndUser { + spend: Some(7.5), + blocked: true, + max_budget: expected_budget.map(|budget| budget.max_budget), + limits: RateLimits { + rpm: expected_budget.map(|budget| budget.rpm_limit), + tpm: expected_budget.map(|budget| budget.tpm_limit), + }, + } + ); + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/error.rs b/litellm-rust/crates/db/src/error.rs new file mode 100644 index 00000000000..af0ee030896 --- /dev/null +++ b/litellm-rust/crates/db/src/error.rs @@ -0,0 +1,79 @@ +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("database query failed")] + Query(#[from] sqlx::Error), + #[error("spend batch has {ids} ids and {costs} costs")] + InvalidSpendBatch { ids: usize, costs: usize }, + #[error("invalid spend timestamp")] + InvalidTimestamp(#[from] chrono::ParseError), +} + +impl Error { + pub fn is_retryable(&self) -> bool { + match self { + Self::Query(sqlx::Error::Database(error)) => { + matches!( + error.code().as_deref(), + Some("40P01" | "55P03" | "40001" | "53300") + ) + } + Self::Query(sqlx::Error::Io(_) | sqlx::Error::PoolTimedOut) => true, + _ => false, + } + } +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use rstest::rstest; + use sqlx::PgPool; + + use super::Error; + use crate::test_support::database_pool; + + #[rstest] + #[case::closed_pool(sqlx::Error::PoolClosed, false)] + #[case::pool_timeout(sqlx::Error::PoolTimedOut, true)] + #[case::io(sqlx::Error::Io(std::io::ErrorKind::ConnectionReset.into()), true)] + #[case::missing_row(sqlx::Error::RowNotFound, false)] + fn classifies_client_errors(#[case] error: sqlx::Error, #[case] retryable: bool) { + assert_eq!(Error::from(error).is_retryable(), retryable); + } + + #[rstest] + #[case::connection_capacity("53300", true)] + #[case::deadlock("40P01", true)] + #[case::lock_unavailable("55P03", true)] + #[case::serialization_failure("40001", true)] + #[case::unique_violation("23505", false)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn classifies_server_errors( + #[case] code: &str, + #[case] retryable: bool, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Test query must raise the requested SQLSTATE" + )] + let error = sqlx::query(sqlx::AssertSqlSafe(format!( + "DO $$ BEGIN RAISE EXCEPTION USING ERRCODE = '{code}'; END $$" + ))) + .execute(&mut *tx) + .await + .expect_err("PostgreSQL must raise the requested error"); + + assert_eq!( + error.as_database_error().and_then(|error| error.code()), + Some(code.into()) + ); + assert_eq!(Error::from(error).is_retryable(), retryable); + tx.rollback().await?; + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/lib.rs b/litellm-rust/crates/db/src/lib.rs index b7ce211d906..ca5cd366eec 100644 --- a/litellm-rust/crates/db/src/lib.rs +++ b/litellm-rust/crates/db/src/lib.rs @@ -1,3 +1,19 @@ -mod failure; +macro_rules_attribute::attribute_alias! { + #[apply(db_input)] = + #[derive(Clone, Debug, PartialEq)]; + #[apply(db_row)] = + #[derive(Clone, Debug, PartialEq, sqlx::FromRow)]; +} + +mod daily_spend; +mod end_user; +mod error; +mod failure; +#[cfg(test)] +mod test_support; + +pub use daily_spend::{DailyEndUserRow, DailyTeamRow, DailyUserRow}; +pub use end_user::{EndUser, RateLimits, find_end_user}; +pub use error::Error; pub use failure::DbFailure; diff --git a/litellm-rust/crates/db/src/test_support.rs b/litellm-rust/crates/db/src/test_support.rs new file mode 100644 index 00000000000..992be4a4447 --- /dev/null +++ b/litellm-rust/crates/db/src/test_support.rs @@ -0,0 +1,10 @@ +use std::error::Error as StdError; + +use rstest::fixture; +use sqlx::PgPool; + +#[fixture] +pub async fn database_pool() -> Result> { + let database_url = std::env::var("DATABASE_URL")?; + Ok(PgPool::connect(&database_url).await?) +} From 35cd5fc05f7e91c7bc9c91637e14452a3ed55360 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 26 Oct 2026 17:05:04 -0700 Subject: [PATCH 088/155] feat(db): add typed key spend query (#48217) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...10731a0cacc1ae5b92eec32967f539948f563.json | 15 + .../crates/db/queries/spend/add_key_spend.sql | 16 + litellm-rust/crates/db/src/lib.rs | 2 + litellm-rust/crates/db/src/spend/key_spend.rs | 309 ++++++++++++++++++ litellm-rust/crates/db/src/spend/mod.rs | 3 + 5 files changed, 345 insertions(+) create mode 100644 litellm-rust/crates/db/.sqlx/query-3829a383d0ae50578f0fd75da3910731a0cacc1ae5b92eec32967f539948f563.json create mode 100644 litellm-rust/crates/db/queries/spend/add_key_spend.sql create mode 100644 litellm-rust/crates/db/src/spend/key_spend.rs create mode 100644 litellm-rust/crates/db/src/spend/mod.rs diff --git a/litellm-rust/crates/db/.sqlx/query-3829a383d0ae50578f0fd75da3910731a0cacc1ae5b92eec32967f539948f563.json b/litellm-rust/crates/db/.sqlx/query-3829a383d0ae50578f0fd75da3910731a0cacc1ae5b92eec32967f539948f563.json new file mode 100644 index 00000000000..e85c0986a60 --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-3829a383d0ae50578f0fd75da3910731a0cacc1ae5b92eec32967f539948f563.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH added AS (\n SELECT id, SUM(cost) AS cost\n FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)\n GROUP BY id\n), locked AS (\n SELECT t.token FROM \"LiteLLM_VerificationToken\" t\n WHERE t.token IN (SELECT id FROM added)\n ORDER BY t.token FOR UPDATE\n)\nUPDATE \"LiteLLM_VerificationToken\" t\nSET spend = t.spend + added.cost,\n total_spend = t.total_spend + added.cost,\n last_active = (clock_timestamp() AT TIME ZONE 'UTC'),\n updated_at = (clock_timestamp() AT TIME ZONE 'UTC')\nFROM added\nWHERE t.token = added.id AND t.token IN (SELECT token FROM locked)\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "Float8Array" + ] + }, + "nullable": [] + }, + "hash": "3829a383d0ae50578f0fd75da3910731a0cacc1ae5b92eec32967f539948f563" +} diff --git a/litellm-rust/crates/db/queries/spend/add_key_spend.sql b/litellm-rust/crates/db/queries/spend/add_key_spend.sql new file mode 100644 index 00000000000..61850714acd --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/add_key_spend.sql @@ -0,0 +1,16 @@ +WITH added AS ( + SELECT id, SUM(cost) AS cost + FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost) + GROUP BY id +), locked AS ( + SELECT t.token FROM "LiteLLM_VerificationToken" t + WHERE t.token IN (SELECT id FROM added) + ORDER BY t.token FOR UPDATE +) +UPDATE "LiteLLM_VerificationToken" t +SET spend = t.spend + added.cost, + total_spend = t.total_spend + added.cost, + last_active = (clock_timestamp() AT TIME ZONE 'UTC'), + updated_at = (clock_timestamp() AT TIME ZONE 'UTC') +FROM added +WHERE t.token = added.id AND t.token IN (SELECT token FROM locked) diff --git a/litellm-rust/crates/db/src/lib.rs b/litellm-rust/crates/db/src/lib.rs index ca5cd366eec..b3abf5f5173 100644 --- a/litellm-rust/crates/db/src/lib.rs +++ b/litellm-rust/crates/db/src/lib.rs @@ -10,6 +10,7 @@ mod daily_spend; mod end_user; mod error; mod failure; +mod spend; #[cfg(test)] mod test_support; @@ -17,3 +18,4 @@ pub use daily_spend::{DailyEndUserRow, DailyTeamRow, DailyUserRow}; pub use end_user::{EndUser, RateLimits, find_end_user}; pub use error::Error; pub use failure::DbFailure; +pub use spend::{EntitySpend, add_key_spend}; diff --git a/litellm-rust/crates/db/src/spend/key_spend.rs b/litellm-rust/crates/db/src/spend/key_spend.rs new file mode 100644 index 00000000000..5a6ba13307c --- /dev/null +++ b/litellm-rust/crates/db/src/spend/key_spend.rs @@ -0,0 +1,309 @@ +use sqlx::PgExecutor; +use std::collections::BTreeMap; + +use crate::Error; + +#[macro_rules_attribute::apply(crate::db_input)] +#[derive(Default)] +pub struct EntitySpend { + pub ids: Vec, + pub costs: Vec, +} + +impl EntitySpend { + pub(crate) fn validate(&self) -> Result<(), Error> { + if self.ids.len() != self.costs.len() { + return Err(Error::InvalidSpendBatch { + ids: self.ids.len(), + costs: self.costs.len(), + }); + } + Ok(()) + } + + pub fn is_empty(&self) -> bool { + self.ids.is_empty() + } +} + +impl FromIterator<(String, f64)> for EntitySpend { + fn from_iter>(entries: I) -> Self { + let entries = entries + .into_iter() + .fold(BTreeMap::new(), |mut sums, (id, cost)| { + *sums.entry(id).or_insert(0.0) += cost; + sums + }); + let (ids, costs) = entries.into_iter().unzip(); + Self { ids, costs } + } +} + +pub async fn add_key_spend( + executor: impl PgExecutor<'_>, + spend: &EntitySpend, +) -> Result<(), Error> { + spend.validate()?; + if spend.is_empty() { + return Ok(()); + } + sqlx::query_file!("queries/spend/add_key_spend.sql", &spend.ids, &spend.costs) + .execute(executor) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{EntitySpend, add_key_spend}; + use crate::{Error, test_support::database_pool}; + use rstest::rstest; + use sqlx::PgPool; + + #[rstest] + #[case::direct(false)] + #[case::collected(true)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn duplicate_key_spend_accumulates( + #[case] collected: bool, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + sqlx::query( + r#"INSERT INTO "LiteLLM_VerificationToken" (token, spend, total_spend, updated_at) + VALUES ('key-spend-duplicate', 2, 7, '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let spend = if collected { + [ + ("key-spend-duplicate".into(), 0.25), + ("key-spend-duplicate".into(), 0.75), + ] + .into_iter() + .collect() + } else { + EntitySpend { + ids: vec!["key-spend-duplicate".into(), "key-spend-duplicate".into()], + costs: vec![0.25, 0.75], + } + }; + add_key_spend(&mut *tx, &spend).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let row: (f64, f64) = sqlx::query_as( + r#"SELECT spend, total_spend FROM "LiteLLM_VerificationToken" + WHERE token = 'key-spend-duplicate'"#, + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!(row, (3.0, 8.0)); + Ok(()) + } + + #[rstest] + #[case::extra_id(vec!["key-spend-invalid".into(), "key-spend-invalid".into()], vec![0.25])] + #[case::extra_cost(vec!["key-spend-invalid".into()], vec![0.25, 0.75])] + #[case::empty_ids(vec![], vec![0.25])] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn malformed_key_spend_leaves_transaction_usable( + #[case] ids: Vec, + #[case] costs: Vec, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + sqlx::query( + r#"INSERT INTO "LiteLLM_VerificationToken" + (token, spend, total_spend, last_active, updated_at) + VALUES ('key-spend-invalid', 2, 7, NULL, '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let spend = EntitySpend { ids, costs }; + let result = add_key_spend(&mut *tx, &spend).await; + assert!(matches!( + result, + Err(Error::InvalidSpendBatch { ids, costs }) + if ids == spend.ids.len() && costs == spend.costs.len() + )); + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let unchanged: (f64, f64, bool, bool) = sqlx::query_as( + r#"SELECT spend, total_spend, last_active IS NULL, + updated_at = TIMESTAMP '2000-01-01' + FROM "LiteLLM_VerificationToken" WHERE token = 'key-spend-invalid'"#, + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!(unchanged, (2.0, 7.0, true, true)); + let valid = EntitySpend { + ids: vec!["key-spend-invalid".into()], + costs: vec![0.5], + }; + add_key_spend(&mut *tx, &valid).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let updated: (f64, f64) = sqlx::query_as( + r#"SELECT spend, total_spend FROM "LiteLLM_VerificationToken" + WHERE token = 'key-spend-invalid'"#, + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!(updated, (2.5, 7.5)); + Ok(()) + } + + #[rstest] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn key_activity_uses_update_time_in_old_transaction( + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + sqlx::query( + r#"INSERT INTO "LiteLLM_VerificationToken" (token, spend, total_spend, updated_at) + VALUES ('key-spend-old-transaction', 2, 7, '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + #[expect( + clippy::disallowed_methods, + reason = "Transaction timing setup is outside the compile-checked query set" + )] + sqlx::query("SELECT pg_sleep(0.02)") + .execute(&mut *tx) + .await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let boundary: chrono::NaiveDateTime = sqlx::query_scalar( + "SELECT date_trunc('milliseconds', clock_timestamp() AT TIME ZONE 'UTC')", + ) + .fetch_one(&mut *tx) + .await?; + let spend = EntitySpend { + ids: vec!["key-spend-old-transaction".into()], + costs: vec![0.25], + }; + add_key_spend(&mut *tx, &spend).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let row: (chrono::NaiveDateTime, chrono::NaiveDateTime) = sqlx::query_as( + r#"SELECT last_active, updated_at FROM "LiteLLM_VerificationToken" + WHERE token = 'key-spend-old-transaction'"#, + ) + .fetch_one(&mut *tx) + .await?; + assert!(row.0 >= boundary, "last_active predates update boundary"); + assert!(row.1 >= boundary, "updated_at predates update boundary"); + Ok(()) + } + + #[rstest] + #[case::fractional_costs(0.25, 2.5, 7.5, 4.0, 12.0)] + #[case::zero_costs(0.0, 2.0, 7.0, 3.0, 11.0)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn key_spend_matches_python_update_many( + #[case] cost: f64, + #[case] expected_a_spend: f64, + #[case] expected_a_total: f64, + #[case] expected_z_spend: f64, + #[case] expected_z_total: f64, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _inserted = sqlx::query( + r#"INSERT INTO "LiteLLM_VerificationToken" + (token, spend, total_spend, last_active, updated_at) + VALUES ('key-spend-a', 2, 7, NULL, '2000-01-01'), + ('key-spend-z', 3, 11, NULL, '2000-01-01'), + ('key-spend-untouched', 100, 200, NULL, '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let spend = EntitySpend { + ids: vec![ + "key-spend-z".into(), + "key-spend-a".into(), + "key-spend-missing".into(), + ], + costs: vec![2.0 * cost, cost, 50.0], + }; + add_key_spend(&mut *tx, &EntitySpend::default()).await?; + add_key_spend(&mut *tx, &spend).await?; + add_key_spend(&mut *tx, &spend).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let rows: Vec<(String, f64, f64, bool, bool)> = sqlx::query_as( + r#"SELECT token, spend, total_spend, last_active IS NOT NULL, + updated_at > TIMESTAMP '2000-01-01' + FROM "LiteLLM_VerificationToken" + WHERE token = ANY($1) ORDER BY token"#, + ) + .bind(vec![ + "key-spend-a", + "key-spend-z", + "key-spend-missing", + "key-spend-untouched", + ]) + .fetch_all(&mut *tx) + .await?; + assert_eq!( + rows, + vec![ + ( + "key-spend-a".into(), + expected_a_spend, + expected_a_total, + true, + true + ), + ("key-spend-untouched".into(), 100.0, 200.0, false, false), + ( + "key-spend-z".into(), + expected_z_spend, + expected_z_total, + true, + true + ), + ] + ); + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/spend/mod.rs b/litellm-rust/crates/db/src/spend/mod.rs new file mode 100644 index 00000000000..e34886c6130 --- /dev/null +++ b/litellm-rust/crates/db/src/spend/mod.rs @@ -0,0 +1,3 @@ +mod key_spend; + +pub use key_spend::{EntitySpend, add_key_spend}; From 8e61c22fe0be640b4cfb0d22c9bfa178c250cc45 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 26 Oct 2026 17:05:04 -0700 Subject: [PATCH 089/155] feat(db): add typed user spend query (#48229) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...3b87a20a2664b0bef3616631d2c24487c1e54.json | 15 ++ .../db/queries/spend/add_user_spend.sql | 14 ++ litellm-rust/crates/db/src/lib.rs | 2 +- litellm-rust/crates/db/src/spend/mod.rs | 2 + .../crates/db/src/spend/user_spend.rs | 173 ++++++++++++++++++ 5 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 litellm-rust/crates/db/.sqlx/query-43f2e12a3ffb1de29b5b031a1173b87a20a2664b0bef3616631d2c24487c1e54.json create mode 100644 litellm-rust/crates/db/queries/spend/add_user_spend.sql create mode 100644 litellm-rust/crates/db/src/spend/user_spend.rs diff --git a/litellm-rust/crates/db/.sqlx/query-43f2e12a3ffb1de29b5b031a1173b87a20a2664b0bef3616631d2c24487c1e54.json b/litellm-rust/crates/db/.sqlx/query-43f2e12a3ffb1de29b5b031a1173b87a20a2664b0bef3616631d2c24487c1e54.json new file mode 100644 index 00000000000..a758ae5126a --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-43f2e12a3ffb1de29b5b031a1173b87a20a2664b0bef3616631d2c24487c1e54.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH added AS (\n SELECT id, SUM(cost) AS cost\n FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)\n GROUP BY id\n), locked AS (\n SELECT u.user_id FROM \"LiteLLM_UserTable\" u\n WHERE u.user_id IN (SELECT id FROM added)\n ORDER BY u.user_id FOR UPDATE\n)\nUPDATE \"LiteLLM_UserTable\" u\nSET spend = u.spend + added.cost,\n updated_at = (NOW() AT TIME ZONE 'UTC')\nFROM added\nWHERE u.user_id = added.id AND u.user_id IN (SELECT user_id FROM locked)\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "Float8Array" + ] + }, + "nullable": [] + }, + "hash": "43f2e12a3ffb1de29b5b031a1173b87a20a2664b0bef3616631d2c24487c1e54" +} diff --git a/litellm-rust/crates/db/queries/spend/add_user_spend.sql b/litellm-rust/crates/db/queries/spend/add_user_spend.sql new file mode 100644 index 00000000000..27fc12e82f0 --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/add_user_spend.sql @@ -0,0 +1,14 @@ +WITH added AS ( + SELECT id, SUM(cost) AS cost + FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost) + GROUP BY id +), locked AS ( + SELECT u.user_id FROM "LiteLLM_UserTable" u + WHERE u.user_id IN (SELECT id FROM added) + ORDER BY u.user_id FOR UPDATE +) +UPDATE "LiteLLM_UserTable" u +SET spend = u.spend + added.cost, + updated_at = (NOW() AT TIME ZONE 'UTC') +FROM added +WHERE u.user_id = added.id AND u.user_id IN (SELECT user_id FROM locked) diff --git a/litellm-rust/crates/db/src/lib.rs b/litellm-rust/crates/db/src/lib.rs index b3abf5f5173..6589b46b24a 100644 --- a/litellm-rust/crates/db/src/lib.rs +++ b/litellm-rust/crates/db/src/lib.rs @@ -18,4 +18,4 @@ pub use daily_spend::{DailyEndUserRow, DailyTeamRow, DailyUserRow}; pub use end_user::{EndUser, RateLimits, find_end_user}; pub use error::Error; pub use failure::DbFailure; -pub use spend::{EntitySpend, add_key_spend}; +pub use spend::{EntitySpend, add_key_spend, add_user_spend}; diff --git a/litellm-rust/crates/db/src/spend/mod.rs b/litellm-rust/crates/db/src/spend/mod.rs index e34886c6130..98530805fdd 100644 --- a/litellm-rust/crates/db/src/spend/mod.rs +++ b/litellm-rust/crates/db/src/spend/mod.rs @@ -1,3 +1,5 @@ mod key_spend; +mod user_spend; pub use key_spend::{EntitySpend, add_key_spend}; +pub use user_spend::add_user_spend; diff --git a/litellm-rust/crates/db/src/spend/user_spend.rs b/litellm-rust/crates/db/src/spend/user_spend.rs new file mode 100644 index 00000000000..2c89c04c1ef --- /dev/null +++ b/litellm-rust/crates/db/src/spend/user_spend.rs @@ -0,0 +1,173 @@ +use sqlx::PgExecutor; + +use super::EntitySpend; +use crate::Error; + +pub async fn add_user_spend( + executor: impl PgExecutor<'_>, + spend: &EntitySpend, +) -> Result<(), Error> { + spend.validate()?; + if spend.is_empty() { + return Ok(()); + } + sqlx::query_file!("queries/spend/add_user_spend.sql", &spend.ids, &spend.costs) + .execute(executor) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{EntitySpend, add_user_spend}; + use crate::{Error, test_support::database_pool}; + use rstest::rstest; + use sqlx::PgPool; + + #[rstest] + #[case::fractional_costs(0.25, 2.5, 4.0)] + #[case::zero_costs(0.0, 2.0, 3.0)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn user_spend_matches_python_update_many( + #[case] cost: f64, + #[case] expected_a_spend: f64, + #[case] expected_z_spend: f64, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _inserted = sqlx::query( + r#"INSERT INTO "LiteLLM_UserTable" (user_id, spend, updated_at) + VALUES ('user-spend-a', 2, '2000-01-01'), + ('user-spend-z', 3, '2000-01-01'), + ('user-spend-untouched', 100, '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let spend = EntitySpend { + ids: vec![ + "user-spend-z".into(), + "user-spend-a".into(), + "user-spend-missing".into(), + ], + costs: vec![2.0 * cost, cost, 50.0], + }; + add_user_spend(&mut *tx, &EntitySpend::default()).await?; + add_user_spend(&mut *tx, &spend).await?; + add_user_spend(&mut *tx, &spend).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let rows: Vec<(String, f64, bool)> = sqlx::query_as( + r#"SELECT user_id, spend, updated_at > TIMESTAMP '2000-01-01' + FROM "LiteLLM_UserTable" + WHERE user_id = ANY($1) ORDER BY user_id"#, + ) + .bind(vec![ + "user-spend-a", + "user-spend-z", + "user-spend-missing", + "user-spend-untouched", + ]) + .fetch_all(&mut *tx) + .await?; + assert_eq!( + rows, + vec![ + ("user-spend-a".into(), expected_a_spend, true), + ("user-spend-untouched".into(), 100.0, false), + ("user-spend-z".into(), expected_z_spend, true), + ] + ); + Ok(()) + } + + #[rstest] + #[case::direct_duplicate_batch(vec![0.25, 0.75], 3.0)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn user_spend_sums_duplicate_ids( + #[case] costs: Vec, + #[case] expected_spend: f64, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _inserted = sqlx::query( + r#"INSERT INTO "LiteLLM_UserTable" (user_id, spend, updated_at) + VALUES ('user-spend-duplicate', 2, '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let spend = EntitySpend { + ids: vec!["user-spend-duplicate".into(), "user-spend-duplicate".into()], + costs, + }; + add_user_spend(&mut *tx, &spend).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let row: (f64, bool) = sqlx::query_as( + r#"SELECT spend, updated_at > TIMESTAMP '2000-01-01' + FROM "LiteLLM_UserTable" WHERE user_id = 'user-spend-duplicate'"#, + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!(row, (expected_spend, true)); + Ok(()) + } + + #[rstest] + #[case::missing_cost(vec![])] + #[case::extra_cost(vec![0.25, 0.75])] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn user_spend_rejects_malformed_batch_without_writes( + #[case] costs: Vec, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _inserted = sqlx::query( + r#"INSERT INTO "LiteLLM_UserTable" (user_id, spend, updated_at) + VALUES ('user-spend-malformed', 2, '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let spend = EntitySpend { + ids: vec!["user-spend-malformed".into()], + costs, + }; + let result = add_user_spend(&mut *tx, &spend).await; + assert!(matches!(result, Err(Error::InvalidSpendBatch { .. }))); + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let row: (f64, bool) = sqlx::query_as( + r#"SELECT spend, updated_at = TIMESTAMP '2000-01-01' + FROM "LiteLLM_UserTable" WHERE user_id = 'user-spend-malformed'"#, + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!(row, (2.0, true)); + Ok(()) + } +} From 3b72c08c4fc15a4607914652b54c21df71b530a1 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 26 Oct 2026 17:05:05 -0700 Subject: [PATCH 090/155] feat(db): add typed team spend query (#48230) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...edd37a575d641bb68d91e61f5123173c275ce.json | 15 ++ .../db/queries/spend/add_team_spend.sql | 15 ++ litellm-rust/crates/db/src/lib.rs | 2 +- litellm-rust/crates/db/src/spend/mod.rs | 2 + .../crates/db/src/spend/team_spend.rs | 182 ++++++++++++++++++ 5 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 litellm-rust/crates/db/.sqlx/query-c3e5542906e89995fae2a16eca1edd37a575d641bb68d91e61f5123173c275ce.json create mode 100644 litellm-rust/crates/db/queries/spend/add_team_spend.sql create mode 100644 litellm-rust/crates/db/src/spend/team_spend.rs diff --git a/litellm-rust/crates/db/.sqlx/query-c3e5542906e89995fae2a16eca1edd37a575d641bb68d91e61f5123173c275ce.json b/litellm-rust/crates/db/.sqlx/query-c3e5542906e89995fae2a16eca1edd37a575d641bb68d91e61f5123173c275ce.json new file mode 100644 index 00000000000..250971b053a --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-c3e5542906e89995fae2a16eca1edd37a575d641bb68d91e61f5123173c275ce.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH added AS (\n SELECT id, SUM(cost) AS cost\n FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)\n GROUP BY id\n),\nlocked AS (\n SELECT t.team_id FROM \"LiteLLM_TeamTable\" t\n WHERE t.team_id IN (SELECT id FROM added)\n ORDER BY t.team_id FOR UPDATE\n)\nUPDATE \"LiteLLM_TeamTable\" t\nSET spend = t.spend + added.cost,\n updated_at = (NOW() AT TIME ZONE 'UTC')\nFROM added\nWHERE t.team_id = added.id AND t.team_id IN (SELECT team_id FROM locked)\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "Float8Array" + ] + }, + "nullable": [] + }, + "hash": "c3e5542906e89995fae2a16eca1edd37a575d641bb68d91e61f5123173c275ce" +} diff --git a/litellm-rust/crates/db/queries/spend/add_team_spend.sql b/litellm-rust/crates/db/queries/spend/add_team_spend.sql new file mode 100644 index 00000000000..58b8e3d6ab9 --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/add_team_spend.sql @@ -0,0 +1,15 @@ +WITH added AS ( + SELECT id, SUM(cost) AS cost + FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost) + GROUP BY id +), +locked AS ( + SELECT t.team_id FROM "LiteLLM_TeamTable" t + WHERE t.team_id IN (SELECT id FROM added) + ORDER BY t.team_id FOR UPDATE +) +UPDATE "LiteLLM_TeamTable" t +SET spend = t.spend + added.cost, + updated_at = (NOW() AT TIME ZONE 'UTC') +FROM added +WHERE t.team_id = added.id AND t.team_id IN (SELECT team_id FROM locked) diff --git a/litellm-rust/crates/db/src/lib.rs b/litellm-rust/crates/db/src/lib.rs index 6589b46b24a..1dda3a987b0 100644 --- a/litellm-rust/crates/db/src/lib.rs +++ b/litellm-rust/crates/db/src/lib.rs @@ -18,4 +18,4 @@ pub use daily_spend::{DailyEndUserRow, DailyTeamRow, DailyUserRow}; pub use end_user::{EndUser, RateLimits, find_end_user}; pub use error::Error; pub use failure::DbFailure; -pub use spend::{EntitySpend, add_key_spend, add_user_spend}; +pub use spend::{EntitySpend, add_key_spend, add_team_spend, add_user_spend}; diff --git a/litellm-rust/crates/db/src/spend/mod.rs b/litellm-rust/crates/db/src/spend/mod.rs index 98530805fdd..775be335150 100644 --- a/litellm-rust/crates/db/src/spend/mod.rs +++ b/litellm-rust/crates/db/src/spend/mod.rs @@ -1,5 +1,7 @@ mod key_spend; +mod team_spend; mod user_spend; pub use key_spend::{EntitySpend, add_key_spend}; +pub use team_spend::add_team_spend; pub use user_spend::add_user_spend; diff --git a/litellm-rust/crates/db/src/spend/team_spend.rs b/litellm-rust/crates/db/src/spend/team_spend.rs new file mode 100644 index 00000000000..a105899bc2a --- /dev/null +++ b/litellm-rust/crates/db/src/spend/team_spend.rs @@ -0,0 +1,182 @@ +use sqlx::PgExecutor; + +use super::EntitySpend; +use crate::Error; + +pub async fn add_team_spend( + executor: impl PgExecutor<'_>, + spend: &EntitySpend, +) -> Result<(), Error> { + spend.validate()?; + if spend.is_empty() { + return Ok(()); + } + sqlx::query_file!("queries/spend/add_team_spend.sql", &spend.ids, &spend.costs) + .execute(executor) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{EntitySpend, add_team_spend}; + use crate::Error; + use crate::test_support::database_pool; + use rstest::rstest; + use sqlx::PgPool; + + #[rstest] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn duplicate_team_ids_add_every_cost( + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _inserted = sqlx::query( + r#"INSERT INTO "LiteLLM_TeamTable" (team_id, spend, created_at, updated_at) + VALUES ('team-spend-duplicate-a', 2, '2000-01-01', '2000-01-01'), + ('team-spend-duplicate-z', 3, '2000-01-01', '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let spend = EntitySpend { + ids: vec![ + "team-spend-duplicate-z".into(), + "team-spend-duplicate-a".into(), + "team-spend-duplicate-z".into(), + "team-spend-duplicate-a".into(), + ], + costs: vec![0.5, 0.25, 1.0, 0.5], + }; + add_team_spend(&mut *tx, &spend).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let rows: Vec<(String, f64)> = sqlx::query_as( + r#"SELECT team_id, spend FROM "LiteLLM_TeamTable" + WHERE team_id = ANY($1) ORDER BY team_id"#, + ) + .bind(&spend.ids) + .fetch_all(&mut *tx) + .await?; + assert_eq!( + rows, + vec![ + ("team-spend-duplicate-a".into(), 2.75), + ("team-spend-duplicate-z".into(), 4.5), + ] + ); + Ok(()) + } + + #[rstest] + #[case::missing_cost(vec!["team-spend-malformed".into()], vec![])] + #[case::extra_cost(vec!["team-spend-malformed".into()], vec![0.25, 0.5])] + #[case::empty_ids(vec![], vec![0.25])] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn malformed_team_batch_does_not_write( + #[case] ids: Vec, + #[case] costs: Vec, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _inserted = sqlx::query( + r#"INSERT INTO "LiteLLM_TeamTable" (team_id, spend, created_at, updated_at) + VALUES ('team-spend-malformed', 2, '2000-01-01', '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let result = add_team_spend(&mut *tx, &EntitySpend { ids, costs }).await; + assert!(matches!(result, Err(Error::InvalidSpendBatch { .. }))); + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let row: (f64, bool) = sqlx::query_as( + r#"SELECT spend, updated_at = TIMESTAMP '2000-01-01' + FROM "LiteLLM_TeamTable" WHERE team_id = 'team-spend-malformed'"#, + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!(row, (2.0, true)); + Ok(()) + } + + #[rstest] + #[case::fractional_costs(0.25, 2.5, 4.0)] + #[case::zero_costs(0.0, 2.0, 3.0)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn team_spend_matches_python_update_many( + #[case] cost: f64, + #[case] expected_a_spend: f64, + #[case] expected_z_spend: f64, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _inserted = sqlx::query( + r#"INSERT INTO "LiteLLM_TeamTable" (team_id, spend, created_at, updated_at) + VALUES ('team-spend-a', 2, '2000-01-01', '2000-01-01'), + ('team-spend-z', 3, '2000-01-01', '2000-01-01'), + ('team-spend-untouched', 100, '2000-01-01', '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let spend = EntitySpend { + ids: vec![ + "team-spend-z".into(), + "team-spend-a".into(), + "team-spend-missing".into(), + ], + costs: vec![2.0 * cost, cost, 50.0], + }; + add_team_spend(&mut *tx, &EntitySpend::default()).await?; + add_team_spend(&mut *tx, &spend).await?; + add_team_spend(&mut *tx, &spend).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let rows: Vec<(String, f64, bool)> = sqlx::query_as( + r#"SELECT team_id, spend, updated_at > TIMESTAMP '2000-01-01' + FROM "LiteLLM_TeamTable" + WHERE team_id = ANY($1) ORDER BY team_id"#, + ) + .bind(vec![ + "team-spend-a", + "team-spend-z", + "team-spend-missing", + "team-spend-untouched", + ]) + .fetch_all(&mut *tx) + .await?; + assert_eq!( + rows, + vec![ + ("team-spend-a".into(), expected_a_spend, true), + ("team-spend-untouched".into(), 100.0, false), + ("team-spend-z".into(), expected_z_spend, true), + ] + ); + Ok(()) + } +} From 8f456eb7d51f52d62621e5672ee47ae9079e078d Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 26 Oct 2026 17:05:05 -0700 Subject: [PATCH 091/155] feat(db): add typed end-user spend query (#48231) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...2288e0f2c1f12a2a79ed53f0002826724cd0a.json | 15 ++ .../db/queries/spend/add_end_user_spend.sql | 6 + litellm-rust/crates/db/src/lib.rs | 2 +- .../crates/db/src/spend/end_user_spend.rs | 197 ++++++++++++++++++ litellm-rust/crates/db/src/spend/mod.rs | 2 + 5 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 litellm-rust/crates/db/.sqlx/query-345c02595f55a7f0274ef7214b02288e0f2c1f12a2a79ed53f0002826724cd0a.json create mode 100644 litellm-rust/crates/db/queries/spend/add_end_user_spend.sql create mode 100644 litellm-rust/crates/db/src/spend/end_user_spend.rs diff --git a/litellm-rust/crates/db/.sqlx/query-345c02595f55a7f0274ef7214b02288e0f2c1f12a2a79ed53f0002826724cd0a.json b/litellm-rust/crates/db/.sqlx/query-345c02595f55a7f0274ef7214b02288e0f2c1f12a2a79ed53f0002826724cd0a.json new file mode 100644 index 00000000000..123ff15ade2 --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-345c02595f55a7f0274ef7214b02288e0f2c1f12a2a79ed53f0002826724cd0a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO \"LiteLLM_EndUserTable\" (user_id, spend, blocked)\nSELECT id, SUM(cost), false FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)\nGROUP BY id\nORDER BY id\nON CONFLICT (user_id) DO UPDATE\nSET spend = \"LiteLLM_EndUserTable\".spend + EXCLUDED.spend\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "Float8Array" + ] + }, + "nullable": [] + }, + "hash": "345c02595f55a7f0274ef7214b02288e0f2c1f12a2a79ed53f0002826724cd0a" +} diff --git a/litellm-rust/crates/db/queries/spend/add_end_user_spend.sql b/litellm-rust/crates/db/queries/spend/add_end_user_spend.sql new file mode 100644 index 00000000000..06a5b96d254 --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/add_end_user_spend.sql @@ -0,0 +1,6 @@ +INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked) +SELECT id, SUM(cost), false FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost) +GROUP BY id +ORDER BY id +ON CONFLICT (user_id) DO UPDATE +SET spend = "LiteLLM_EndUserTable".spend + EXCLUDED.spend diff --git a/litellm-rust/crates/db/src/lib.rs b/litellm-rust/crates/db/src/lib.rs index 1dda3a987b0..3a9032d73dc 100644 --- a/litellm-rust/crates/db/src/lib.rs +++ b/litellm-rust/crates/db/src/lib.rs @@ -18,4 +18,4 @@ pub use daily_spend::{DailyEndUserRow, DailyTeamRow, DailyUserRow}; pub use end_user::{EndUser, RateLimits, find_end_user}; pub use error::Error; pub use failure::DbFailure; -pub use spend::{EntitySpend, add_key_spend, add_team_spend, add_user_spend}; +pub use spend::{EntitySpend, add_end_user_spend, add_key_spend, add_team_spend, add_user_spend}; diff --git a/litellm-rust/crates/db/src/spend/end_user_spend.rs b/litellm-rust/crates/db/src/spend/end_user_spend.rs new file mode 100644 index 00000000000..ca327ac7e18 --- /dev/null +++ b/litellm-rust/crates/db/src/spend/end_user_spend.rs @@ -0,0 +1,197 @@ +use sqlx::PgExecutor; + +use super::EntitySpend; +use crate::Error; + +pub async fn add_end_user_spend( + executor: impl PgExecutor<'_>, + spend: &EntitySpend, +) -> Result<(), Error> { + spend.validate()?; + if spend.is_empty() { + return Ok(()); + } + sqlx::query_file!( + "queries/spend/add_end_user_spend.sql", + &spend.ids, + &spend.costs + ) + .execute(executor) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{EntitySpend, add_end_user_spend}; + use crate::test_support::database_pool; + use rstest::rstest; + use sqlx::PgPool; + + #[rstest] + #[case::fractional_costs(0.25, 0.75, false, 2.5, 1.5)] + #[case::zero_costs(0.0, 0.0, false, 2.0, 0.0)] + #[case::duplicate_ids(0.25, 0.75, true, 3.0, 3.0)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn end_user_spend_matches_python_upsert( + #[case] existing_cost: f64, + #[case] new_cost: f64, + #[case] duplicate_ids: bool, + #[case] expected_existing_spend: f64, + #[case] expected_new_spend: f64, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _budget = sqlx::query( + r#"INSERT INTO "LiteLLM_BudgetTable" (budget_id, created_by, updated_by) + VALUES ('end-user-spend-budget', 'db-query-test', 'db-query-test')"#, + ) + .execute(&mut *tx) + .await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _users = sqlx::query( + r#"INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked, budget_id) + VALUES ('end-user-spend-z', 2, true, 'end-user-spend-budget'), + ('end-user-spend-untouched', 100, true, NULL)"#, + ) + .execute(&mut *tx) + .await?; + let spend = if duplicate_ids { + EntitySpend { + ids: vec![ + "end-user-spend-z".into(), + "end-user-spend-a".into(), + "end-user-spend-z".into(), + "end-user-spend-a".into(), + ], + costs: vec![existing_cost, new_cost, existing_cost, new_cost], + } + } else { + EntitySpend { + ids: vec!["end-user-spend-z".into(), "end-user-spend-a".into()], + costs: vec![existing_cost, new_cost], + } + }; + add_end_user_spend(&mut *tx, &spend).await?; + add_end_user_spend(&mut *tx, &spend).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let rows: Vec<(String, f64, bool, Option)> = sqlx::query_as( + r#"SELECT user_id, spend, blocked, budget_id FROM "LiteLLM_EndUserTable" + WHERE user_id = ANY($1) ORDER BY user_id"#, + ) + .bind(vec![ + "end-user-spend-a", + "end-user-spend-z", + "end-user-spend-untouched", + ]) + .fetch_all(&mut *tx) + .await?; + assert_eq!( + rows, + vec![ + ("end-user-spend-a".into(), expected_new_spend, false, None), + ("end-user-spend-untouched".into(), 100.0, true, None), + ( + "end-user-spend-z".into(), + expected_existing_spend, + true, + Some("end-user-spend-budget".into()), + ), + ] + ); + Ok(()) + } + + #[rstest] + #[case::missing_cost(vec!["end-user-invalid-existing".into(), "end-user-invalid-new".into()], vec![1.0])] + #[case::extra_cost(vec!["end-user-invalid-existing".into()], vec![1.0, 2.0])] + #[case::empty_ids(vec![], vec![1.0])] + #[case::empty_costs(vec!["end-user-invalid-existing".into()], vec![])] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn malformed_end_user_spend_is_rejected_before_writes( + #[case] ids: Vec, + #[case] costs: Vec, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _user = sqlx::query( + r#"INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked) + VALUES ('end-user-invalid-existing', 8.5, true)"#, + ) + .execute(&mut *tx) + .await?; + let spend = EntitySpend { ids, costs }; + let result = add_end_user_spend(&mut *tx, &spend).await; + assert!(matches!( + result, + Err(crate::Error::InvalidSpendBatch { ids, costs }) + if ids == spend.ids.len() && costs == spend.costs.len() + )); + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let rows: Vec<(String, f64, bool)> = sqlx::query_as( + r#"SELECT user_id, spend, blocked FROM "LiteLLM_EndUserTable" + WHERE user_id IN ('end-user-invalid-existing', 'end-user-invalid-new')"#, + ) + .fetch_all(&mut *tx) + .await?; + assert_eq!(rows, vec![("end-user-invalid-existing".into(), 8.5, true)]); + Ok(()) + } + + #[rstest] + #[case::empty_batch()] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn empty_end_user_spend_is_a_noop( + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _user = sqlx::query( + r#"INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked) + VALUES ('end-user-empty-spend', 8.5, true)"#, + ) + .execute(&mut *tx) + .await?; + add_end_user_spend(&mut *tx, &EntitySpend::default()).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let row: (f64, bool, Option) = sqlx::query_as( + r#"SELECT spend, blocked, budget_id FROM "LiteLLM_EndUserTable" + WHERE user_id = 'end-user-empty-spend'"#, + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!(row, (8.5, true, None)); + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/spend/mod.rs b/litellm-rust/crates/db/src/spend/mod.rs index 775be335150..5191da2adb5 100644 --- a/litellm-rust/crates/db/src/spend/mod.rs +++ b/litellm-rust/crates/db/src/spend/mod.rs @@ -1,7 +1,9 @@ +mod end_user_spend; mod key_spend; mod team_spend; mod user_spend; +pub use end_user_spend::add_end_user_spend; pub use key_spend::{EntitySpend, add_key_spend}; pub use team_spend::add_team_spend; pub use user_spend::add_user_spend; From c531692976b348dc4c44f9c6d235e0cf981632fe Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 26 Oct 2026 17:05:06 -0700 Subject: [PATCH 092/155] feat(db): add transaction-scoped team lock query (#48232) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...7bb2d97532a9cd461cc5d8da1e1090560ca1f.json | 23 +++++ ...92bf33e16fea1dbb0c32cf89a376a874927f4.json | 21 ++++ .../crates/db/queries/spend/lock_team.sql | 1 + litellm-rust/crates/db/src/lib.rs | 5 +- litellm-rust/crates/db/src/spend/mod.rs | 2 + litellm-rust/crates/db/src/spend/team_lock.rs | 98 +++++++++++++++++++ 6 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 litellm-rust/crates/db/.sqlx/query-085edfb9af2a914466c75b447bc7bb2d97532a9cd461cc5d8da1e1090560ca1f.json create mode 100644 litellm-rust/crates/db/.sqlx/query-5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4.json create mode 100644 litellm-rust/crates/db/queries/spend/lock_team.sql create mode 100644 litellm-rust/crates/db/src/spend/team_lock.rs diff --git a/litellm-rust/crates/db/.sqlx/query-085edfb9af2a914466c75b447bc7bb2d97532a9cd461cc5d8da1e1090560ca1f.json b/litellm-rust/crates/db/.sqlx/query-085edfb9af2a914466c75b447bc7bb2d97532a9cd461cc5d8da1e1090560ca1f.json new file mode 100644 index 00000000000..a415666d0be --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-085edfb9af2a914466c75b447bc7bb2d97532a9cd461cc5d8da1e1090560ca1f.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtext($1)) IS NULL AS \"locked!\"\n", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "locked!", + "type_info": "Bool", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "085edfb9af2a914466c75b447bc7bb2d97532a9cd461cc5d8da1e1090560ca1f" +} diff --git a/litellm-rust/crates/db/.sqlx/query-5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4.json b/litellm-rust/crates/db/.sqlx/query-5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4.json new file mode 100644 index 00000000000..7eef5278cb5 --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT set_config('lock_timeout', '5000ms', true)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "set_config", + "type_info": "Text", + "origin": "Expression" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4" +} diff --git a/litellm-rust/crates/db/queries/spend/lock_team.sql b/litellm-rust/crates/db/queries/spend/lock_team.sql new file mode 100644 index 00000000000..06cdb806ef0 --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/lock_team.sql @@ -0,0 +1 @@ +SELECT pg_advisory_xact_lock(hashtext($1)) IS NULL AS "locked!" diff --git a/litellm-rust/crates/db/src/lib.rs b/litellm-rust/crates/db/src/lib.rs index 3a9032d73dc..4f009b411fd 100644 --- a/litellm-rust/crates/db/src/lib.rs +++ b/litellm-rust/crates/db/src/lib.rs @@ -18,4 +18,7 @@ pub use daily_spend::{DailyEndUserRow, DailyTeamRow, DailyUserRow}; pub use end_user::{EndUser, RateLimits, find_end_user}; pub use error::Error; pub use failure::DbFailure; -pub use spend::{EntitySpend, add_end_user_spend, add_key_spend, add_team_spend, add_user_spend}; +pub use spend::{ + EntitySpend, add_end_user_spend, add_key_spend, add_team_spend, add_user_spend, begin, + lock_team, +}; diff --git a/litellm-rust/crates/db/src/spend/mod.rs b/litellm-rust/crates/db/src/spend/mod.rs index 5191da2adb5..0459e63aaf8 100644 --- a/litellm-rust/crates/db/src/spend/mod.rs +++ b/litellm-rust/crates/db/src/spend/mod.rs @@ -1,9 +1,11 @@ mod end_user_spend; mod key_spend; +mod team_lock; mod team_spend; mod user_spend; pub use end_user_spend::add_end_user_spend; pub use key_spend::{EntitySpend, add_key_spend}; +pub use team_lock::{begin, lock_team}; pub use team_spend::add_team_spend; pub use user_spend::add_user_spend; diff --git a/litellm-rust/crates/db/src/spend/team_lock.rs b/litellm-rust/crates/db/src/spend/team_lock.rs new file mode 100644 index 00000000000..7a6b9c7f7d0 --- /dev/null +++ b/litellm-rust/crates/db/src/spend/team_lock.rs @@ -0,0 +1,98 @@ +use sqlx::{PgPool, Postgres, Transaction}; + +use crate::Error; + +pub async fn begin(pool: &PgPool) -> Result, Error> { + let mut transaction = pool.begin().await?; + sqlx::query!("SELECT set_config('lock_timeout', '5000ms', true)") + .fetch_one(&mut *transaction) + .await?; + Ok(transaction) +} + +pub async fn lock_team( + transaction: &mut Transaction<'_, Postgres>, + team_id: &str, +) -> Result<(), Error> { + sqlx::query_file!("queries/spend/lock_team.sql", team_id) + .fetch_one(&mut **transaction) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{begin, lock_team}; + use crate::test_support::database_pool; + use rstest::rstest; + use sqlx::{PgPool, postgres::PgPoolOptions}; + + #[rstest] + #[case::transaction_scoped("team-lock-scope")] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn team_lock_is_held_until_rollback( + #[case] team_id: &str, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let database_pool = database_pool?; + let pool = PgPoolOptions::new() + .max_connections(2) + .connect_with(database_pool.connect_options().as_ref().clone()) + .await?; + let mut first = begin(&pool).await?; + let mut second = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let original_timeout: String = sqlx::query_scalar("SELECT current_setting('lock_timeout')") + .fetch_one(&mut *second) + .await?; + lock_team(&mut first, team_id).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let (backend, timeout): (i32, String) = + sqlx::query_as("SELECT pg_backend_pid(), current_setting('lock_timeout')") + .fetch_one(&mut *first) + .await?; + assert_eq!(timeout, "5s"); + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let held: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(hashtext($1))") + .bind(team_id) + .fetch_one(&mut *second) + .await?; + assert!(!held); + first.rollback().await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let released: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(hashtext($1))") + .bind(team_id) + .fetch_one(&mut *second) + .await?; + assert!(released); + let mut next = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let (next_backend, next_timeout): (i32, String) = + sqlx::query_as("SELECT pg_backend_pid(), current_setting('lock_timeout')") + .fetch_one(&mut *next) + .await?; + assert_eq!(next_backend, backend); + assert_eq!(next_timeout, original_timeout); + next.rollback().await?; + second.rollback().await?; + Ok(()) + } +} From 46410c8da9589c716fcd2619ea4a0358dac1adc8 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 26 Oct 2026 17:05:06 -0700 Subject: [PATCH 093/155] feat(db): add typed team-member spend upsert (#48233) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/Cargo.lock | 1 + ...27df60ff71d8fed05b53be4458b335af88147.json | 14 ++ litellm-rust/crates/db/Cargo.toml | 1 + .../spend/upsert_team_member_spend.sql | 15 ++ litellm-rust/crates/db/src/lib.rs | 4 +- litellm-rust/crates/db/src/spend/mod.rs | 2 + .../crates/db/src/spend/team_member_spend.rs | 162 ++++++++++++++++++ 7 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 litellm-rust/crates/db/.sqlx/query-b0eb48b795bdaf728c79e986d2027df60ff71d8fed05b53be4458b335af88147.json create mode 100644 litellm-rust/crates/db/queries/spend/upsert_team_member_spend.sql create mode 100644 litellm-rust/crates/db/src/spend/team_member_spend.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index d58ba9ecb5f..d0757ebbd2c 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3699,6 +3699,7 @@ dependencies = [ "macro_rules_attribute", "rstest", "serde", + "serde_json", "sqlx", "thiserror 2.0.19", "tokio", diff --git a/litellm-rust/crates/db/.sqlx/query-b0eb48b795bdaf728c79e986d2027df60ff71d8fed05b53be4458b335af88147.json b/litellm-rust/crates/db/.sqlx/query-b0eb48b795bdaf728c79e986d2027df60ff71d8fed05b53be4458b335af88147.json new file mode 100644 index 00000000000..b333ec814a1 --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-b0eb48b795bdaf728c79e986d2027df60ff71d8fed05b53be4458b335af88147.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO \"LiteLLM_TeamMembership\" (user_id, team_id, spend, total_spend)\nSELECT member.user_id, member.team_id, member.cost, member.cost\nFROM jsonb_to_recordset($1::jsonb) AS member(user_id text, team_id text, cost float8)\nWHERE EXISTS (\n SELECT 1 FROM \"LiteLLM_TeamTable\" t\n WHERE t.team_id = member.team_id\n AND t.members_with_roles @> jsonb_build_array(jsonb_build_object('user_id', member.user_id))\n)\n OR EXISTS (\n SELECT 1 FROM \"LiteLLM_TeamMembership\" m\n WHERE m.user_id = member.user_id AND m.team_id = member.team_id\n)\nON CONFLICT (user_id, team_id) DO UPDATE\nSET spend = \"LiteLLM_TeamMembership\".spend + EXCLUDED.spend,\n total_spend = \"LiteLLM_TeamMembership\".total_spend + EXCLUDED.total_spend\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "b0eb48b795bdaf728c79e986d2027df60ff71d8fed05b53be4458b335af88147" +} diff --git a/litellm-rust/crates/db/Cargo.toml b/litellm-rust/crates/db/Cargo.toml index bb6d5655b60..8c1a35c7535 100644 --- a/litellm-rust/crates/db/Cargo.toml +++ b/litellm-rust/crates/db/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true chrono.workspace = true macro_rules_attribute.workspace = true serde.workspace = true +serde_json.workspace = true sqlx.workspace = true thiserror.workspace = true diff --git a/litellm-rust/crates/db/queries/spend/upsert_team_member_spend.sql b/litellm-rust/crates/db/queries/spend/upsert_team_member_spend.sql new file mode 100644 index 00000000000..460c631cbbd --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/upsert_team_member_spend.sql @@ -0,0 +1,15 @@ +INSERT INTO "LiteLLM_TeamMembership" (user_id, team_id, spend, total_spend) +SELECT member.user_id, member.team_id, member.cost, member.cost +FROM jsonb_to_recordset($1::jsonb) AS member(user_id text, team_id text, cost float8) +WHERE EXISTS ( + SELECT 1 FROM "LiteLLM_TeamTable" t + WHERE t.team_id = member.team_id + AND t.members_with_roles @> jsonb_build_array(jsonb_build_object('user_id', member.user_id)) +) + OR EXISTS ( + SELECT 1 FROM "LiteLLM_TeamMembership" m + WHERE m.user_id = member.user_id AND m.team_id = member.team_id +) +ON CONFLICT (user_id, team_id) DO UPDATE +SET spend = "LiteLLM_TeamMembership".spend + EXCLUDED.spend, + total_spend = "LiteLLM_TeamMembership".total_spend + EXCLUDED.total_spend diff --git a/litellm-rust/crates/db/src/lib.rs b/litellm-rust/crates/db/src/lib.rs index 4f009b411fd..afa962cff0e 100644 --- a/litellm-rust/crates/db/src/lib.rs +++ b/litellm-rust/crates/db/src/lib.rs @@ -19,6 +19,6 @@ pub use end_user::{EndUser, RateLimits, find_end_user}; pub use error::Error; pub use failure::DbFailure; pub use spend::{ - EntitySpend, add_end_user_spend, add_key_spend, add_team_spend, add_user_spend, begin, - lock_team, + EntitySpend, TeamMemberSpend, add_end_user_spend, add_key_spend, add_team_member_spend, + add_team_spend, add_user_spend, begin, lock_team, }; diff --git a/litellm-rust/crates/db/src/spend/mod.rs b/litellm-rust/crates/db/src/spend/mod.rs index 0459e63aaf8..1acf629af4e 100644 --- a/litellm-rust/crates/db/src/spend/mod.rs +++ b/litellm-rust/crates/db/src/spend/mod.rs @@ -1,11 +1,13 @@ mod end_user_spend; mod key_spend; mod team_lock; +mod team_member_spend; mod team_spend; mod user_spend; pub use end_user_spend::add_end_user_spend; pub use key_spend::{EntitySpend, add_key_spend}; pub use team_lock::{begin, lock_team}; +pub use team_member_spend::{TeamMemberSpend, add_team_member_spend}; pub use team_spend::add_team_spend; pub use user_spend::add_user_spend; diff --git a/litellm-rust/crates/db/src/spend/team_member_spend.rs b/litellm-rust/crates/db/src/spend/team_member_spend.rs new file mode 100644 index 00000000000..965a39ada4f --- /dev/null +++ b/litellm-rust/crates/db/src/spend/team_member_spend.rs @@ -0,0 +1,162 @@ +use sqlx::{Postgres, Transaction}; + +use super::lock_team; +use crate::Error; + +#[macro_rules_attribute::apply(crate::db_input)] +pub struct TeamMemberSpend { + pub team_id: String, + pub user_id: String, + pub cost: f64, +} + +pub async fn add_team_member_spend( + transaction: &mut Transaction<'_, Postgres>, + members: &[TeamMemberSpend], +) -> Result<(), Error> { + if members.is_empty() { + return Ok(()); + } + let mut members = members.to_vec(); + members.sort_by(|left, right| { + (&left.team_id, &left.user_id).cmp(&(&right.team_id, &right.user_id)) + }); + let mut teams: Vec<&str> = members + .iter() + .map(|member| member.team_id.as_str()) + .collect(); + teams.dedup(); + for team in teams { + lock_team(transaction, team).await?; + } + let rows = serde_json::Value::Array( + members + .iter() + .map(|member| { + serde_json::json!({ + "user_id": member.user_id, + "team_id": member.team_id, + "cost": member.cost, + }) + }) + .collect(), + ); + sqlx::query_file!("queries/spend/upsert_team_member_spend.sql", rows) + .execute(&mut **transaction) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::super::begin; + use super::{TeamMemberSpend, add_team_member_spend}; + use crate::test_support::database_pool; + use rstest::rstest; + use sqlx::PgPool; + + fn members(cost: f64) -> [TeamMemberSpend; 3] { + [ + TeamMemberSpend { + team_id: "member-spend-z".into(), + user_id: "off-roster-existing".into(), + cost: 2.0 * cost, + }, + TeamMemberSpend { + team_id: "member-spend-a".into(), + user_id: "on-roster-missing".into(), + cost, + }, + TeamMemberSpend { + team_id: "member-spend-a".into(), + user_id: "off-roster-missing".into(), + cost: 50.0 * cost, + }, + ] + } + + #[rstest] + #[case::zero_then_quarter(0.25, 0.5, 3.0, 8.0)] + #[case::zero_then_eighth(0.125, 0.25, 2.5, 7.5)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn membership_spend_preserves_roster_guard_and_fractional_costs( + #[case] cost: f64, + #[case] expected_created: f64, + #[case] expected_spend: f64, + #[case] expected_total: f64, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = begin(&pool).await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _teams = sqlx::query( + r#"INSERT INTO "LiteLLM_TeamTable" (team_id, members_with_roles) + VALUES ('member-spend-a', '[{"user_id":"on-roster-missing","role":"user"}]'), + ('member-spend-z', '[]')"#, + ) + .execute(&mut *tx) + .await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _members = sqlx::query( + r#"INSERT INTO "LiteLLM_TeamMembership" (team_id, user_id, spend, total_spend) + VALUES ('member-spend-z', 'off-roster-existing', 2, 7), + ('member-spend-a', 'untouched', 100, 200)"#, + ) + .execute(&mut *tx) + .await?; + add_team_member_spend(&mut tx, &[]).await?; + add_team_member_spend(&mut tx, &members(0.0)).await?; + add_team_member_spend(&mut tx, &members(cost)).await?; + add_team_member_spend(&mut tx, &members(cost)).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let rows: Vec<(String, String, f64, f64)> = sqlx::query_as( + r#"SELECT team_id, user_id, spend, total_spend FROM "LiteLLM_TeamMembership" + WHERE team_id IN ('member-spend-a', 'member-spend-z') ORDER BY team_id, user_id"#, + ) + .fetch_all(&mut *tx) + .await?; + assert_eq!( + rows, + vec![ + ( + "member-spend-a".into(), + "on-roster-missing".into(), + expected_created, + expected_created + ), + ("member-spend-a".into(), "untouched".into(), 100.0, 200.0), + ( + "member-spend-z".into(), + "off-roster-existing".into(), + expected_spend, + expected_total + ), + ] + ); + tx.rollback().await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let remaining: i64 = sqlx::query_scalar( + r#"SELECT COUNT(*) FROM "LiteLLM_TeamMembership" + WHERE team_id IN ('member-spend-a', 'member-spend-z')"#, + ) + .fetch_one(&pool) + .await?; + assert_eq!(remaining, 0); + Ok(()) + } +} From b446d589090e73f118462f4d1d4d57823cd03bcf Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 26 Oct 2026 17:05:06 -0700 Subject: [PATCH 094/155] feat(db): add typed spend-log insert query (#48234) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...acf6d06eb22facdf478f88107c9dccdac326b.json | 40 ++ .../db/queries/spend/insert_spend_logs.sql | 26 + litellm-rust/crates/db/src/lib.rs | 4 +- litellm-rust/crates/db/src/spend/mod.rs | 2 + .../crates/db/src/spend/spend_logs.rs | 517 ++++++++++++++++++ 5 files changed, 587 insertions(+), 2 deletions(-) create mode 100644 litellm-rust/crates/db/.sqlx/query-32f3c07ced96324b535bbc703c1acf6d06eb22facdf478f88107c9dccdac326b.json create mode 100644 litellm-rust/crates/db/queries/spend/insert_spend_logs.sql create mode 100644 litellm-rust/crates/db/src/spend/spend_logs.rs diff --git a/litellm-rust/crates/db/.sqlx/query-32f3c07ced96324b535bbc703c1acf6d06eb22facdf478f88107c9dccdac326b.json b/litellm-rust/crates/db/.sqlx/query-32f3c07ced96324b535bbc703c1acf6d06eb22facdf478f88107c9dccdac326b.json new file mode 100644 index 00000000000..c4eb1737d85 --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-32f3c07ced96324b535bbc703c1acf6d06eb22facdf478f88107c9dccdac326b.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO \"LiteLLM_SpendLogs\" (\n request_id, call_type, api_key, spend, total_tokens, prompt_tokens,\n completion_tokens, \"startTime\", \"endTime\", \"completionStartTime\",\n request_duration_ms, model, model_id, model_group, custom_llm_provider, api_base,\n \"user\", metadata, cache_hit, cache_key, request_tags, team_id, organization_id,\n end_user, requester_ip_address, messages, response, proxy_server_request,\n session_id, status, litellm_call_id\n)\nSELECT l.request_id, l.call_type, l.api_key, l.spend, l.total_tokens, l.prompt_tokens,\n l.completion_tokens, l.start_time::timestamp(3), l.end_time::timestamp(3),\n l.completion_start_time::timestamp(3), l.request_duration_ms, l.model,\n l.model_id, l.model_group, l.custom_llm_provider, l.api_base, l.user_id,\n l.metadata, l.cache_hit, l.cache_key, '[]'::jsonb, l.team_id,\n l.organization_id, l.end_user, l.requester_ip_address, '{}'::jsonb,\n '{}'::jsonb, '{}'::jsonb, l.session_id, l.status, l.litellm_call_id\nFROM UNNEST($1::text[], $2::text[], $3::text[], $4::float8[], $5::int4[], $6::int4[],\n $7::int4[], $8::timestamp[], $9::timestamp[], $10::timestamp[], $11::int4[], $12::text[],\n $13::text[], $14::text[], $15::text[], $16::text[], $17::text[],\n $18::jsonb[], $19::text[], $20::text[], $21::text[], $22::text[],\n $23::text[], $24::text[], $25::text[], $26::text[], $27::text[])\n AS l(request_id, call_type, api_key, spend, total_tokens, prompt_tokens,\n completion_tokens, start_time, end_time, completion_start_time,\n request_duration_ms, model, model_id, model_group, custom_llm_provider,\n api_base, user_id, metadata, cache_hit, cache_key, team_id, organization_id,\n end_user, requester_ip_address, session_id, status, litellm_call_id)\nON CONFLICT DO NOTHING\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray", + "TextArray", + "Float8Array", + "Int4Array", + "Int4Array", + "Int4Array", + "TimestampArray", + "TimestampArray", + "TimestampArray", + "Int4Array", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "JsonbArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "32f3c07ced96324b535bbc703c1acf6d06eb22facdf478f88107c9dccdac326b" +} diff --git a/litellm-rust/crates/db/queries/spend/insert_spend_logs.sql b/litellm-rust/crates/db/queries/spend/insert_spend_logs.sql new file mode 100644 index 00000000000..fcffee181a1 --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/insert_spend_logs.sql @@ -0,0 +1,26 @@ +INSERT INTO "LiteLLM_SpendLogs" ( + request_id, call_type, api_key, spend, total_tokens, prompt_tokens, + completion_tokens, "startTime", "endTime", "completionStartTime", + request_duration_ms, model, model_id, model_group, custom_llm_provider, api_base, + "user", metadata, cache_hit, cache_key, request_tags, team_id, organization_id, + end_user, requester_ip_address, messages, response, proxy_server_request, + session_id, status, litellm_call_id +) +SELECT l.request_id, l.call_type, l.api_key, l.spend, l.total_tokens, l.prompt_tokens, + l.completion_tokens, l.start_time::timestamp(3), l.end_time::timestamp(3), + l.completion_start_time::timestamp(3), l.request_duration_ms, l.model, + l.model_id, l.model_group, l.custom_llm_provider, l.api_base, l.user_id, + l.metadata, l.cache_hit, l.cache_key, '[]'::jsonb, l.team_id, + l.organization_id, l.end_user, l.requester_ip_address, '{}'::jsonb, + '{}'::jsonb, '{}'::jsonb, l.session_id, l.status, l.litellm_call_id +FROM UNNEST($1::text[], $2::text[], $3::text[], $4::float8[], $5::int4[], $6::int4[], + $7::int4[], $8::timestamp[], $9::timestamp[], $10::timestamp[], $11::int4[], $12::text[], + $13::text[], $14::text[], $15::text[], $16::text[], $17::text[], + $18::jsonb[], $19::text[], $20::text[], $21::text[], $22::text[], + $23::text[], $24::text[], $25::text[], $26::text[], $27::text[]) + AS l(request_id, call_type, api_key, spend, total_tokens, prompt_tokens, + completion_tokens, start_time, end_time, completion_start_time, + request_duration_ms, model, model_id, model_group, custom_llm_provider, + api_base, user_id, metadata, cache_hit, cache_key, team_id, organization_id, + end_user, requester_ip_address, session_id, status, litellm_call_id) +ON CONFLICT DO NOTHING diff --git a/litellm-rust/crates/db/src/lib.rs b/litellm-rust/crates/db/src/lib.rs index afa962cff0e..511a4bd387a 100644 --- a/litellm-rust/crates/db/src/lib.rs +++ b/litellm-rust/crates/db/src/lib.rs @@ -19,6 +19,6 @@ pub use end_user::{EndUser, RateLimits, find_end_user}; pub use error::Error; pub use failure::DbFailure; pub use spend::{ - EntitySpend, TeamMemberSpend, add_end_user_spend, add_key_spend, add_team_member_spend, - add_team_spend, add_user_spend, begin, lock_team, + EntitySpend, SpendLog, SpendLogRow, TeamMemberSpend, add_end_user_spend, add_key_spend, + add_team_member_spend, add_team_spend, add_user_spend, begin, insert_spend_logs, lock_team, }; diff --git a/litellm-rust/crates/db/src/spend/mod.rs b/litellm-rust/crates/db/src/spend/mod.rs index 1acf629af4e..dcfcee41f83 100644 --- a/litellm-rust/crates/db/src/spend/mod.rs +++ b/litellm-rust/crates/db/src/spend/mod.rs @@ -1,5 +1,6 @@ mod end_user_spend; mod key_spend; +mod spend_logs; mod team_lock; mod team_member_spend; mod team_spend; @@ -7,6 +8,7 @@ mod user_spend; pub use end_user_spend::add_end_user_spend; pub use key_spend::{EntitySpend, add_key_spend}; +pub use spend_logs::{SpendLog, SpendLogRow, insert_spend_logs}; pub use team_lock::{begin, lock_team}; pub use team_member_spend::{TeamMemberSpend, add_team_member_spend}; pub use team_spend::add_team_spend; diff --git a/litellm-rust/crates/db/src/spend/spend_logs.rs b/litellm-rust/crates/db/src/spend/spend_logs.rs new file mode 100644 index 00000000000..57a9ff85301 --- /dev/null +++ b/litellm-rust/crates/db/src/spend/spend_logs.rs @@ -0,0 +1,517 @@ +use chrono::{DateTime, NaiveDateTime}; +use sqlx::PgExecutor; + +use crate::Error; + +#[macro_rules_attribute::apply(crate::db_input)] +#[derive(Default)] +pub struct SpendLog { + pub request_id: String, + pub call_type: String, + pub api_key: String, + pub spend: f64, + pub total_tokens: i32, + pub prompt_tokens: i32, + pub completion_tokens: i32, + pub start_time: String, + pub end_time: String, + pub completion_start_time: Option, + pub request_duration_ms: Option, + pub model: String, + pub model_id: String, + pub model_group: String, + pub custom_llm_provider: String, + pub api_base: String, + pub user: String, + pub metadata: serde_json::Value, + pub cache_hit: String, + pub cache_key: String, + pub team_id: String, + pub organization_id: String, + pub end_user: String, + pub requester_ip_address: Option, + pub session_id: Option, + pub status: String, + pub litellm_call_id: Option, +} + +#[macro_rules_attribute::apply(crate::db_row)] +pub struct SpendLogRow { + pub request_id: String, + pub call_type: String, + pub api_key: String, + pub spend: f64, + pub total_tokens: i32, + pub prompt_tokens: i32, + pub completion_tokens: i32, + #[sqlx(rename = "startTime")] + pub start_time: NaiveDateTime, + #[sqlx(rename = "endTime")] + pub end_time: NaiveDateTime, + #[sqlx(rename = "completionStartTime")] + pub completion_start_time: Option, + pub request_duration_ms: Option, + pub model: String, + pub model_id: Option, + pub model_group: Option, + pub custom_llm_provider: Option, + pub api_base: Option, + pub user: Option, + pub metadata: Option, + pub cache_hit: Option, + pub cache_key: Option, + pub request_tags: Option, + pub team_id: Option, + pub organization_id: Option, + pub end_user: Option, + pub requester_ip_address: Option, + pub messages: Option, + pub response: Option, + pub proxy_server_request: Option, + pub session_id: Option, + pub status: Option, + pub litellm_call_id: Option, +} + +fn parse_timestamp(value: &str) -> Result { + DateTime::parse_from_rfc3339(value) + .map(|timestamp| timestamp.naive_utc()) + .or_else(|_| NaiveDateTime::parse_from_str(value, "%Y-%m-%dT%H:%M:%S%.f")) + .or_else(|_| NaiveDateTime::parse_from_str(value, "%Y-%m-%d %H:%M:%S%.f")) +} + +pub async fn insert_spend_logs( + executor: impl PgExecutor<'_>, + logs: &[SpendLog], +) -> Result<(), Error> { + if logs.is_empty() { + return Ok(()); + } + let start_times = logs + .iter() + .map(|log| parse_timestamp(&log.start_time)) + .collect::, _>>()?; + let end_times = logs + .iter() + .map(|log| parse_timestamp(&log.end_time)) + .collect::, _>>()?; + let completion_start_times = logs + .iter() + .map(|log| { + log.completion_start_time + .as_deref() + .map(parse_timestamp) + .transpose() + }) + .collect::, _>>()?; + macro_rules! column { + ($field:ident) => { + logs.iter() + .map(|log| log.$field.clone()) + .collect::>() + }; + } + sqlx::query_file!( + "queries/spend/insert_spend_logs.sql", + &column!(request_id), + &column!(call_type), + &column!(api_key), + &column!(spend), + &column!(total_tokens), + &column!(prompt_tokens), + &column!(completion_tokens), + &start_times, + &end_times, + &completion_start_times as &[Option], + &column!(request_duration_ms) as &[Option], + &column!(model), + &column!(model_id), + &column!(model_group), + &column!(custom_llm_provider), + &column!(api_base), + &column!(user), + &column!(metadata), + &column!(cache_hit), + &column!(cache_key), + &column!(team_id), + &column!(organization_id), + &column!(end_user), + &column!(requester_ip_address) as &[Option], + &column!(session_id) as &[Option], + &column!(status), + &column!(litellm_call_id) as &[Option], + ) + .execute(executor) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{SpendLog, SpendLogRow, insert_spend_logs}; + use crate::test_support::database_pool; + use rstest::{fixture, rstest}; + use serde_json::json; + use sqlx::{Acquire, Connection, PgExecutor, PgPool, types::chrono::NaiveDateTime}; + + #[fixture] + fn logs() -> [SpendLog; 2] { + [ + SpendLog { + request_id: "spend-logs-a".into(), + call_type: "acompletion".into(), + api_key: "key-a".into(), + spend: 0.125, + total_tokens: 17, + prompt_tokens: 11, + completion_tokens: 6, + start_time: "2026-01-02 03:04:05.123".into(), + end_time: "2026-01-02 03:04:06.789".into(), + completion_start_time: Some("2026-01-02 03:04:05.456".into()), + request_duration_ms: Some(1666), + model: "model-a".into(), + model_id: "deployment-a".into(), + model_group: "group-a".into(), + custom_llm_provider: "provider-a".into(), + api_base: "https://a.example.test".into(), + user: "user-a".into(), + metadata: json!({"tenant": "a", "nested": {"tokens": 11}, "flag": false}), + cache_hit: "False".into(), + cache_key: "cache-a".into(), + team_id: "team-a".into(), + organization_id: "org-a".into(), + end_user: "end-user-a".into(), + requester_ip_address: Some("192.0.2.1".into()), + session_id: Some("session-a".into()), + status: "success".into(), + litellm_call_id: Some("call-a".into()), + }, + SpendLog { + request_id: "spend-logs-b".into(), + call_type: "aembedding".into(), + api_key: "key-b".into(), + spend: 2.75, + total_tokens: 43, + prompt_tokens: 29, + completion_tokens: 14, + start_time: "2026-02-03 14:15:16.234".into(), + end_time: "2026-02-03 14:15:18.567".into(), + completion_start_time: None, + request_duration_ms: None, + model: "model-b".into(), + model_id: "deployment-b".into(), + model_group: "group-b".into(), + custom_llm_provider: "provider-b".into(), + api_base: "https://b.example.test".into(), + user: "user-b".into(), + metadata: json!({"tenant": "b", "tags": ["distinct", 7], "nullable": null}), + cache_hit: "True".into(), + cache_key: "cache-b".into(), + team_id: "team-b".into(), + organization_id: "org-b".into(), + end_user: "end-user-b".into(), + requester_ip_address: None, + session_id: None, + status: "failure".into(), + litellm_call_id: None, + }, + ] + } + + async fn read_log( + executor: impl PgExecutor<'_>, + request_id: &str, + ) -> Result { + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + sqlx::query_as::<_, SpendLogRow>( + r#"SELECT * FROM "LiteLLM_SpendLogs" WHERE request_id = $1"#, + ) + .bind(request_id) + .fetch_one(executor) + .await + } + + fn expected_log(log: &SpendLog) -> Result { + Ok(SpendLogRow { + request_id: log.request_id.clone(), + call_type: log.call_type.clone(), + api_key: log.api_key.clone(), + spend: log.spend, + total_tokens: log.total_tokens, + prompt_tokens: log.prompt_tokens, + completion_tokens: log.completion_tokens, + start_time: log.start_time.replace(' ', "T").parse()?, + end_time: log.end_time.replace(' ', "T").parse()?, + completion_start_time: log + .completion_start_time + .as_ref() + .map(|value| value.replace(' ', "T").parse()) + .transpose()?, + request_duration_ms: log.request_duration_ms, + model: log.model.clone(), + model_id: Some(log.model_id.clone()), + model_group: Some(log.model_group.clone()), + custom_llm_provider: Some(log.custom_llm_provider.clone()), + api_base: Some(log.api_base.clone()), + user: Some(log.user.clone()), + metadata: Some(log.metadata.clone()), + cache_hit: Some(log.cache_hit.clone()), + cache_key: Some(log.cache_key.clone()), + request_tags: Some(json!([])), + team_id: Some(log.team_id.clone()), + organization_id: Some(log.organization_id.clone()), + end_user: Some(log.end_user.clone()), + requester_ip_address: log.requester_ip_address.clone(), + messages: Some(json!({})), + response: Some(json!({})), + proxy_server_request: Some(json!({})), + session_id: log.session_id.clone(), + status: Some(log.status.clone()), + litellm_call_id: log.litellm_call_id.clone(), + }) + } + + #[rstest] + #[case::all_columns_and_duplicate()] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn inserts_distinct_rows_without_replacing_duplicates( + logs: [SpendLog; 2], + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + insert_spend_logs(&mut *tx, &logs).await?; + let replacement = SpendLog { + request_id: logs[0].request_id.clone(), + ..logs[1].clone() + }; + insert_spend_logs(&mut *tx, &[replacement]).await?; + assert_eq!( + read_log(&mut *tx, &logs[0].request_id).await?, + expected_log(&logs[0])? + ); + assert_eq!( + read_log(&mut *tx, &logs[1].request_id).await?, + expected_log(&logs[1])? + ); + tx.rollback().await?; + Ok(()) + } + + #[rstest] + #[case::partitioned_duplicate()] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn inserts_into_partitioned_logs_without_replacing_duplicates( + logs: [SpendLog; 2], + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "The test creates a temporary partitioned table at runtime" + )] + sqlx::raw_sql( + r#"CREATE TEMP TABLE "LiteLLM_SpendLogs" ( + LIKE public."LiteLLM_SpendLogs" INCLUDING DEFAULTS INCLUDING GENERATED, + PRIMARY KEY (request_id, "startTime") + ) PARTITION BY RANGE ("startTime"); + CREATE TEMP TABLE spend_logs_default + PARTITION OF pg_temp."LiteLLM_SpendLogs" DEFAULT; + SET LOCAL search_path TO pg_temp, public;"#, + ) + .execute(&mut *tx) + .await?; + tx.clear_cached_statements().await?; + insert_spend_logs(&mut *tx, &logs).await?; + let replacement = SpendLog { + spend: 99.0, + ..logs[0].clone() + }; + insert_spend_logs(&mut *tx, &[replacement]).await?; + #[expect( + clippy::disallowed_methods, + reason = "The partitioned test fixture is created at runtime" + )] + let count: (i64,) = sqlx::query_as(r#"SELECT count(*) FROM pg_temp."LiteLLM_SpendLogs""#) + .fetch_one(&mut *tx) + .await?; + assert_eq!(count.0, 2); + assert_eq!( + read_log(&mut *tx, &logs[0].request_id).await?, + expected_log(&logs[0])? + ); + assert_eq!( + read_log(&mut *tx, &logs[1].request_id).await?, + expected_log(&logs[1])? + ); + let distinct_partition_key = SpendLog { + request_id: logs[0].request_id.clone(), + ..logs[1].clone() + }; + insert_spend_logs(&mut *tx, &[distinct_partition_key]).await?; + #[expect( + clippy::disallowed_methods, + reason = "The partitioned test fixture is created at runtime" + )] + let count: (i64,) = sqlx::query_as(r#"SELECT count(*) FROM pg_temp."LiteLLM_SpendLogs""#) + .fetch_one(&mut *tx) + .await?; + assert_eq!(count.0, 3); + tx.rollback().await?; + Ok(()) + } + + #[rstest] + #[case::positive_offset_across_midnight( + ["2026-01-02T00:04:05.1236+02:00", "2026-01-02T00:04:06.7894+02:00", "2026-01-02T00:04:05.4567+02:00"], + ["2026-01-01T22:04:05.124", "2026-01-01T22:04:06.789", "2026-01-01T22:04:05.457"] + )] + #[case::negative_offset_across_midnight( + ["2026-01-01T23:04:05-02:00", "2026-01-01T23:04:06-02:00", "2026-01-01T23:04:05.456-02:00"], + ["2026-01-02T01:04:05", "2026-01-02T01:04:06", "2026-01-02T01:04:05.456"] + )] + #[case::utc( + ["2026-01-02T03:04:05Z", "2026-01-02T03:04:06Z", "2026-01-02T03:04:05.456Z"], + ["2026-01-02T03:04:05", "2026-01-02T03:04:06", "2026-01-02T03:04:05.456"] + )] + #[case::naive_iso( + ["2026-01-02T03:04:05", "2026-01-02T03:04:06", "2026-01-02T03:04:05.456"], + ["2026-01-02T03:04:05", "2026-01-02T03:04:06", "2026-01-02T03:04:05.456"] + )] + #[case::naive_space( + ["2026-01-02 03:04:05", "2026-01-02 03:04:06", "2026-01-02 03:04:05.456"], + ["2026-01-02T03:04:05", "2026-01-02T03:04:06", "2026-01-02T03:04:05.456"] + )] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn stores_utc_timestamps( + #[case] input: [&str; 3], + #[case] expected: [&str; 3], + logs: [SpendLog; 2], + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "The test sets a transaction-local timezone to verify timestamp handling" + )] + sqlx::raw_sql("SET LOCAL TIME ZONE 'America/Los_Angeles'") + .execute(&mut *tx) + .await?; + let log = SpendLog { + start_time: input[0].into(), + end_time: input[1].into(), + completion_start_time: Some(input[2].into()), + ..logs[0].clone() + }; + insert_spend_logs(&mut *tx, std::slice::from_ref(&log)).await?; + let row = read_log(&mut *tx, &log.request_id).await?; + assert_eq!(row.start_time, expected[0].parse::()?); + assert_eq!(row.end_time, expected[1].parse::()?); + assert_eq!( + row.completion_start_time, + Some(expected[2].parse::()?) + ); + tx.rollback().await?; + Ok(()) + } + + #[rstest] + #[case::empty_batch()] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn empty_input_leaves_rows_unchanged( + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let before: (i64,) = sqlx::query_as(r#"SELECT count(*) FROM "LiteLLM_SpendLogs""#) + .fetch_one(&mut *tx) + .await?; + insert_spend_logs(&mut *tx, &[]).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let after: (i64,) = sqlx::query_as(r#"SELECT count(*) FROM "LiteLLM_SpendLogs""#) + .fetch_one(&mut *tx) + .await?; + assert_eq!(before, after); + tx.rollback().await?; + Ok(()) + } + + #[rstest] + #[case::invalid_timestamp("not-a-timestamp", "2026-01-02T03:04:06", None)] + #[case::invalid_end("2026-01-02T03:04:05", "not-a-timestamp", None)] + #[case::invalid_completion( + "2026-01-02T03:04:05", + "2026-01-02T03:04:06", + Some("not-a-timestamp") + )] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn invalid_timestamp_allows_caller_rollback( + #[case] start_time: &str, + #[case] end_time: &str, + #[case] completion_start_time: Option<&str>, + logs: [SpendLog; 2], + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut outer = pool.begin().await?; + let mut tx = outer.begin().await?; + let valid = SpendLog { + request_id: "spend-logs-rollback-valid".into(), + ..logs[0].clone() + }; + let invalid = SpendLog { + request_id: "spend-logs-rollback-invalid".into(), + start_time: start_time.into(), + end_time: end_time.into(), + completion_start_time: completion_start_time.map(str::to_owned), + ..logs[1].clone() + }; + insert_spend_logs(&mut *tx, std::slice::from_ref(&valid)).await?; + let result = insert_spend_logs(&mut *tx, &[logs[0].clone(), invalid]).await; + assert!(matches!(result, Err(crate::Error::InvalidTimestamp(_)))); + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let partial_batch: (i64,) = sqlx::query_as( + r#"SELECT count(*) FROM "LiteLLM_SpendLogs" WHERE request_id = ANY($1)"#, + ) + .bind([logs[0].request_id.as_str(), "spend-logs-rollback-invalid"].as_slice()) + .fetch_one(&mut *tx) + .await?; + assert_eq!(partial_batch.0, 0); + tx.rollback().await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let remaining: (i64,) = sqlx::query_as( + r#"SELECT count(*) FROM "LiteLLM_SpendLogs" WHERE request_id = ANY($1)"#, + ) + .bind([valid.request_id.as_str(), "spend-logs-rollback-invalid"].as_slice()) + .fetch_one(&mut *outer) + .await?; + assert_eq!(remaining.0, 0); + outer.rollback().await?; + Ok(()) + } +} From 1b874ceb85313d5ff9a27e18db75e5cb90fcad81 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 26 Oct 2026 17:05:07 -0700 Subject: [PATCH 095/155] feat(db): add typed daily user spend upsert (#48235) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...8c2ebd7fc9fcbf19c1c25aa131a1053d96cfe.json | 31 ++ .../queries/spend/upsert_daily_user_spend.sql | 40 ++ litellm-rust/crates/db/src/lib.rs | 5 +- .../crates/db/src/spend/daily_user_spend.rs | 500 ++++++++++++++++++ litellm-rust/crates/db/src/spend/mod.rs | 2 + 5 files changed, 576 insertions(+), 2 deletions(-) create mode 100644 litellm-rust/crates/db/.sqlx/query-bec3b53762af021a118d63b841b8c2ebd7fc9fcbf19c1c25aa131a1053d96cfe.json create mode 100644 litellm-rust/crates/db/queries/spend/upsert_daily_user_spend.sql create mode 100644 litellm-rust/crates/db/src/spend/daily_user_spend.rs diff --git a/litellm-rust/crates/db/.sqlx/query-bec3b53762af021a118d63b841b8c2ebd7fc9fcbf19c1c25aa131a1053d96cfe.json b/litellm-rust/crates/db/.sqlx/query-bec3b53762af021a118d63b841b8c2ebd7fc9fcbf19c1c25aa131a1053d96cfe.json new file mode 100644 index 00000000000..6c56868e48e --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-bec3b53762af021a118d63b841b8c2ebd7fc9fcbf19c1c25aa131a1053d96cfe.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO \"LiteLLM_DailyUserSpend\" (\n id, user_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,\n completion_tokens, api_requests, successful_requests, failed_requests,\n cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,\n timed_requests, spend, updated_at\n)\nSELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,\n r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,\n r.completion_tokens, r.api_requests, r.successful_requests,\n r.failed_requests, r.cache_read, r.cache_creation, r.response_time,\n r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')\nFROM UNNEST(\n $1::text[], $2::text[], $3::text[], $4::text[], $5::text[],\n $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],\n $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],\n $16::int8[], $17::float8[], $18::text[]\n) AS r(\n entity_id, date, api_key, model, custom_llm_provider, endpoint,\n model_group, prompt_tokens, completion_tokens, api_requests,\n successful_requests, failed_requests, cache_read, cache_creation,\n response_time, timed_requests, spend, mcp_namespaced_tool_name\n)\nORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,\n r.mcp_namespaced_tool_name, r.endpoint\nON CONFLICT (\n user_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint\n) DO UPDATE SET\n prompt_tokens = \"LiteLLM_DailyUserSpend\".prompt_tokens + EXCLUDED.prompt_tokens,\n completion_tokens = \"LiteLLM_DailyUserSpend\".completion_tokens + EXCLUDED.completion_tokens,\n api_requests = \"LiteLLM_DailyUserSpend\".api_requests + EXCLUDED.api_requests,\n successful_requests = \"LiteLLM_DailyUserSpend\".successful_requests + EXCLUDED.successful_requests,\n failed_requests = \"LiteLLM_DailyUserSpend\".failed_requests + EXCLUDED.failed_requests,\n cache_read_input_tokens = \"LiteLLM_DailyUserSpend\".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,\n cache_creation_input_tokens = \"LiteLLM_DailyUserSpend\".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,\n total_response_time_ms = \"LiteLLM_DailyUserSpend\".total_response_time_ms + EXCLUDED.total_response_time_ms,\n timed_requests = \"LiteLLM_DailyUserSpend\".timed_requests + EXCLUDED.timed_requests,\n spend = \"LiteLLM_DailyUserSpend\".spend + EXCLUDED.spend,\n updated_at = (NOW() AT TIME ZONE 'UTC')\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Float8Array", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "bec3b53762af021a118d63b841b8c2ebd7fc9fcbf19c1c25aa131a1053d96cfe" +} diff --git a/litellm-rust/crates/db/queries/spend/upsert_daily_user_spend.sql b/litellm-rust/crates/db/queries/spend/upsert_daily_user_spend.sql new file mode 100644 index 00000000000..4a5db90dee2 --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/upsert_daily_user_spend.sql @@ -0,0 +1,40 @@ +INSERT INTO "LiteLLM_DailyUserSpend" ( + id, user_id, date, api_key, model, custom_llm_provider, + mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens, + completion_tokens, api_requests, successful_requests, failed_requests, + cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms, + timed_requests, spend, updated_at +) +SELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model, + r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens, + r.completion_tokens, r.api_requests, r.successful_requests, + r.failed_requests, r.cache_read, r.cache_creation, r.response_time, + r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC') +FROM UNNEST( + $1::text[], $2::text[], $3::text[], $4::text[], $5::text[], + $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[], + $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[], + $16::int8[], $17::float8[], $18::text[] +) AS r( + entity_id, date, api_key, model, custom_llm_provider, endpoint, + model_group, prompt_tokens, completion_tokens, api_requests, + successful_requests, failed_requests, cache_read, cache_creation, + response_time, timed_requests, spend, mcp_namespaced_tool_name +) +ORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider, + r.mcp_namespaced_tool_name, r.endpoint +ON CONFLICT ( + user_id, date, api_key, model, custom_llm_provider, + mcp_namespaced_tool_name, endpoint +) DO UPDATE SET + prompt_tokens = "LiteLLM_DailyUserSpend".prompt_tokens + EXCLUDED.prompt_tokens, + completion_tokens = "LiteLLM_DailyUserSpend".completion_tokens + EXCLUDED.completion_tokens, + api_requests = "LiteLLM_DailyUserSpend".api_requests + EXCLUDED.api_requests, + successful_requests = "LiteLLM_DailyUserSpend".successful_requests + EXCLUDED.successful_requests, + failed_requests = "LiteLLM_DailyUserSpend".failed_requests + EXCLUDED.failed_requests, + cache_read_input_tokens = "LiteLLM_DailyUserSpend".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens, + cache_creation_input_tokens = "LiteLLM_DailyUserSpend".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens, + total_response_time_ms = "LiteLLM_DailyUserSpend".total_response_time_ms + EXCLUDED.total_response_time_ms, + timed_requests = "LiteLLM_DailyUserSpend".timed_requests + EXCLUDED.timed_requests, + spend = "LiteLLM_DailyUserSpend".spend + EXCLUDED.spend, + updated_at = (NOW() AT TIME ZONE 'UTC') diff --git a/litellm-rust/crates/db/src/lib.rs b/litellm-rust/crates/db/src/lib.rs index 511a4bd387a..3bcd050440a 100644 --- a/litellm-rust/crates/db/src/lib.rs +++ b/litellm-rust/crates/db/src/lib.rs @@ -19,6 +19,7 @@ pub use end_user::{EndUser, RateLimits, find_end_user}; pub use error::Error; pub use failure::DbFailure; pub use spend::{ - EntitySpend, SpendLog, SpendLogRow, TeamMemberSpend, add_end_user_spend, add_key_spend, - add_team_member_spend, add_team_spend, add_user_spend, begin, insert_spend_logs, lock_team, + DailySpend, EntitySpend, SpendLog, SpendLogRow, TeamMemberSpend, add_end_user_spend, + add_key_spend, add_team_member_spend, add_team_spend, add_user_spend, begin, insert_spend_logs, + lock_team, upsert_daily_user_spend, }; diff --git a/litellm-rust/crates/db/src/spend/daily_user_spend.rs b/litellm-rust/crates/db/src/spend/daily_user_spend.rs new file mode 100644 index 00000000000..3530eb245c5 --- /dev/null +++ b/litellm-rust/crates/db/src/spend/daily_user_spend.rs @@ -0,0 +1,500 @@ +use sqlx::PgExecutor; + +use crate::Error; + +#[macro_rules_attribute::apply(crate::db_input)] +#[derive(Default)] +pub struct DailySpend { + pub entity_id: String, + pub date: String, + pub api_key: String, + pub model: String, + pub custom_llm_provider: String, + pub mcp_namespaced_tool_name: String, + pub endpoint: String, + pub model_group: Option, + pub prompt_tokens: i64, + pub completion_tokens: i64, + pub api_requests: i64, + pub successful_requests: i64, + pub failed_requests: i64, + pub cache_read_input_tokens: i64, + pub cache_creation_input_tokens: i64, + pub total_response_time_ms: i64, + pub timed_requests: i64, + pub spend: f64, +} + +pub async fn upsert_daily_user_spend( + executor: impl PgExecutor<'_>, + rows: &[DailySpend], +) -> Result<(), Error> { + if rows.is_empty() { + return Ok(()); + } + macro_rules! column { + ($field:ident) => { + rows.iter() + .map(|row| row.$field.clone()) + .collect::>() + }; + } + sqlx::query_file!( + "queries/spend/upsert_daily_user_spend.sql", + &column!(entity_id), + &column!(date), + &column!(api_key), + &column!(model), + &column!(custom_llm_provider), + &column!(endpoint), + &column!(model_group) as &[Option], + &column!(prompt_tokens), + &column!(completion_tokens), + &column!(api_requests), + &column!(successful_requests), + &column!(failed_requests), + &column!(cache_read_input_tokens), + &column!(cache_creation_input_tokens), + &column!(total_response_time_ms), + &column!(timed_requests), + &column!(spend), + &column!(mcp_namespaced_tool_name), + ) + .execute(executor) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{DailySpend, upsert_daily_user_spend}; + use crate::{DailyUserRow, test_support::database_pool}; + use rstest::rstest; + use sqlx::{PgPool, types::chrono::NaiveDateTime}; + + #[rstest] + #[case::distinct_tools("server/search", "server/write")] + #[case::default_empty_tool("", "server/search")] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn daily_user_spend_separates_tools_and_accumulates( + #[case] first_tool: &str, + #[case] second_tool: &str, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + let first = DailySpend { + entity_id: format!("daily-user-tools-{first_tool}"), + date: "2026-01-15".to_owned(), + api_key: "tool-key".to_owned(), + model: "tool-model".to_owned(), + custom_llm_provider: "tool-provider".to_owned(), + endpoint: "/mcp".to_owned(), + model_group: Some("tool-group".to_owned()), + mcp_namespaced_tool_name: first_tool.to_owned(), + prompt_tokens: 3, + api_requests: 1, + spend: 0.25, + ..Default::default() + }; + let second = DailySpend { + mcp_namespaced_tool_name: second_tool.to_owned(), + prompt_tokens: 7, + api_requests: 2, + spend: 0.5, + ..first.clone() + }; + upsert_daily_user_spend(&mut *tx, &[second.clone(), first.clone()]).await?; + upsert_daily_user_spend(&mut *tx, &[first.clone(), second]).await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test reads stored rows to verify tool attribution" + )] + let rows: Vec = sqlx::query_as( + r#"SELECT * FROM "LiteLLM_DailyUserSpend" + WHERE user_id = $1 + ORDER BY mcp_namespaced_tool_name, endpoint"#, + ) + .bind(&first.entity_id) + .fetch_all(&mut *tx) + .await?; + + assert_eq!(rows.len(), 2); + assert_eq!( + rows[0].mcp_namespaced_tool_name.as_deref(), + Some(first_tool) + ); + assert_eq!( + rows[1].mcp_namespaced_tool_name.as_deref(), + Some(second_tool) + ); + assert_eq!( + (rows[0].prompt_tokens, rows[0].api_requests, rows[0].spend), + (6, 2, 0.5) + ); + assert_eq!( + (rows[1].prompt_tokens, rows[1].api_requests, rows[1].spend), + (14, 4, 1.0) + ); + Ok(()) + } + + #[rstest] + #[case::full_keys( + "daily-user-base", + "daily-key-base", + "daily-model", + "daily-provider", + "/chat/completions", + Some("first-group") + )] + #[case::initial_model_group_none( + "daily-user-base", + "daily-key-base", + "daily-model", + "daily-provider", + "/chat/completions", + None + )] + #[case::empty_optional_keys("", "", "", "", "", Some(""))] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn daily_user_spend_upsert_preserves_keys_and_accumulates( + #[case] user_id: &'static str, + #[case] api_key: &'static str, + #[case] model: &'static str, + #[case] custom_llm_provider: &'static str, + #[case] endpoint: &'static str, + #[case] first_model_group: Option<&'static str>, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test exercises UTC timestamp behavior under a non-UTC session" + )] + sqlx::query("SET LOCAL TIME ZONE 'America/Los_Angeles'") + .execute(&mut *tx) + .await?; + + let first_batch = [ + DailySpend { + entity_id: user_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: first_model_group.map(str::to_owned), + prompt_tokens: 1, + completion_tokens: 2, + api_requests: 3, + successful_requests: 4, + failed_requests: 5, + cache_read_input_tokens: 6, + cache_creation_input_tokens: 7, + total_response_time_ms: 8, + timed_requests: 9, + spend: 0.125, + }, + DailySpend { + entity_id: user_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: "endpoint-sentinel".to_owned(), + model_group: Some("sentinel-endpoint-group".to_owned()), + prompt_tokens: 101, + completion_tokens: 102, + api_requests: 103, + successful_requests: 104, + failed_requests: 105, + cache_read_input_tokens: 106, + cache_creation_input_tokens: 107, + total_response_time_ms: 108, + timed_requests: 109, + spend: 1.25, + }, + DailySpend { + entity_id: user_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: "api-key-sentinel".to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("sentinel-key-group".to_owned()), + prompt_tokens: 201, + completion_tokens: 202, + api_requests: 203, + successful_requests: 204, + failed_requests: 205, + cache_read_input_tokens: 206, + cache_creation_input_tokens: 207, + total_response_time_ms: 208, + timed_requests: 209, + spend: 2.5, + }, + DailySpend { + entity_id: "entity-sentinel".to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("sentinel-entity-group".to_owned()), + prompt_tokens: 301, + completion_tokens: 302, + api_requests: 303, + successful_requests: 304, + failed_requests: 305, + cache_read_input_tokens: 306, + cache_creation_input_tokens: 307, + total_response_time_ms: 308, + timed_requests: 309, + spend: 3.75, + }, + ]; + upsert_daily_user_spend(&mut *tx, &first_batch).await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test captures the inserted row identity before conflict handling" + )] + let (initial_id, initial_created_at): (String, NaiveDateTime) = sqlx::query_as( + r#"SELECT id, created_at + FROM "LiteLLM_DailyUserSpend" + WHERE user_id = $1 AND date = $2 AND api_key = $3 AND model = $4 + AND custom_llm_provider = $5 AND endpoint = $6"#, + ) + .bind(user_id) + .bind("2026-01-15") + .bind(api_key) + .bind(model) + .bind(custom_llm_provider) + .bind(endpoint) + .fetch_one(&mut *tx) + .await?; + assert!(!initial_id.is_empty()); + + #[expect( + clippy::disallowed_methods, + reason = "The integration test seeds updated_at before exercising the conflict update" + )] + let seeded_updated_at: NaiveDateTime = sqlx::query_scalar( + r#"UPDATE "LiteLLM_DailyUserSpend" + SET updated_at = TIMESTAMP '2000-01-01' + WHERE id = $1 + RETURNING updated_at"#, + ) + .bind(&initial_id) + .fetch_one(&mut *tx) + .await?; + + let increment = [DailySpend { + entity_id: user_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("second-group".to_owned()), + prompt_tokens: 10, + completion_tokens: 20, + api_requests: 30, + successful_requests: 40, + failed_requests: 50, + cache_read_input_tokens: 60, + cache_creation_input_tokens: 70, + total_response_time_ms: 80, + timed_requests: 90, + spend: 0.625, + }]; + upsert_daily_user_spend(&mut *tx, &increment).await?; + upsert_daily_user_spend(&mut *tx, &[]).await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test captures the transaction timestamp for assertions" + )] + let expected_updated_at: NaiveDateTime = + sqlx::query_scalar("SELECT (NOW() AT TIME ZONE 'UTC')::timestamp(3)") + .fetch_one(&mut *tx) + .await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test reads all rows to verify attribution and accumulation" + )] + let rows: Vec = sqlx::query_as( + r#"SELECT id, user_id, api_key, model, custom_llm_provider, endpoint, + model_group, mcp_namespaced_tool_name, prompt_tokens, + completion_tokens, api_requests, successful_requests, + failed_requests, cache_read_input_tokens, + cache_creation_input_tokens, total_response_time_ms, + timed_requests, spend, date, created_at, updated_at + FROM "LiteLLM_DailyUserSpend" + WHERE date = $1 AND model = $2 AND custom_llm_provider = $3 + AND mcp_namespaced_tool_name = '' AND user_id = ANY($4) + ORDER BY CASE + WHEN user_id = $5 AND api_key = $6 AND endpoint = $7 THEN 0 + WHEN user_id = $5 AND api_key = $6 AND endpoint = $9 THEN 1 + WHEN user_id = $5 AND api_key = $8 AND endpoint = $7 THEN 2 + WHEN user_id = $10 AND api_key = $6 AND endpoint = $7 THEN 3 + ELSE 4 + END"#, + ) + .bind("2026-01-15") + .bind(model) + .bind(custom_llm_provider) + .bind(vec![user_id, "entity-sentinel"]) + .bind(user_id) + .bind(api_key) + .bind(endpoint) + .bind("api-key-sentinel") + .bind("endpoint-sentinel") + .bind("entity-sentinel") + .fetch_all(&mut *tx) + .await?; + + assert_eq!(rows.len(), 4); + assert_eq!(rows[0].id, initial_id); + assert_eq!(rows[0].user_id.as_deref(), Some(user_id)); + assert_eq!(rows[0].api_key, api_key); + assert_eq!(rows[0].model.as_deref(), Some(model)); + assert_eq!( + rows[0].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[0].endpoint.as_deref(), Some(endpoint)); + assert_eq!(rows[0].model_group.as_deref(), first_model_group); + assert_eq!(rows[0].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[0].prompt_tokens, + rows[0].completion_tokens, + rows[0].api_requests, + rows[0].successful_requests, + rows[0].failed_requests, + rows[0].cache_read_input_tokens, + rows[0].cache_creation_input_tokens, + rows[0].total_response_time_ms, + rows[0].timed_requests, + rows[0].spend + ), + (11, 22, 33, 44, 55, 66, 77, 88, 99, 0.75) + ); + assert_eq!(rows[0].date, "2026-01-15"); + assert_eq!(rows[0].created_at, initial_created_at); + assert!(rows[0].updated_at > seeded_updated_at); + assert_eq!(rows[0].updated_at, expected_updated_at); + + assert!(!rows[1].id.is_empty()); + assert_eq!(rows[1].user_id.as_deref(), Some(user_id)); + assert_eq!(rows[1].api_key, api_key); + assert_eq!(rows[1].model.as_deref(), Some(model)); + assert_eq!( + rows[1].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[1].endpoint.as_deref(), Some("endpoint-sentinel")); + assert_eq!( + rows[1].model_group.as_deref(), + Some("sentinel-endpoint-group") + ); + assert_eq!(rows[1].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[1].prompt_tokens, + rows[1].completion_tokens, + rows[1].api_requests, + rows[1].successful_requests, + rows[1].failed_requests, + rows[1].cache_read_input_tokens, + rows[1].cache_creation_input_tokens, + rows[1].total_response_time_ms, + rows[1].timed_requests, + rows[1].spend + ), + (101, 102, 103, 104, 105, 106, 107, 108, 109, 1.25) + ); + assert_eq!(rows[1].date, "2026-01-15"); + assert!(rows[1].updated_at > seeded_updated_at); + assert_eq!(rows[1].updated_at, expected_updated_at); + + assert!(!rows[2].id.is_empty()); + assert_eq!(rows[2].user_id.as_deref(), Some(user_id)); + assert_eq!(rows[2].api_key, "api-key-sentinel"); + assert_eq!(rows[2].model.as_deref(), Some(model)); + assert_eq!( + rows[2].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[2].endpoint.as_deref(), Some(endpoint)); + assert_eq!(rows[2].model_group.as_deref(), Some("sentinel-key-group")); + assert_eq!(rows[2].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[2].prompt_tokens, + rows[2].completion_tokens, + rows[2].api_requests, + rows[2].successful_requests, + rows[2].failed_requests, + rows[2].cache_read_input_tokens, + rows[2].cache_creation_input_tokens, + rows[2].total_response_time_ms, + rows[2].timed_requests, + rows[2].spend + ), + (201, 202, 203, 204, 205, 206, 207, 208, 209, 2.5) + ); + assert_eq!(rows[2].date, "2026-01-15"); + assert!(rows[2].updated_at > seeded_updated_at); + assert_eq!(rows[2].updated_at, expected_updated_at); + + assert!(!rows[3].id.is_empty()); + assert_eq!(rows[3].user_id.as_deref(), Some("entity-sentinel")); + assert_eq!(rows[3].api_key, api_key); + assert_eq!(rows[3].model.as_deref(), Some(model)); + assert_eq!( + rows[3].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[3].endpoint.as_deref(), Some(endpoint)); + assert_eq!( + rows[3].model_group.as_deref(), + Some("sentinel-entity-group") + ); + assert_eq!(rows[3].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[3].prompt_tokens, + rows[3].completion_tokens, + rows[3].api_requests, + rows[3].successful_requests, + rows[3].failed_requests, + rows[3].cache_read_input_tokens, + rows[3].cache_creation_input_tokens, + rows[3].total_response_time_ms, + rows[3].timed_requests, + rows[3].spend + ), + (301, 302, 303, 304, 305, 306, 307, 308, 309, 3.75) + ); + assert_eq!(rows[3].date, "2026-01-15"); + assert!(rows[3].updated_at > seeded_updated_at); + assert_eq!(rows[3].updated_at, expected_updated_at); + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/spend/mod.rs b/litellm-rust/crates/db/src/spend/mod.rs index dcfcee41f83..f50440e06de 100644 --- a/litellm-rust/crates/db/src/spend/mod.rs +++ b/litellm-rust/crates/db/src/spend/mod.rs @@ -1,3 +1,4 @@ +mod daily_user_spend; mod end_user_spend; mod key_spend; mod spend_logs; @@ -6,6 +7,7 @@ mod team_member_spend; mod team_spend; mod user_spend; +pub use daily_user_spend::{DailySpend, upsert_daily_user_spend}; pub use end_user_spend::add_end_user_spend; pub use key_spend::{EntitySpend, add_key_spend}; pub use spend_logs::{SpendLog, SpendLogRow, insert_spend_logs}; From a2545134f370696ae1dd2ba7102d0ac4da552968 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 26 Oct 2026 17:05:07 -0700 Subject: [PATCH 096/155] feat(db): add typed daily team spend upsert (#48236) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...74dca981ff954452b35ffbace3e022035b416.json | 31 ++ .../queries/spend/upsert_daily_team_spend.sql | 40 ++ litellm-rust/crates/db/src/lib.rs | 2 +- .../crates/db/src/spend/daily_team_spend.rs | 485 ++++++++++++++++++ litellm-rust/crates/db/src/spend/mod.rs | 2 + 5 files changed, 559 insertions(+), 1 deletion(-) create mode 100644 litellm-rust/crates/db/.sqlx/query-9c51401a8907f818e96da28050b74dca981ff954452b35ffbace3e022035b416.json create mode 100644 litellm-rust/crates/db/queries/spend/upsert_daily_team_spend.sql create mode 100644 litellm-rust/crates/db/src/spend/daily_team_spend.rs diff --git a/litellm-rust/crates/db/.sqlx/query-9c51401a8907f818e96da28050b74dca981ff954452b35ffbace3e022035b416.json b/litellm-rust/crates/db/.sqlx/query-9c51401a8907f818e96da28050b74dca981ff954452b35ffbace3e022035b416.json new file mode 100644 index 00000000000..9886a9505d8 --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-9c51401a8907f818e96da28050b74dca981ff954452b35ffbace3e022035b416.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO \"LiteLLM_DailyTeamSpend\" (\n id, team_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,\n completion_tokens, api_requests, successful_requests, failed_requests,\n cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,\n timed_requests, spend, updated_at\n)\nSELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,\n r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,\n r.completion_tokens, r.api_requests, r.successful_requests,\n r.failed_requests, r.cache_read, r.cache_creation, r.response_time,\n r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')\nFROM UNNEST(\n $1::text[], $2::text[], $3::text[], $4::text[], $5::text[],\n $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],\n $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],\n $16::int8[], $17::float8[], $18::text[]\n) AS r(\n entity_id, date, api_key, model, custom_llm_provider, endpoint,\n model_group, prompt_tokens, completion_tokens, api_requests,\n successful_requests, failed_requests, cache_read, cache_creation,\n response_time, timed_requests, spend, mcp_namespaced_tool_name\n)\nORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,\n r.mcp_namespaced_tool_name, r.endpoint\nON CONFLICT (\n team_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint\n) DO UPDATE SET\n prompt_tokens = \"LiteLLM_DailyTeamSpend\".prompt_tokens + EXCLUDED.prompt_tokens,\n completion_tokens = \"LiteLLM_DailyTeamSpend\".completion_tokens + EXCLUDED.completion_tokens,\n api_requests = \"LiteLLM_DailyTeamSpend\".api_requests + EXCLUDED.api_requests,\n successful_requests = \"LiteLLM_DailyTeamSpend\".successful_requests + EXCLUDED.successful_requests,\n failed_requests = \"LiteLLM_DailyTeamSpend\".failed_requests + EXCLUDED.failed_requests,\n cache_read_input_tokens = \"LiteLLM_DailyTeamSpend\".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,\n cache_creation_input_tokens = \"LiteLLM_DailyTeamSpend\".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,\n total_response_time_ms = \"LiteLLM_DailyTeamSpend\".total_response_time_ms + EXCLUDED.total_response_time_ms,\n timed_requests = \"LiteLLM_DailyTeamSpend\".timed_requests + EXCLUDED.timed_requests,\n spend = \"LiteLLM_DailyTeamSpend\".spend + EXCLUDED.spend,\n updated_at = (NOW() AT TIME ZONE 'UTC')\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Float8Array", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "9c51401a8907f818e96da28050b74dca981ff954452b35ffbace3e022035b416" +} diff --git a/litellm-rust/crates/db/queries/spend/upsert_daily_team_spend.sql b/litellm-rust/crates/db/queries/spend/upsert_daily_team_spend.sql new file mode 100644 index 00000000000..a9bdef34043 --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/upsert_daily_team_spend.sql @@ -0,0 +1,40 @@ +INSERT INTO "LiteLLM_DailyTeamSpend" ( + id, team_id, date, api_key, model, custom_llm_provider, + mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens, + completion_tokens, api_requests, successful_requests, failed_requests, + cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms, + timed_requests, spend, updated_at +) +SELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model, + r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens, + r.completion_tokens, r.api_requests, r.successful_requests, + r.failed_requests, r.cache_read, r.cache_creation, r.response_time, + r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC') +FROM UNNEST( + $1::text[], $2::text[], $3::text[], $4::text[], $5::text[], + $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[], + $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[], + $16::int8[], $17::float8[], $18::text[] +) AS r( + entity_id, date, api_key, model, custom_llm_provider, endpoint, + model_group, prompt_tokens, completion_tokens, api_requests, + successful_requests, failed_requests, cache_read, cache_creation, + response_time, timed_requests, spend, mcp_namespaced_tool_name +) +ORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider, + r.mcp_namespaced_tool_name, r.endpoint +ON CONFLICT ( + team_id, date, api_key, model, custom_llm_provider, + mcp_namespaced_tool_name, endpoint +) DO UPDATE SET + prompt_tokens = "LiteLLM_DailyTeamSpend".prompt_tokens + EXCLUDED.prompt_tokens, + completion_tokens = "LiteLLM_DailyTeamSpend".completion_tokens + EXCLUDED.completion_tokens, + api_requests = "LiteLLM_DailyTeamSpend".api_requests + EXCLUDED.api_requests, + successful_requests = "LiteLLM_DailyTeamSpend".successful_requests + EXCLUDED.successful_requests, + failed_requests = "LiteLLM_DailyTeamSpend".failed_requests + EXCLUDED.failed_requests, + cache_read_input_tokens = "LiteLLM_DailyTeamSpend".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens, + cache_creation_input_tokens = "LiteLLM_DailyTeamSpend".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens, + total_response_time_ms = "LiteLLM_DailyTeamSpend".total_response_time_ms + EXCLUDED.total_response_time_ms, + timed_requests = "LiteLLM_DailyTeamSpend".timed_requests + EXCLUDED.timed_requests, + spend = "LiteLLM_DailyTeamSpend".spend + EXCLUDED.spend, + updated_at = (NOW() AT TIME ZONE 'UTC') diff --git a/litellm-rust/crates/db/src/lib.rs b/litellm-rust/crates/db/src/lib.rs index 3bcd050440a..5b88bcabf32 100644 --- a/litellm-rust/crates/db/src/lib.rs +++ b/litellm-rust/crates/db/src/lib.rs @@ -21,5 +21,5 @@ pub use failure::DbFailure; pub use spend::{ DailySpend, EntitySpend, SpendLog, SpendLogRow, TeamMemberSpend, add_end_user_spend, add_key_spend, add_team_member_spend, add_team_spend, add_user_spend, begin, insert_spend_logs, - lock_team, upsert_daily_user_spend, + lock_team, upsert_daily_team_spend, upsert_daily_user_spend, }; diff --git a/litellm-rust/crates/db/src/spend/daily_team_spend.rs b/litellm-rust/crates/db/src/spend/daily_team_spend.rs new file mode 100644 index 00000000000..f2acd23f468 --- /dev/null +++ b/litellm-rust/crates/db/src/spend/daily_team_spend.rs @@ -0,0 +1,485 @@ +use sqlx::PgExecutor; + +use super::DailySpend; +use crate::Error; + +pub async fn upsert_daily_team_spend( + executor: impl PgExecutor<'_>, + rows: &[DailySpend], +) -> Result<(), Error> { + if rows.is_empty() { + return Ok(()); + } + macro_rules! column { + ($field:ident) => { + rows.iter() + .map(|row| row.$field.clone()) + .collect::>() + }; + } + sqlx::query_file!( + "queries/spend/upsert_daily_team_spend.sql", + &column!(entity_id), + &column!(date), + &column!(api_key), + &column!(model), + &column!(custom_llm_provider), + &column!(endpoint), + &column!(model_group) as &[Option], + &column!(prompt_tokens), + &column!(completion_tokens), + &column!(api_requests), + &column!(successful_requests), + &column!(failed_requests), + &column!(cache_read_input_tokens), + &column!(cache_creation_input_tokens), + &column!(total_response_time_ms), + &column!(timed_requests), + &column!(spend), + &column!(mcp_namespaced_tool_name), + ) + .execute(executor) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{DailySpend, upsert_daily_team_spend}; + use crate::{DailyTeamRow, test_support::database_pool}; + use rstest::rstest; + use sqlx::{PgPool, types::chrono::NaiveDateTime}; + + #[rstest] + #[case::distinct_tools("server.fetch", "server.search")] + #[case::default_and_named_tool("", "server.search")] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn daily_team_spend_separates_tools_and_accumulates( + #[case] first_tool: &str, + #[case] second_tool: &str, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + let first = DailySpend { + entity_id: "daily-team-tools".to_owned(), + date: "2026-01-15".to_owned(), + api_key: "daily-key-tools".to_owned(), + model: "daily-model".to_owned(), + custom_llm_provider: "daily-provider".to_owned(), + endpoint: "/mcp".to_owned(), + mcp_namespaced_tool_name: first_tool.to_owned(), + api_requests: 1, + spend: 0.125, + ..Default::default() + }; + let second = DailySpend { + mcp_namespaced_tool_name: second_tool.to_owned(), + api_requests: 2, + spend: 0.25, + ..first.clone() + }; + upsert_daily_team_spend(&mut *tx, &[second.clone(), first.clone()]).await?; + upsert_daily_team_spend( + &mut *tx, + &[ + DailySpend { + api_requests: 3, + spend: 0.5, + ..first + }, + DailySpend { + api_requests: 5, + spend: 1.0, + ..second + }, + ], + ) + .await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test reads stored tool attribution and counters" + )] + let rows = sqlx::query_as::<_, DailyTeamRow>( + r#"SELECT * FROM "LiteLLM_DailyTeamSpend" + WHERE team_id = $1 AND date = $2 + ORDER BY mcp_namespaced_tool_name"#, + ) + .bind("daily-team-tools") + .bind("2026-01-15") + .fetch_all(&mut *tx) + .await?; + + assert_eq!(rows.len(), 2); + assert_eq!( + rows[0].mcp_namespaced_tool_name.as_deref(), + Some(first_tool) + ); + assert_eq!( + rows[1].mcp_namespaced_tool_name.as_deref(), + Some(second_tool) + ); + assert_eq!((rows[0].api_requests, rows[0].spend), (4, 0.625)); + assert_eq!((rows[1].api_requests, rows[1].spend), (7, 1.25)); + Ok(()) + } + + #[rstest] + #[case::full_keys( + "daily-team-base", + "daily-key-base", + "daily-model", + "daily-provider", + "/chat/completions", + Some("first-group") + )] + #[case::initial_model_group_none( + "daily-team-base", + "daily-key-base", + "daily-model", + "daily-provider", + "/chat/completions", + None + )] + #[case::empty_optional_keys("", "", "", "", "", Some(""))] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn daily_team_spend_upsert_preserves_keys_and_accumulates( + #[case] team_id: &'static str, + #[case] api_key: &'static str, + #[case] model: &'static str, + #[case] custom_llm_provider: &'static str, + #[case] endpoint: &'static str, + #[case] first_model_group: Option<&'static str>, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test exercises UTC timestamp behavior under a non-UTC session" + )] + sqlx::query("SET LOCAL TIME ZONE 'America/Los_Angeles'") + .execute(&mut *tx) + .await?; + + let first_batch = [ + DailySpend { + entity_id: team_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: first_model_group.map(str::to_owned), + prompt_tokens: 1, + completion_tokens: 2, + api_requests: 3, + successful_requests: 4, + failed_requests: 5, + cache_read_input_tokens: 6, + cache_creation_input_tokens: 7, + total_response_time_ms: 8, + timed_requests: 9, + spend: 0.125, + }, + DailySpend { + entity_id: team_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: "endpoint-sentinel".to_owned(), + model_group: Some("sentinel-endpoint-group".to_owned()), + prompt_tokens: 101, + completion_tokens: 102, + api_requests: 103, + successful_requests: 104, + failed_requests: 105, + cache_read_input_tokens: 106, + cache_creation_input_tokens: 107, + total_response_time_ms: 108, + timed_requests: 109, + spend: 1.25, + }, + DailySpend { + entity_id: team_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: "api-key-sentinel".to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("sentinel-key-group".to_owned()), + prompt_tokens: 201, + completion_tokens: 202, + api_requests: 203, + successful_requests: 204, + failed_requests: 205, + cache_read_input_tokens: 206, + cache_creation_input_tokens: 207, + total_response_time_ms: 208, + timed_requests: 209, + spend: 2.5, + }, + DailySpend { + entity_id: "entity-sentinel".to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("sentinel-entity-group".to_owned()), + prompt_tokens: 301, + completion_tokens: 302, + api_requests: 303, + successful_requests: 304, + failed_requests: 305, + cache_read_input_tokens: 306, + cache_creation_input_tokens: 307, + total_response_time_ms: 308, + timed_requests: 309, + spend: 3.75, + }, + ]; + upsert_daily_team_spend(&mut *tx, &first_batch).await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test captures the inserted row identity before conflict handling" + )] + let (initial_id, initial_created_at): (String, NaiveDateTime) = sqlx::query_as( + r#"SELECT id, created_at + FROM "LiteLLM_DailyTeamSpend" + WHERE team_id = $1 AND date = $2 AND api_key = $3 AND model = $4 + AND custom_llm_provider = $5 AND endpoint = $6"#, + ) + .bind(team_id) + .bind("2026-01-15") + .bind(api_key) + .bind(model) + .bind(custom_llm_provider) + .bind(endpoint) + .fetch_one(&mut *tx) + .await?; + assert!(!initial_id.is_empty()); + + #[expect( + clippy::disallowed_methods, + reason = "The integration test seeds updated_at before exercising the conflict update" + )] + let seeded_updated_at: NaiveDateTime = sqlx::query_scalar( + r#"UPDATE "LiteLLM_DailyTeamSpend" + SET updated_at = TIMESTAMP '2000-01-01' + WHERE id = $1 + RETURNING updated_at"#, + ) + .bind(&initial_id) + .fetch_one(&mut *tx) + .await?; + + let increment = [DailySpend { + entity_id: team_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("second-group".to_owned()), + prompt_tokens: 10, + completion_tokens: 20, + api_requests: 30, + successful_requests: 40, + failed_requests: 50, + cache_read_input_tokens: 60, + cache_creation_input_tokens: 70, + total_response_time_ms: 80, + timed_requests: 90, + spend: 0.625, + }]; + upsert_daily_team_spend(&mut *tx, &increment).await?; + upsert_daily_team_spend(&mut *tx, &[]).await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test captures the transaction timestamp for assertions" + )] + let expected_updated_at: NaiveDateTime = + sqlx::query_scalar("SELECT (NOW() AT TIME ZONE 'UTC')::timestamp(3)") + .fetch_one(&mut *tx) + .await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test reads all rows to verify attribution and accumulation" + )] + let rows: Vec = sqlx::query_as( + r#"SELECT id, team_id, api_key, model, custom_llm_provider, endpoint, + model_group, mcp_namespaced_tool_name, prompt_tokens, + completion_tokens, api_requests, successful_requests, + failed_requests, cache_read_input_tokens, + cache_creation_input_tokens, total_response_time_ms, + timed_requests, spend, date, created_at, updated_at + FROM "LiteLLM_DailyTeamSpend" + WHERE date = $1 AND model = $2 AND custom_llm_provider = $3 + AND mcp_namespaced_tool_name = '' AND team_id = ANY($4) + ORDER BY CASE + WHEN team_id = $5 AND api_key = $6 AND endpoint = $7 THEN 0 + WHEN team_id = $5 AND api_key = $6 AND endpoint = $9 THEN 1 + WHEN team_id = $5 AND api_key = $8 AND endpoint = $7 THEN 2 + WHEN team_id = $10 AND api_key = $6 AND endpoint = $7 THEN 3 + ELSE 4 + END"#, + ) + .bind("2026-01-15") + .bind(model) + .bind(custom_llm_provider) + .bind(vec![team_id, "entity-sentinel"]) + .bind(team_id) + .bind(api_key) + .bind(endpoint) + .bind("api-key-sentinel") + .bind("endpoint-sentinel") + .bind("entity-sentinel") + .fetch_all(&mut *tx) + .await?; + + assert_eq!(rows.len(), 4); + assert_eq!(rows[0].id, initial_id); + assert_eq!(rows[0].team_id.as_deref(), Some(team_id)); + assert_eq!(rows[0].api_key, api_key); + assert_eq!(rows[0].model.as_deref(), Some(model)); + assert_eq!( + rows[0].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[0].endpoint.as_deref(), Some(endpoint)); + assert_eq!(rows[0].model_group.as_deref(), first_model_group); + assert_eq!(rows[0].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[0].prompt_tokens, + rows[0].completion_tokens, + rows[0].api_requests, + rows[0].successful_requests, + rows[0].failed_requests, + rows[0].cache_read_input_tokens, + rows[0].cache_creation_input_tokens, + rows[0].total_response_time_ms, + rows[0].timed_requests, + rows[0].spend + ), + (11, 22, 33, 44, 55, 66, 77, 88, 99, 0.75) + ); + assert_eq!(rows[0].date, "2026-01-15"); + assert_eq!(rows[0].created_at, initial_created_at); + assert!(rows[0].updated_at > seeded_updated_at); + assert_eq!(rows[0].updated_at, expected_updated_at); + + assert!(!rows[1].id.is_empty()); + assert_eq!(rows[1].team_id.as_deref(), Some(team_id)); + assert_eq!(rows[1].api_key, api_key); + assert_eq!(rows[1].model.as_deref(), Some(model)); + assert_eq!( + rows[1].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[1].endpoint.as_deref(), Some("endpoint-sentinel")); + assert_eq!( + rows[1].model_group.as_deref(), + Some("sentinel-endpoint-group") + ); + assert_eq!(rows[1].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[1].prompt_tokens, + rows[1].completion_tokens, + rows[1].api_requests, + rows[1].successful_requests, + rows[1].failed_requests, + rows[1].cache_read_input_tokens, + rows[1].cache_creation_input_tokens, + rows[1].total_response_time_ms, + rows[1].timed_requests, + rows[1].spend + ), + (101, 102, 103, 104, 105, 106, 107, 108, 109, 1.25) + ); + assert_eq!(rows[1].date, "2026-01-15"); + assert!(rows[1].updated_at > seeded_updated_at); + assert_eq!(rows[1].updated_at, expected_updated_at); + + assert!(!rows[2].id.is_empty()); + assert_eq!(rows[2].team_id.as_deref(), Some(team_id)); + assert_eq!(rows[2].api_key, "api-key-sentinel"); + assert_eq!(rows[2].model.as_deref(), Some(model)); + assert_eq!( + rows[2].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[2].endpoint.as_deref(), Some(endpoint)); + assert_eq!(rows[2].model_group.as_deref(), Some("sentinel-key-group")); + assert_eq!(rows[2].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[2].prompt_tokens, + rows[2].completion_tokens, + rows[2].api_requests, + rows[2].successful_requests, + rows[2].failed_requests, + rows[2].cache_read_input_tokens, + rows[2].cache_creation_input_tokens, + rows[2].total_response_time_ms, + rows[2].timed_requests, + rows[2].spend + ), + (201, 202, 203, 204, 205, 206, 207, 208, 209, 2.5) + ); + assert_eq!(rows[2].date, "2026-01-15"); + assert!(rows[2].updated_at > seeded_updated_at); + assert_eq!(rows[2].updated_at, expected_updated_at); + + assert!(!rows[3].id.is_empty()); + assert_eq!(rows[3].team_id.as_deref(), Some("entity-sentinel")); + assert_eq!(rows[3].api_key, api_key); + assert_eq!(rows[3].model.as_deref(), Some(model)); + assert_eq!( + rows[3].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[3].endpoint.as_deref(), Some(endpoint)); + assert_eq!( + rows[3].model_group.as_deref(), + Some("sentinel-entity-group") + ); + assert_eq!(rows[3].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[3].prompt_tokens, + rows[3].completion_tokens, + rows[3].api_requests, + rows[3].successful_requests, + rows[3].failed_requests, + rows[3].cache_read_input_tokens, + rows[3].cache_creation_input_tokens, + rows[3].total_response_time_ms, + rows[3].timed_requests, + rows[3].spend + ), + (301, 302, 303, 304, 305, 306, 307, 308, 309, 3.75) + ); + assert_eq!(rows[3].date, "2026-01-15"); + assert!(rows[3].updated_at > seeded_updated_at); + assert_eq!(rows[3].updated_at, expected_updated_at); + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/spend/mod.rs b/litellm-rust/crates/db/src/spend/mod.rs index f50440e06de..3f71ae4bf9f 100644 --- a/litellm-rust/crates/db/src/spend/mod.rs +++ b/litellm-rust/crates/db/src/spend/mod.rs @@ -1,3 +1,4 @@ +mod daily_team_spend; mod daily_user_spend; mod end_user_spend; mod key_spend; @@ -7,6 +8,7 @@ mod team_member_spend; mod team_spend; mod user_spend; +pub use daily_team_spend::upsert_daily_team_spend; pub use daily_user_spend::{DailySpend, upsert_daily_user_spend}; pub use end_user_spend::add_end_user_spend; pub use key_spend::{EntitySpend, add_key_spend}; From a783f8e78b87b3c61e834725592c665f7124f7ac Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 26 Oct 2026 17:05:08 -0700 Subject: [PATCH 097/155] feat(db): add typed daily end-user spend upsert (#48237) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/Cargo.lock | 1 + ...3bcab69ff6ad3555a3926537586f5d81d933d.json | 31 + litellm-rust/crates/db/Cargo.toml | 1 + .../spend/upsert_daily_end_user_spend.sql | 40 ++ litellm-rust/crates/db/src/lib.rs | 7 +- .../db/src/spend/daily_end_user_spend.rs | 584 ++++++++++++++++++ litellm-rust/crates/db/src/spend/mod.rs | 2 + 7 files changed, 663 insertions(+), 3 deletions(-) create mode 100644 litellm-rust/crates/db/.sqlx/query-f207ff0f5310dea0833361d55993bcab69ff6ad3555a3926537586f5d81d933d.json create mode 100644 litellm-rust/crates/db/queries/spend/upsert_daily_end_user_spend.sql create mode 100644 litellm-rust/crates/db/src/spend/daily_end_user_spend.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index d0757ebbd2c..32ae431f863 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3701,6 +3701,7 @@ dependencies = [ "serde", "serde_json", "sqlx", + "strum", "thiserror 2.0.19", "tokio", ] diff --git a/litellm-rust/crates/db/.sqlx/query-f207ff0f5310dea0833361d55993bcab69ff6ad3555a3926537586f5d81d933d.json b/litellm-rust/crates/db/.sqlx/query-f207ff0f5310dea0833361d55993bcab69ff6ad3555a3926537586f5d81d933d.json new file mode 100644 index 00000000000..b2635c549ac --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-f207ff0f5310dea0833361d55993bcab69ff6ad3555a3926537586f5d81d933d.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO \"LiteLLM_DailyEndUserSpend\" (\n id, end_user_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,\n completion_tokens, api_requests, successful_requests, failed_requests,\n cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,\n timed_requests, spend, updated_at\n)\nSELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,\n r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,\n r.completion_tokens, r.api_requests, r.successful_requests,\n r.failed_requests, r.cache_read, r.cache_creation, r.response_time,\n r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')\nFROM UNNEST(\n $1::text[], $2::text[], $3::text[], $4::text[], $5::text[],\n $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],\n $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],\n $16::int8[], $17::float8[], $18::text[]\n) AS r(\n entity_id, date, api_key, model, custom_llm_provider, endpoint,\n model_group, prompt_tokens, completion_tokens, api_requests,\n successful_requests, failed_requests, cache_read, cache_creation,\n response_time, timed_requests, spend, mcp_namespaced_tool_name\n)\nORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,\n r.mcp_namespaced_tool_name, r.endpoint\nON CONFLICT (\n end_user_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint\n) DO UPDATE SET\n prompt_tokens = \"LiteLLM_DailyEndUserSpend\".prompt_tokens + EXCLUDED.prompt_tokens,\n completion_tokens = \"LiteLLM_DailyEndUserSpend\".completion_tokens + EXCLUDED.completion_tokens,\n api_requests = \"LiteLLM_DailyEndUserSpend\".api_requests + EXCLUDED.api_requests,\n successful_requests = \"LiteLLM_DailyEndUserSpend\".successful_requests + EXCLUDED.successful_requests,\n failed_requests = \"LiteLLM_DailyEndUserSpend\".failed_requests + EXCLUDED.failed_requests,\n cache_read_input_tokens = \"LiteLLM_DailyEndUserSpend\".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,\n cache_creation_input_tokens = \"LiteLLM_DailyEndUserSpend\".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,\n total_response_time_ms = \"LiteLLM_DailyEndUserSpend\".total_response_time_ms + EXCLUDED.total_response_time_ms,\n timed_requests = \"LiteLLM_DailyEndUserSpend\".timed_requests + EXCLUDED.timed_requests,\n spend = \"LiteLLM_DailyEndUserSpend\".spend + EXCLUDED.spend,\n updated_at = (NOW() AT TIME ZONE 'UTC')\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Float8Array", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "f207ff0f5310dea0833361d55993bcab69ff6ad3555a3926537586f5d81d933d" +} diff --git a/litellm-rust/crates/db/Cargo.toml b/litellm-rust/crates/db/Cargo.toml index 8c1a35c7535..4f8fe4e28e2 100644 --- a/litellm-rust/crates/db/Cargo.toml +++ b/litellm-rust/crates/db/Cargo.toml @@ -11,6 +11,7 @@ macro_rules_attribute.workspace = true serde.workspace = true serde_json.workspace = true sqlx.workspace = true +strum.workspace = true thiserror.workspace = true [dev-dependencies] diff --git a/litellm-rust/crates/db/queries/spend/upsert_daily_end_user_spend.sql b/litellm-rust/crates/db/queries/spend/upsert_daily_end_user_spend.sql new file mode 100644 index 00000000000..8c4936e1a7a --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/upsert_daily_end_user_spend.sql @@ -0,0 +1,40 @@ +INSERT INTO "LiteLLM_DailyEndUserSpend" ( + id, end_user_id, date, api_key, model, custom_llm_provider, + mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens, + completion_tokens, api_requests, successful_requests, failed_requests, + cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms, + timed_requests, spend, updated_at +) +SELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model, + r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens, + r.completion_tokens, r.api_requests, r.successful_requests, + r.failed_requests, r.cache_read, r.cache_creation, r.response_time, + r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC') +FROM UNNEST( + $1::text[], $2::text[], $3::text[], $4::text[], $5::text[], + $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[], + $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[], + $16::int8[], $17::float8[], $18::text[] +) AS r( + entity_id, date, api_key, model, custom_llm_provider, endpoint, + model_group, prompt_tokens, completion_tokens, api_requests, + successful_requests, failed_requests, cache_read, cache_creation, + response_time, timed_requests, spend, mcp_namespaced_tool_name +) +ORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider, + r.mcp_namespaced_tool_name, r.endpoint +ON CONFLICT ( + end_user_id, date, api_key, model, custom_llm_provider, + mcp_namespaced_tool_name, endpoint +) DO UPDATE SET + prompt_tokens = "LiteLLM_DailyEndUserSpend".prompt_tokens + EXCLUDED.prompt_tokens, + completion_tokens = "LiteLLM_DailyEndUserSpend".completion_tokens + EXCLUDED.completion_tokens, + api_requests = "LiteLLM_DailyEndUserSpend".api_requests + EXCLUDED.api_requests, + successful_requests = "LiteLLM_DailyEndUserSpend".successful_requests + EXCLUDED.successful_requests, + failed_requests = "LiteLLM_DailyEndUserSpend".failed_requests + EXCLUDED.failed_requests, + cache_read_input_tokens = "LiteLLM_DailyEndUserSpend".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens, + cache_creation_input_tokens = "LiteLLM_DailyEndUserSpend".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens, + total_response_time_ms = "LiteLLM_DailyEndUserSpend".total_response_time_ms + EXCLUDED.total_response_time_ms, + timed_requests = "LiteLLM_DailyEndUserSpend".timed_requests + EXCLUDED.timed_requests, + spend = "LiteLLM_DailyEndUserSpend".spend + EXCLUDED.spend, + updated_at = (NOW() AT TIME ZONE 'UTC') diff --git a/litellm-rust/crates/db/src/lib.rs b/litellm-rust/crates/db/src/lib.rs index 5b88bcabf32..541f97c7db9 100644 --- a/litellm-rust/crates/db/src/lib.rs +++ b/litellm-rust/crates/db/src/lib.rs @@ -19,7 +19,8 @@ pub use end_user::{EndUser, RateLimits, find_end_user}; pub use error::Error; pub use failure::DbFailure; pub use spend::{ - DailySpend, EntitySpend, SpendLog, SpendLogRow, TeamMemberSpend, add_end_user_spend, - add_key_spend, add_team_member_spend, add_team_spend, add_user_spend, begin, insert_spend_logs, - lock_team, upsert_daily_team_spend, upsert_daily_user_spend, + DailySpend, DailyTable, EntitySpend, SpendLog, SpendLogRow, TeamMemberSpend, + add_end_user_spend, add_key_spend, add_team_member_spend, add_team_spend, add_user_spend, + begin, insert_spend_logs, lock_team, upsert_daily_end_user_spend, upsert_daily_spend, + upsert_daily_team_spend, upsert_daily_user_spend, }; diff --git a/litellm-rust/crates/db/src/spend/daily_end_user_spend.rs b/litellm-rust/crates/db/src/spend/daily_end_user_spend.rs new file mode 100644 index 00000000000..6c88d47cdcc --- /dev/null +++ b/litellm-rust/crates/db/src/spend/daily_end_user_spend.rs @@ -0,0 +1,584 @@ +use sqlx::PgExecutor; + +use super::{DailySpend, upsert_daily_team_spend, upsert_daily_user_spend}; +use crate::Error; + +pub async fn upsert_daily_end_user_spend( + executor: impl PgExecutor<'_>, + rows: &[DailySpend], +) -> Result<(), Error> { + if rows.is_empty() { + return Ok(()); + } + macro_rules! column { + ($field:ident) => { + rows.iter() + .map(|row| row.$field.clone()) + .collect::>() + }; + } + sqlx::query_file!( + "queries/spend/upsert_daily_end_user_spend.sql", + &column!(entity_id), + &column!(date), + &column!(api_key), + &column!(model), + &column!(custom_llm_provider), + &column!(endpoint), + &column!(model_group) as &[Option], + &column!(prompt_tokens), + &column!(completion_tokens), + &column!(api_requests), + &column!(successful_requests), + &column!(failed_requests), + &column!(cache_read_input_tokens), + &column!(cache_creation_input_tokens), + &column!(total_response_time_ms), + &column!(timed_requests), + &column!(spend), + &column!(mcp_namespaced_tool_name), + ) + .execute(executor) + .await?; + Ok(()) +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, strum::VariantArray)] +pub enum DailyTable { + User, + Team, + EndUser, +} + +pub async fn upsert_daily_spend( + executor: impl PgExecutor<'_>, + table: DailyTable, + rows: &[DailySpend], +) -> Result<(), Error> { + match table { + DailyTable::User => upsert_daily_user_spend(executor, rows).await, + DailyTable::Team => upsert_daily_team_spend(executor, rows).await, + DailyTable::EndUser => upsert_daily_end_user_spend(executor, rows).await, + } +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{DailySpend, DailyTable, upsert_daily_end_user_spend, upsert_daily_spend}; + use crate::{DailyEndUserRow, test_support::database_pool}; + use rstest::rstest; + use sqlx::{PgPool, types::chrono::NaiveDateTime}; + + #[rstest] + #[case::full_keys( + "daily-end-user-base", + "daily-key-base", + "daily-model", + "daily-provider", + "/chat/completions", + Some("first-group") + )] + #[case::initial_model_group_none( + "daily-end-user-base", + "daily-key-base", + "daily-model", + "daily-provider", + "/chat/completions", + None + )] + #[case::empty_optional_keys("", "", "", "", "", Some(""))] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn daily_end_user_spend_upsert_preserves_keys_and_accumulates( + #[case] end_user_id: &'static str, + #[case] api_key: &'static str, + #[case] model: &'static str, + #[case] custom_llm_provider: &'static str, + #[case] endpoint: &'static str, + #[case] first_model_group: Option<&'static str>, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test exercises UTC timestamp behavior under a non-UTC session" + )] + sqlx::query("SET LOCAL TIME ZONE 'America/Los_Angeles'") + .execute(&mut *tx) + .await?; + + let first_batch = [ + DailySpend { + entity_id: end_user_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: first_model_group.map(str::to_owned), + prompt_tokens: 1, + completion_tokens: 2, + api_requests: 3, + successful_requests: 4, + failed_requests: 5, + cache_read_input_tokens: 6, + cache_creation_input_tokens: 7, + total_response_time_ms: 8, + timed_requests: 9, + spend: 0.125, + }, + DailySpend { + entity_id: end_user_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: "endpoint-sentinel".to_owned(), + model_group: Some("sentinel-endpoint-group".to_owned()), + prompt_tokens: 101, + completion_tokens: 102, + api_requests: 103, + successful_requests: 104, + failed_requests: 105, + cache_read_input_tokens: 106, + cache_creation_input_tokens: 107, + total_response_time_ms: 108, + timed_requests: 109, + spend: 1.25, + }, + DailySpend { + entity_id: end_user_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: "api-key-sentinel".to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("sentinel-key-group".to_owned()), + prompt_tokens: 201, + completion_tokens: 202, + api_requests: 203, + successful_requests: 204, + failed_requests: 205, + cache_read_input_tokens: 206, + cache_creation_input_tokens: 207, + total_response_time_ms: 208, + timed_requests: 209, + spend: 2.5, + }, + DailySpend { + entity_id: "entity-sentinel".to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("sentinel-entity-group".to_owned()), + prompt_tokens: 301, + completion_tokens: 302, + api_requests: 303, + successful_requests: 304, + failed_requests: 305, + cache_read_input_tokens: 306, + cache_creation_input_tokens: 307, + total_response_time_ms: 308, + timed_requests: 309, + spend: 3.75, + }, + ]; + upsert_daily_end_user_spend(&mut *tx, &first_batch).await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test captures the inserted row identity before conflict handling" + )] + let (initial_id, initial_created_at): (String, NaiveDateTime) = sqlx::query_as( + r#"SELECT id, created_at + FROM "LiteLLM_DailyEndUserSpend" + WHERE end_user_id = $1 AND date = $2 AND api_key = $3 AND model = $4 + AND custom_llm_provider = $5 AND endpoint = $6"#, + ) + .bind(end_user_id) + .bind("2026-01-15") + .bind(api_key) + .bind(model) + .bind(custom_llm_provider) + .bind(endpoint) + .fetch_one(&mut *tx) + .await?; + assert!(!initial_id.is_empty()); + + #[expect( + clippy::disallowed_methods, + reason = "The integration test seeds updated_at before exercising the conflict update" + )] + let seeded_updated_at: NaiveDateTime = sqlx::query_scalar( + r#"UPDATE "LiteLLM_DailyEndUserSpend" + SET updated_at = TIMESTAMP '2000-01-01' + WHERE id = $1 + RETURNING updated_at"#, + ) + .bind(&initial_id) + .fetch_one(&mut *tx) + .await?; + + let increment = [DailySpend { + entity_id: end_user_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("second-group".to_owned()), + prompt_tokens: 10, + completion_tokens: 20, + api_requests: 30, + successful_requests: 40, + failed_requests: 50, + cache_read_input_tokens: 60, + cache_creation_input_tokens: 70, + total_response_time_ms: 80, + timed_requests: 90, + spend: 0.625, + }]; + upsert_daily_end_user_spend(&mut *tx, &increment).await?; + upsert_daily_end_user_spend(&mut *tx, &[]).await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test captures the transaction timestamp for assertions" + )] + let expected_updated_at: NaiveDateTime = + sqlx::query_scalar("SELECT (NOW() AT TIME ZONE 'UTC')::timestamp(3)") + .fetch_one(&mut *tx) + .await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test reads all rows to verify attribution and accumulation" + )] + let rows: Vec = sqlx::query_as( + r#"SELECT id, end_user_id, api_key, model, custom_llm_provider, endpoint, + model_group, mcp_namespaced_tool_name, prompt_tokens, + completion_tokens, api_requests, successful_requests, + failed_requests, cache_read_input_tokens, + cache_creation_input_tokens, total_response_time_ms, + timed_requests, spend, date, created_at, updated_at + FROM "LiteLLM_DailyEndUserSpend" + WHERE date = $1 AND model = $2 AND custom_llm_provider = $3 + AND mcp_namespaced_tool_name = '' AND end_user_id = ANY($4) + ORDER BY CASE + WHEN end_user_id = $5 AND api_key = $6 AND endpoint = $7 THEN 0 + WHEN end_user_id = $5 AND api_key = $6 AND endpoint = $9 THEN 1 + WHEN end_user_id = $5 AND api_key = $8 AND endpoint = $7 THEN 2 + WHEN end_user_id = $10 AND api_key = $6 AND endpoint = $7 THEN 3 + ELSE 4 + END"#, + ) + .bind("2026-01-15") + .bind(model) + .bind(custom_llm_provider) + .bind(vec![end_user_id, "entity-sentinel"]) + .bind(end_user_id) + .bind(api_key) + .bind(endpoint) + .bind("api-key-sentinel") + .bind("endpoint-sentinel") + .bind("entity-sentinel") + .fetch_all(&mut *tx) + .await?; + + assert_eq!(rows.len(), 4); + assert_eq!(rows[0].id, initial_id); + assert_eq!(rows[0].end_user_id.as_deref(), Some(end_user_id)); + assert_eq!(rows[0].api_key, api_key); + assert_eq!(rows[0].model.as_deref(), Some(model)); + assert_eq!( + rows[0].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[0].endpoint.as_deref(), Some(endpoint)); + assert_eq!(rows[0].model_group.as_deref(), first_model_group); + assert_eq!(rows[0].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[0].prompt_tokens, + rows[0].completion_tokens, + rows[0].api_requests, + rows[0].successful_requests, + rows[0].failed_requests, + rows[0].cache_read_input_tokens, + rows[0].cache_creation_input_tokens, + rows[0].total_response_time_ms, + rows[0].timed_requests, + rows[0].spend + ), + (11, 22, 33, 44, 55, 66, 77, 88, 99, 0.75) + ); + assert_eq!(rows[0].date, "2026-01-15"); + assert_eq!(rows[0].created_at, initial_created_at); + assert!(rows[0].updated_at > seeded_updated_at); + assert_eq!(rows[0].updated_at, expected_updated_at); + + assert!(!rows[1].id.is_empty()); + assert_eq!(rows[1].end_user_id.as_deref(), Some(end_user_id)); + assert_eq!(rows[1].api_key, api_key); + assert_eq!(rows[1].model.as_deref(), Some(model)); + assert_eq!( + rows[1].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[1].endpoint.as_deref(), Some("endpoint-sentinel")); + assert_eq!( + rows[1].model_group.as_deref(), + Some("sentinel-endpoint-group") + ); + assert_eq!(rows[1].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[1].prompt_tokens, + rows[1].completion_tokens, + rows[1].api_requests, + rows[1].successful_requests, + rows[1].failed_requests, + rows[1].cache_read_input_tokens, + rows[1].cache_creation_input_tokens, + rows[1].total_response_time_ms, + rows[1].timed_requests, + rows[1].spend + ), + (101, 102, 103, 104, 105, 106, 107, 108, 109, 1.25) + ); + assert_eq!(rows[1].date, "2026-01-15"); + assert!(rows[1].updated_at > seeded_updated_at); + assert_eq!(rows[1].updated_at, expected_updated_at); + + assert!(!rows[2].id.is_empty()); + assert_eq!(rows[2].end_user_id.as_deref(), Some(end_user_id)); + assert_eq!(rows[2].api_key, "api-key-sentinel"); + assert_eq!(rows[2].model.as_deref(), Some(model)); + assert_eq!( + rows[2].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[2].endpoint.as_deref(), Some(endpoint)); + assert_eq!(rows[2].model_group.as_deref(), Some("sentinel-key-group")); + assert_eq!(rows[2].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[2].prompt_tokens, + rows[2].completion_tokens, + rows[2].api_requests, + rows[2].successful_requests, + rows[2].failed_requests, + rows[2].cache_read_input_tokens, + rows[2].cache_creation_input_tokens, + rows[2].total_response_time_ms, + rows[2].timed_requests, + rows[2].spend + ), + (201, 202, 203, 204, 205, 206, 207, 208, 209, 2.5) + ); + assert_eq!(rows[2].date, "2026-01-15"); + assert!(rows[2].updated_at > seeded_updated_at); + assert_eq!(rows[2].updated_at, expected_updated_at); + + assert!(!rows[3].id.is_empty()); + assert_eq!(rows[3].end_user_id.as_deref(), Some("entity-sentinel")); + assert_eq!(rows[3].api_key, api_key); + assert_eq!(rows[3].model.as_deref(), Some(model)); + assert_eq!( + rows[3].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[3].endpoint.as_deref(), Some(endpoint)); + assert_eq!( + rows[3].model_group.as_deref(), + Some("sentinel-entity-group") + ); + assert_eq!(rows[3].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[3].prompt_tokens, + rows[3].completion_tokens, + rows[3].api_requests, + rows[3].successful_requests, + rows[3].failed_requests, + rows[3].cache_read_input_tokens, + rows[3].cache_creation_input_tokens, + rows[3].total_response_time_ms, + rows[3].timed_requests, + rows[3].spend + ), + (301, 302, 303, 304, 305, 306, 307, 308, 309, 3.75) + ); + assert_eq!(rows[3].date, "2026-01-15"); + assert!(rows[3].updated_at > seeded_updated_at); + assert_eq!(rows[3].updated_at, expected_updated_at); + Ok(()) + } + + #[rstest] + #[case::distinct_tools("server/tool-a", "server/tool-b")] + #[case::empty_and_named_tool("", "server/tool-b")] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn daily_end_user_spend_preserves_tool_attribution( + #[case] first_tool: &str, + #[case] second_tool: &str, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + let first = DailySpend { + entity_id: "daily-end-user-tools".to_owned(), + date: "2026-02-21".to_owned(), + api_key: "tool-key".to_owned(), + model: "tool-model".to_owned(), + custom_llm_provider: "tool-provider".to_owned(), + endpoint: "/mcp".to_owned(), + mcp_namespaced_tool_name: first_tool.to_owned(), + prompt_tokens: 2, + spend: 0.25, + ..Default::default() + }; + let second = DailySpend { + mcp_namespaced_tool_name: second_tool.to_owned(), + prompt_tokens: 5, + spend: 0.5, + ..first.clone() + }; + upsert_daily_end_user_spend(&mut *tx, &[second, first.clone()]).await?; + upsert_daily_end_user_spend(&mut *tx, &[first]).await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test reads stored tool attribution and counters" + )] + let rows: Vec = sqlx::query_as( + r#"SELECT * FROM "LiteLLM_DailyEndUserSpend" + WHERE end_user_id = $1 AND date = $2 AND api_key = $3 + ORDER BY mcp_namespaced_tool_name"#, + ) + .bind("daily-end-user-tools") + .bind("2026-02-21") + .bind("tool-key") + .fetch_all(&mut *tx) + .await?; + + assert_eq!(rows.len(), 2); + assert_eq!( + rows[0].mcp_namespaced_tool_name.as_deref(), + Some(first_tool) + ); + assert_eq!( + rows[1].mcp_namespaced_tool_name.as_deref(), + Some(second_tool) + ); + assert_eq!((rows[0].prompt_tokens, rows[0].spend), (4, 0.5)); + assert_eq!((rows[1].prompt_tokens, rows[1].spend), (5, 0.5)); + assert_ne!(rows[0].id, rows[1].id); + Ok(()) + } + + #[rstest] + #[case::user(DailyTable::User, "daily-dispatch-user")] + #[case::team(DailyTable::Team, "daily-dispatch-team")] + #[case::end_user(DailyTable::EndUser, "daily-dispatch-end-user")] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn daily_spend_dispatches_to_selected_table( + #[case] table: DailyTable, + #[case] entity_id: &'static str, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + let api_key = format!("{entity_id}-key"); + let rows = [DailySpend { + entity_id: entity_id.to_owned(), + date: "2026-02-20".to_owned(), + api_key: api_key.clone(), + model: "dispatcher-model".to_owned(), + custom_llm_provider: "dispatcher-provider".to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: "/dispatcher".to_owned(), + model_group: Some("dispatcher-group".to_owned()), + prompt_tokens: 1, + completion_tokens: 2, + api_requests: 3, + successful_requests: 4, + failed_requests: 5, + cache_read_input_tokens: 6, + cache_creation_input_tokens: 7, + total_response_time_ms: 8, + timed_requests: 9, + spend: 0.125, + }]; + upsert_daily_spend(&mut *tx, table, &rows).await?; + + let count: i64 = match table { + DailyTable::User => + { + #[expect( + clippy::disallowed_methods, + reason = "The integration test checks the user-table dispatcher route" + )] + sqlx::query_scalar( + r#"SELECT COUNT(*) FROM "LiteLLM_DailyUserSpend" + WHERE user_id = $1 AND date = $2 AND api_key = $3 AND model = $4"#, + ) + .bind(entity_id) + .bind("2026-02-20") + .bind(&api_key) + .bind("dispatcher-model") + .fetch_one(&mut *tx) + .await? + } + DailyTable::Team => + { + #[expect( + clippy::disallowed_methods, + reason = "The integration test checks the team-table dispatcher route" + )] + sqlx::query_scalar( + r#"SELECT COUNT(*) FROM "LiteLLM_DailyTeamSpend" + WHERE team_id = $1 AND date = $2 AND api_key = $3 AND model = $4"#, + ) + .bind(entity_id) + .bind("2026-02-20") + .bind(&api_key) + .bind("dispatcher-model") + .fetch_one(&mut *tx) + .await? + } + DailyTable::EndUser => + { + #[expect( + clippy::disallowed_methods, + reason = "The integration test checks the end-user-table dispatcher route" + )] + sqlx::query_scalar( + r#"SELECT COUNT(*) FROM "LiteLLM_DailyEndUserSpend" + WHERE end_user_id = $1 AND date = $2 AND api_key = $3 AND model = $4"#, + ) + .bind(entity_id) + .bind("2026-02-20") + .bind(&api_key) + .bind("dispatcher-model") + .fetch_one(&mut *tx) + .await? + } + }; + assert_eq!(count, 1); + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/spend/mod.rs b/litellm-rust/crates/db/src/spend/mod.rs index 3f71ae4bf9f..da255843547 100644 --- a/litellm-rust/crates/db/src/spend/mod.rs +++ b/litellm-rust/crates/db/src/spend/mod.rs @@ -1,3 +1,4 @@ +mod daily_end_user_spend; mod daily_team_spend; mod daily_user_spend; mod end_user_spend; @@ -8,6 +9,7 @@ mod team_member_spend; mod team_spend; mod user_spend; +pub use daily_end_user_spend::{DailyTable, upsert_daily_end_user_spend, upsert_daily_spend}; pub use daily_team_spend::upsert_daily_team_spend; pub use daily_user_spend::{DailySpend, upsert_daily_user_spend}; pub use end_user_spend::add_end_user_spend; From b9825e1f87aecd13c87f53851f3fd0d4cb16bfef Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 26 Oct 2026 17:05:08 -0700 Subject: [PATCH 098/155] docs: document incremental database access migration (#48476) * docs(rust): document incremental database access migration * docs(rust): organize database guidance by task * docs: move database access skill to repository root --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .agents/skills/database-access/SKILL.md | 52 +++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .agents/skills/database-access/SKILL.md diff --git a/.agents/skills/database-access/SKILL.md b/.agents/skills/database-access/SKILL.md new file mode 100644 index 00000000000..796dfafe4f6 --- /dev/null +++ b/.agents/skills/database-access/SKILL.md @@ -0,0 +1,52 @@ +--- +name: database-access +description: Add database queries, extract existing Prisma calls into repositories, or explicitly port queries to Rust and SQLx without expanding migration scope +--- + +# Database access + +## Adding new database access + +- Put Python database access behind an operation in `litellm/repositories` +- Give callers typed inputs and outputs, not Prisma clients, models, or query dictionaries +- Keep query execution and backend selection inside the repository +- Prisma is fine for the implementation, adding a query does not require a Rust port +- Native Rust callers use `litellm-rust/crates/db` directly + +## Asked to move existing access behind a repository + +- Identify the calls being moved and their transaction boundary +- Move the existing Prisma implementation into the relevant repository +- Replace those direct database calls with repository operations +- Preserve return values, errors, and transaction behavior, with focused regression tests +- Keep this an extraction, not a backend migration or unrelated cleanup + +## An equivalent Rust or SQLx query already exists + +- Leave it and the current backend selection unchanged unless integration is explicitly requested +- Its existence alone does not justify switching a Prisma caller to Rust +- Repository extraction can keep using Prisma even when the Rust query is available + +## Asked to port a Prisma query to Rust or SQLx + +- Identify the existing inputs, outputs, errors, and transaction boundary +- Check `litellm-rust/crates/db` for an equivalent query and reuse it if its behavior matches +- Otherwise follow `litellm-rust/crates/db/AGENTS.md` + - Put static SQL in `queries/` and compile-check it with SQLx + - Keep typed inputs and reusable row projections in `src/` + - Refresh SQLx query metadata after SQL changes +- Keep Python conversion in the Python bridge, not in `litellm-rust/crates/db` +- When caller integration is requested, expose the Rust operation through the bridge and call it from the repository + - A query-only port can land without switching callers +- Verify behavior against real PostgreSQL before switching callers + - Check relevant null handling, conflicts, ordering, errors, and transaction semantics +- Remove the old implementation only when no callers need it + +## Migration context and boundaries + +- Long-term direction is to centralize Python database access in repositories, then replace their Prisma implementations incrementally with Rust APIs +- Mixed Prisma and Rust implementations are expected during the transition, unrelated existing calls can stay as they are +- Both backends may have separate pools through PgBouncer, size them together and verify pooling-mode compatibility +- PgBouncer does not share transactions between Prisma and SQLx, keep each atomic unit of work on one backend and transaction +- Never retry a failed Rust write through Prisma when it may already have committed +- Prisma owns schema migrations today, eventual SQLx migration ownership and Prisma removal are separate planned steps, not part of a query port From 1263f70ab63dc662ce043687a85cece965d76c46 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:05:37 -0700 Subject: [PATCH 099/155] feat(ui): budget period tooltip on key, team, user and org spend tiles (#48487) * feat(ui): budget period tooltip on key, team, user and org spend tiles Adds a shared BudgetPeriodTooltip to the four detail-view spend tiles showing the current budget period, max budget, reset interval and spend so far, and makes organization create schedule budget_reset_at and organization info return it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): give the budget period tooltip trigger its own aria label The default SimpleTooltip trigger shares the question-circle label with InheritedBudgetHint, which broke the key budget display tests that count that icon Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): match the proxy's budget window rules in the period tooltip 30d and Nmo now span calendar months like get_budget_window_start, the arithmetic runs in UTC so a viewer's DST change cannot shift the start, the hourly/daily/weekly/monthly aliases the proxy accepts are normalized before parsing, and spend without a duration reads as lifetime spend Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: joshua Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 2 +- .../organization_endpoints.py | 3 +- .../test_organization_endpoints.py | 65 +++++++++++++++ .../_components/view_users/user_info_view.tsx | 11 ++- .../organization/organization_view.tsx | 11 ++- .../shared/BudgetPeriodTooltip.test.tsx | 83 +++++++++++++++++++ .../components/shared/BudgetPeriodTooltip.tsx | 65 +++++++++++++++ .../src/components/team/TeamInfo.tsx | 11 ++- .../components/templates/key_info_view.tsx | 11 ++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- .../src/utils/budgetUtils.test.ts | 40 ++++++++- ui/litellm-dashboard/src/utils/budgetUtils.ts | 41 +++++++++ 12 files changed, 336 insertions(+), 9 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/shared/BudgetPeriodTooltip.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/BudgetPeriodTooltip.tsx diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 6280e2caf87..49d23d82aa9 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3277,7 +3277,7 @@ class LiteLLM_OrganizationTableWithMembers(LiteLLM_OrganizationTable): members: list[LiteLLM_OrganizationMembershipTable] = [] teams: list[LiteLLM_TeamTable] = [] - litellm_budget_table: LiteLLM_BudgetTable | None = None + litellm_budget_table: LiteLLM_BudgetTableFull | None = None created_at: datetime updated_at: datetime diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py index d7a44cdd0b8..e1d19d8efae 100644 --- a/litellm/proxy/management_endpoints/organization_endpoints.py +++ b/litellm/proxy/management_endpoints/organization_endpoints.py @@ -504,9 +504,8 @@ async def new_organization( _budget: Final = await _table(BudgetRepository(prisma_client)).create( data={ - **new_budget, + **build_budget_write_data(new_budget, user_api_key_dict.user_id or litellm_proxy_admin_name), "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name, - "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name, } ) diff --git a/tests/unit/proxy/management_endpoints/test_organization_endpoints.py b/tests/unit/proxy/management_endpoints/test_organization_endpoints.py index a5cae02ea10..16c59aa20b9 100644 --- a/tests/unit/proxy/management_endpoints/test_organization_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_organization_endpoints.py @@ -1615,3 +1615,68 @@ async def test_delete_organization_is_refused_while_one_of_its_teams_holds_a_ptu prisma_client.db.litellm_teamtable.delete_many.assert_not_called() prisma_client.db.litellm_verificationtoken.delete_many.assert_not_called() prisma_client.db.litellm_organizationtable.delete.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("budget_duration", ["30d", None]) +async def test_new_organization_schedules_budget_reset_from_duration(monkeypatch, budget_duration): + """/organization/new with an inline budget_duration must write budget_reset_at on the auto-created + budget row, the way team create and /budget/new do, or the org budget never resets and the UI + cannot show the current budget period.""" + from datetime import datetime + + from litellm.proxy._types import LitellmUserRoles, NewOrganizationRequest, UserAPIKeyAuth + from litellm.proxy.management_endpoints.organization_endpoints import new_organization + from litellm.proxy.utils import PrismaClient + + prisma_client = MagicMock() + prisma_client.jsonify_object = MagicMock(side_effect=lambda data: PrismaClient.jsonify_object(prisma_client, data)) + prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None) + prisma_client.db.litellm_budgettable.create = AsyncMock(return_value=MagicMock(budget_id="budget-1")) + prisma_client.db.litellm_organizationtable.create = AsyncMock(return_value={"organization_id": "org-1"}) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", MagicMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True, raising=False) + + await new_organization( + data=NewOrganizationRequest(organization_alias="org", max_budget=100, budget_duration=budget_duration), + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + budget_write = prisma_client.db.litellm_budgettable.create.await_args.kwargs["data"] + if budget_duration is None: + assert "budget_reset_at" not in budget_write, budget_write + return + reset_at = budget_write["budget_reset_at"] + assert isinstance(reset_at, datetime) and reset_at.tzinfo is not None, budget_write + assert budget_write["budget_duration"] == budget_duration + + +def test_organization_info_response_keeps_budget_reset_at(): + """/organization/info must return the budget row's budget_reset_at so the dashboard can show + which budget period the org's spend covers.""" + from datetime import datetime, timezone + + from litellm.proxy._types import LiteLLM_OrganizationTableWithMembers + + reset_at = datetime(2026, 11, 1, tzinfo=timezone.utc) + created_at = datetime(2026, 10, 1, tzinfo=timezone.utc) + response = LiteLLM_OrganizationTableWithMembers( + organization_id="org-1", + organization_alias="org", + budget_id="budget-1", + models=[], + created_by="admin", + updated_by="admin", + created_at=created_at, + updated_at=created_at, + litellm_budget_table={ + "budget_id": "budget-1", + "max_budget": 100.0, + "budget_duration": "30d", + "budget_reset_at": reset_at, + "created_at": created_at, + }, + ) + + assert response.model_dump()["litellm_budget_table"]["budget_reset_at"] == reset_at diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx index 2056142b50a..476c043a258 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx @@ -47,6 +47,7 @@ import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/u import ScopedSavingsTab from "@/components/shared/ScopedSavingsTab"; import { AutoRouterUsageView } from "@/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab"; import { useActivityDateRange } from "@/app/(dashboard)/cost-optimization/_components/useDailyActivityRange"; +import { BudgetPeriodTooltip } from "@/components/shared/BudgetPeriodTooltip"; interface UserInfoViewProps { userId: string; @@ -507,7 +508,15 @@ export default function UserInfoView({
-

Spend

+

+ Spend + +

${formatNumberWithCommas(userData.spend || 0, 2)}

diff --git a/ui/litellm-dashboard/src/components/organization/organization_view.tsx b/ui/litellm-dashboard/src/components/organization/organization_view.tsx index f325e92d2b5..2be549261c3 100644 --- a/ui/litellm-dashboard/src/components/organization/organization_view.tsx +++ b/ui/litellm-dashboard/src/components/organization/organization_view.tsx @@ -27,6 +27,7 @@ import ObjectPermissionsView from "../object_permissions_view"; import MemberModal from "../team/EditMembership"; import { OrgSettingsForm } from "./org-settings/OrgSettingsForm"; import { ORGANIZATION_TAB_URL_KEY, ORGANIZATION_TABS, type OrganizationTab } from "./organizationTabs"; +import { BudgetPeriodTooltip } from "@/components/shared/BudgetPeriodTooltip"; interface OrganizationInfoProps { organizationId: string; @@ -193,7 +194,15 @@ const OrganizationInfoView: React.FC = ({ -

Budget Status

+

+ Budget Status + +

${formatNumberWithCommas(orgData.spend, 4)}

diff --git a/ui/litellm-dashboard/src/components/shared/BudgetPeriodTooltip.test.tsx b/ui/litellm-dashboard/src/components/shared/BudgetPeriodTooltip.test.tsx new file mode 100644 index 00000000000..4d77218a776 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/BudgetPeriodTooltip.test.tsx @@ -0,0 +1,83 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, describe, expect, it } from "vitest"; + +import { BudgetPeriodTooltip, budgetPeriodLines } from "./BudgetPeriodTooltip"; + +const originalTimezone: string | undefined = process.env.TZ; + +afterEach(() => { + if (originalTimezone === undefined) { + delete process.env.TZ; + return; + } + process.env.TZ = originalTimezone; +}); + +const scheduled = { spend: 12.5, maxBudget: 100, budgetDuration: "30d", budgetResetAt: "2026-04-01T00:00:00Z" }; +const unparseableDuration = { ...scheduled, spend: 0, budgetDuration: "fortnight" }; +const unscheduled = { spend: 0, maxBudget: 100, budgetDuration: "30d", budgetResetAt: null }; +const lifetime = { spend: 3, maxBudget: 50, budgetDuration: null, budgetResetAt: null }; +const noBudget = { spend: null, maxBudget: null, budgetDuration: "30d", budgetResetAt: null }; +const noBudgetNoDuration = { spend: 2, maxBudget: null, budgetDuration: null, budgetResetAt: null }; + +describe("budgetPeriodLines", () => { + it("describes the current period, max budget, interval and spend when a reset is scheduled", () => { + process.env.TZ = "UTC"; + expect(budgetPeriodLines(scheduled)).toEqual([ + "Budget period: Mar 1, 2026 00:00 to Apr 1, 2026 00:00 (UTC)", + "Max budget: $100.00", + "Reset interval: monthly", + "Spend this period: $12.5000", + ]); + }); + + it("falls back to the next reset when the duration cannot be parsed", () => { + process.env.TZ = "UTC"; + expect(budgetPeriodLines(unparseableDuration)).toEqual([ + "Next reset: Apr 1, 2026 00:00 (UTC)", + "Max budget: $100.00", + "Reset interval: fortnight", + "Spend this period: $0.0000", + ]); + }); + + it("omits the period line when no reset has been scheduled yet", () => { + expect(budgetPeriodLines(unscheduled)).toEqual([ + "Max budget: $100.00", + "Reset interval: monthly", + "Spend this period: $0.0000", + ]); + }); + + it("reports a lifetime budget when there is no duration", () => { + expect(budgetPeriodLines(lifetime)).toEqual([ + "Lifetime budget, never resets", + "Max budget: $50.00", + "Spend so far: $3.0000", + ]); + }); + + it("reports no budget when max budget is unset", () => { + expect(budgetPeriodLines(noBudget)).toEqual(["No budget set", "Spend this period: $0.0000"]); + }); + + it("uses lifetime wording when there is neither a budget nor a duration", () => { + expect(budgetPeriodLines(noBudgetNoDuration)).toEqual(["No budget set", "Spend so far: $2.0000"]); + }); +}); + +describe("BudgetPeriodTooltip", () => { + it("shows the budget period details on hover", async () => { + const user = userEvent.setup(); + render(); + + await user.hover(screen.getByLabelText("budget-period-details")); + + const tooltip = await screen.findByTestId("budget-period-tooltip"); + expect(tooltip).toHaveTextContent("Max budget: $10.00"); + expect(tooltip).toHaveTextContent("Reset interval: 1d"); + expect(tooltip).toHaveTextContent("Spend this period: $1.0000"); + expect(tooltip).not.toHaveTextContent("Invalid Date"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/BudgetPeriodTooltip.tsx b/ui/litellm-dashboard/src/components/shared/BudgetPeriodTooltip.tsx new file mode 100644 index 00000000000..56c0667b2a5 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/BudgetPeriodTooltip.tsx @@ -0,0 +1,65 @@ +"use client"; + +import dayjs from "dayjs"; +import { CircleHelp } from "lucide-react"; + +import { getBudgetDurationLabel } from "@/components/common_components/budget_duration_dropdown"; +import { SimpleTooltip } from "@/components/ui/tooltip"; +import { budgetPeriodRange } from "@/utils/budgetUtils"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; + +export interface BudgetPeriodTooltipProps { + spend: number | null | undefined; + maxBudget: number | null | undefined; + budgetDuration: string | null | undefined; + budgetResetAt: string | null | undefined; +} + +const formatPeriodDate = (iso: string): string => dayjs(iso).format("MMM D, YYYY HH:mm"); + +const describePeriod = (budgetDuration: string, budgetResetAt: string | null | undefined): string | null => { + const timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone; + const period = budgetPeriodRange(budgetDuration, budgetResetAt); + if (period) { + return `Budget period: ${formatPeriodDate(period.start)} to ${formatPeriodDate(period.end)} (${timeZone})`; + } + if (budgetResetAt && dayjs(budgetResetAt).isValid()) { + return `Next reset: ${formatPeriodDate(budgetResetAt)} (${timeZone})`; + } + return null; +}; + +export const budgetPeriodLines = ({ + spend, + maxBudget, + budgetDuration, + budgetResetAt, +}: BudgetPeriodTooltipProps): readonly string[] => { + const spendLine = `${budgetDuration ? "Spend this period" : "Spend so far"}: $${formatNumberWithCommas(spend ?? 0, 4)}`; + if (maxBudget == null) return ["No budget set", spendLine]; + const maxBudgetLine = `Max budget: $${formatNumberWithCommas(maxBudget, 2)}`; + if (!budgetDuration) return ["Lifetime budget, never resets", maxBudgetLine, spendLine]; + const periodLine = describePeriod(budgetDuration, budgetResetAt); + return [ + ...(periodLine ? [periodLine] : []), + maxBudgetLine, + `Reset interval: ${getBudgetDurationLabel(budgetDuration)}`, + spendLine, + ]; +}; + +export function BudgetPeriodTooltip(props: BudgetPeriodTooltipProps) { + return ( + + {budgetPeriodLines(props).map((line) => ( + {line} + ))} +

+ } + > + + + ); +} diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index 0d63f5d0833..0d20d160283 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -135,6 +135,7 @@ import ResetMemberBudgetsDialog from "./ResetMemberBudgetsDialog"; import { customBudgetMemberUserIds, shouldPromptMemberBudgetReset } from "./memberBudgetReset"; import { useMemberBudgetReset } from "./useMemberBudgetReset"; import { fetchClient } from "@/lib/http/api"; +import { BudgetPeriodTooltip } from "@/components/shared/BudgetPeriodTooltip"; const UI_MANAGED_METADATA_KEYS: ReadonlySet = new Set([ "logging", @@ -1375,7 +1376,15 @@ const TeamInfoView: React.FC = ({ children: (
-

Budget Status

+

+ Budget Status + +

${formatNumberWithCommas(info.spend, 2)}

of {info.max_budget === null ? "Unlimited" : `$${formatNumberWithCommas(info.max_budget, 2)}`}

diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index 7b924243fb5..4680368943a 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -57,6 +57,7 @@ import { } from "../shared/InheritedBudgetHint"; import { KeyEditView } from "./key_edit_view"; import { isTeamAdminEditingMemberKey, teamAdminMemberKeyPayload } from "./teamAdminMemberKeyPayload"; +import { BudgetPeriodTooltip } from "@/components/shared/BudgetPeriodTooltip"; export function needsLifetimeSpendBackfill(spend: number, totalSpend: number | null | undefined): boolean { return (totalSpend ?? 0) < spend; @@ -722,7 +723,15 @@ export default function KeyInfoView({
-

Spend

+

+ Spend + +

${formatNumberWithCommas(currentKeyData.spend, 4)}

diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d3c1e0422df..ababe4507b8 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35066,7 +35066,7 @@ export interface components { created_at: string; /** Created By */ created_by: string; - litellm_budget_table?: components["schemas"]["LiteLLM_BudgetTable"] | null; + litellm_budget_table?: components["schemas"]["LiteLLM_BudgetTableFull"] | null; /** * Members * @default [] diff --git a/ui/litellm-dashboard/src/utils/budgetUtils.test.ts b/ui/litellm-dashboard/src/utils/budgetUtils.test.ts index a3b5f4da00a..e33c779bb65 100644 --- a/ui/litellm-dashboard/src/utils/budgetUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/budgetUtils.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it } from "vitest"; -import { describeBudgetResetSchedule, formatBudgetReset, formatBudgetResetUtc } from "./budgetUtils"; +import { budgetPeriodRange, describeBudgetResetSchedule, formatBudgetReset, formatBudgetResetUtc } from "./budgetUtils"; const originalTimezone: string | undefined = process.env.TZ; @@ -44,3 +44,41 @@ describe("describeBudgetResetSchedule", () => { expect(schedule.tooltip).toContain("carries into the next cycle"); }); }); + +describe("budgetPeriodRange", () => { + const resetAt = "2026-04-01T00:00:00Z"; + + it.each([ + ["30d", "2026-03-01T00:00:00.000Z"], + ["monthly", "2026-03-01T00:00:00.000Z"], + ["1mo", "2026-03-01T00:00:00.000Z"], + ["2mo", "2026-02-01T00:00:00.000Z"], + ["7d", "2026-03-25T00:00:00.000Z"], + ["weekly", "2026-03-25T00:00:00.000Z"], + ["1w", "2026-03-25T00:00:00.000Z"], + ["15d", "2026-03-17T00:00:00.000Z"], + ["24h", "2026-03-31T00:00:00.000Z"], + ["daily", "2026-03-31T00:00:00.000Z"], + ["1h", "2026-03-31T23:00:00.000Z"], + ["hourly", "2026-03-31T23:00:00.000Z"], + ])("computes the period start for %s the way the proxy does", (duration, start) => { + expect(budgetPeriodRange(duration, resetAt)).toEqual({ start, end: "2026-04-01T00:00:00.000Z" }); + }); + + it("spans a calendar month for 30d even when the viewer is in a DST-shifting timezone", () => { + process.env.TZ = "America/New_York"; + expect(budgetPeriodRange("30d", "2026-04-01T00:00:00Z")?.start).toBe("2026-03-01T00:00:00.000Z"); + expect(budgetPeriodRange("30d", "2026-03-01T00:00:00Z")?.start).toBe("2026-02-01T00:00:00.000Z"); + }); + + it.each([ + ["bogus", resetAt], + ["0d", resetAt], + ["30d", "not-a-date"], + ["30d", null], + [null, resetAt], + ["", resetAt], + ])("returns null for duration %s and reset %s instead of an invalid date", (duration, reset) => { + expect(budgetPeriodRange(duration, reset)).toBeNull(); + }); +}); diff --git a/ui/litellm-dashboard/src/utils/budgetUtils.ts b/ui/litellm-dashboard/src/utils/budgetUtils.ts index 171ce5818f7..23af15eb539 100644 --- a/ui/litellm-dashboard/src/utils/budgetUtils.ts +++ b/ui/litellm-dashboard/src/utils/budgetUtils.ts @@ -54,3 +54,44 @@ export function describeBudgetResetSchedule( tooltip: `Resets every ${budgetDuration}, but the next reset time has not been scheduled yet. ${ROLLOVER_NOTE}`, }; } + +const BUDGET_DURATION_WORD_ALIASES: Record = { + hourly: "1h", + daily: "24h", + weekly: "7d", + monthly: "30d", +}; + +const BUDGET_DURATION_UNITS: Record = { + s: "second", + m: "minute", + h: "hour", + d: "day", + w: "week", + mo: "month", +}; + +export interface BudgetPeriod { + start: string; + end: string; +} + +/** Mirrors the proxy's get_budget_window_start: `30d` and `Nmo` span calendar months, the rest are fixed deltas. */ +export function budgetPeriodRange( + budgetDuration: string | null | undefined, + budgetResetAt: string | null | undefined, +): BudgetPeriod | null { + if (!budgetDuration || !budgetResetAt) return null; + const end = dayjs.utc(budgetResetAt); + if (!end.isValid()) return null; + const normalized = budgetDuration.trim().toLowerCase(); + const match = /^(\d+)(mo|s|m|h|d|w)$/.exec(BUDGET_DURATION_WORD_ALIASES[normalized] ?? normalized); + if (!match) return null; + const amount = Number(match[1]); + if (amount <= 0) return null; + const start = + match[2] === "d" && amount === 30 + ? end.subtract(1, "month") + : end.subtract(amount, BUDGET_DURATION_UNITS[match[2]]); + return { start: start.toISOString(), end: end.toISOString() }; +} From 76d001a7c06d5dbf93f9191b247db0ef981b6c66 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:06:09 -0700 Subject: [PATCH 100/155] fix(router): serve a team's wildcard deployment before a proxy-wide wildcard (#48490) * fix(router): serve a team's wildcard deployment before a proxy-wide wildcard Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): keep the wildcard precedence helpers free of partially typed locals Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): call each pattern router once in the wildcard precedence helpers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(ptu): a team's wildcard reservation serves beside a proxy-wide catch-all Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: joshua Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 20 +++++------ .../test_ptu_consumption.py | 16 +++------ tests/unit/test_router/test_router.py | 36 ++++++++++++++++++- 3 files changed, 50 insertions(+), 22 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 548df8e6721..ddb4b9bc075 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -13788,21 +13788,21 @@ class Router: return [{**DeploymentTypedDict(**m), "model_name": model_name} for m in matches] def wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]: - """The deployments routing serves ``model`` from through a wildcard: the most specific - proxy-wide pattern matching it, else ``team_id``'s own pattern, never both.""" - proxy_wide: Final[Sequence[Mapping[str, object]]] = self.pattern_router.get_deployments_by_pattern(model=model) + """The deployments routing serves ``model`` from through a wildcard: ``team_id``'s own pattern + matching it, else the most specific proxy-wide pattern, never both.""" team_router: Final = self.team_pattern_routers.get(team_id) if team_id is not None else None - if proxy_wide or team_router is None: - return proxy_wide - return team_router.get_deployments_by_pattern(model=model) + team_owned: Final[Sequence[Mapping[str, object]]] = ( + team_router.get_deployments_by_pattern(model=model) if team_router is not None else () + ) + return team_owned or self.pattern_router.get_deployments_by_pattern(model=model) def wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]: """The models wildcard_route_deployments would route ``model`` to, without copying the deployments.""" - proxy_wide: Final = self.pattern_router.get_models_by_pattern(model=model) team_router: Final = self.team_pattern_routers.get(team_id) if team_id is not None else None - if proxy_wide or team_router is None: - return proxy_wide - return team_router.get_models_by_pattern(model=model) + team_owned: Final = ( + team_router.get_models_by_pattern(model=model) if team_router is not None else frozenset[str]() + ) + return team_owned or self.pattern_router.get_models_by_pattern(model=model) def team_ptu_ceiling(self, team_id: str, model_group: str) -> PTUTeamCeiling | None: """The per-minute ceiling ``team_id``'s PTU shares set on the group serving ``model_group``, diff --git a/tests/unit/proxy/management_endpoints/test_ptu_consumption.py b/tests/unit/proxy/management_endpoints/test_ptu_consumption.py index a446afbd809..065ec93dcd0 100644 --- a/tests/unit/proxy/management_endpoints/test_ptu_consumption.py +++ b/tests/unit/proxy/management_endpoints/test_ptu_consumption.py @@ -232,14 +232,10 @@ def test_rows_keyed_by_a_name_a_wildcard_route_serves_are_sized_by_its_shared_de assert attached.results[0].breakdown.model_groups["ptu-chat"].metrics.ptu_hours == pytest.approx(1.0) -@pytest.mark.parametrize( - ("catch_all_routes", "expected_ptu_hours"), [(False, 1.0), (True, 0.0)], ids=["team-wildcard", "behind-catch-all"] -) -def test_a_teams_wildcard_reservation_reads_ptu_hours_only_while_the_router_serves_from_it( - monkeypatch, catch_all_routes, expected_ptu_hours -): - """The router tries the proxy-wide wildcards before a team's own, so a team's `ptu-*` - reservation behind an open `*` route never serves `ptu-chat`, and its row reads no PTU-hours.""" +@pytest.mark.parametrize("catch_all_routes", [False, True], ids=["team-wildcard", "beside-catch-all"]) +def test_a_teams_wildcard_reservation_reads_ptu_hours_even_beside_a_proxy_wide_catch_all(monkeypatch, catch_all_routes): + """The router tries a team's own wildcard before the proxy-wide ones, so a team's `ptu-*` + reservation serves `ptu-chat` whether or not an open `*` route exists, and its row reads its PTU-hours.""" monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True") reservation: Final = { "model_name": "ptu-team-r", @@ -264,9 +260,7 @@ def test_a_teams_wildcard_reservation_reads_ptu_hours_only_while_the_router_serv attached: Final = with_ptu_consumption(_response(_day("2026-09-23", {"ptu-chat": one_hour})), router, "team-r") - assert attached.results[0].breakdown.model_groups["ptu-chat"].metrics.ptu_hours == pytest.approx( - expected_ptu_hours - ) + assert attached.results[0].breakdown.model_groups["ptu-chat"].metrics.ptu_hours == pytest.approx(1.0) def _mixed_ptu_router() -> Router: diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 55628cb0c34..3004f83b1df 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -8885,7 +8885,8 @@ def test_get_wildcard_deployment_usable_by_team_prefers_the_team_pattern(): pytest.param(False, "rwild-chat", "team-r", ("team-r-wildcard",), id="team-pattern-when-no-proxy-wide-match"), pytest.param(False, "rwild-chat", "team-b", (), id="another-teams-pattern-never-serves"), pytest.param(False, "rwild-chat", None, (), id="no-team-no-team-pattern"), - pytest.param(True, "rwild-chat", "team-r", ("catch-all",), id="proxy-wide-pattern-beats-the-team-pattern"), + pytest.param(True, "rwild-chat", "team-r", ("team-r-wildcard",), id="team-pattern-beats-the-proxy-wide-pattern"), + pytest.param(True, "rwild-chat", "team-b", ("catch-all",), id="another-team-falls-through-to-proxy-wide"), pytest.param(True, "ptu-chat", "team-r", ("ptu-wildcard",), id="only-the-most-specific-proxy-wide-pattern"), ], ) @@ -8924,6 +8925,39 @@ def test_wildcard_route_deployments_is_the_one_pattern_routing_serves_from( ) +@pytest.mark.parametrize( + ("team_id", "expected_id"), + [ + pytest.param("team-a", "team-a-byok", id="team-key-uses-its-own-byok-wildcard"), + pytest.param("team-b", "shared-openrouter", id="team-without-byok-uses-the-shared-wildcard"), + pytest.param(None, "shared-openrouter", id="key-without-team-uses-the-shared-wildcard"), + ], +) +def test_team_byok_wildcard_is_served_before_a_proxy_wide_wildcard_of_the_same_name( + team_id: str | None, expected_id: str +): + """Regression: a proxy-wide openrouter/* shadowed a team's own openrouter/* BYOK deployment for that team's keys.""" + router: Final = litellm.Router( + model_list=[ + { + "model_name": "model_name_team-a_internal", + "litellm_params": {"model": "openrouter/*", "api_key": "sk-team-a"}, + "model_info": {"id": "team-a-byok", "team_id": "team-a", "team_public_model_name": "openrouter/*"}, + }, + { + "model_name": "openrouter/*", + "litellm_params": {"model": "openrouter/*", "api_key": "sk-shared"}, + "model_info": {"id": "shared-openrouter"}, + }, + ] + ) + requested: Final = "openrouter/openai/gpt-5-nano" + + served: Final = router.wildcard_route_deployments(requested, team_id) + + assert tuple(Deployment.model_validate(d).model_info.id for d in served) == (expected_id,) + + def test_get_deployment_credentials_with_provider_aws_bedrock_runtime_endpoint(): """ Test that get_deployment_credentials_with_provider correctly copies From 9873128b42165e8b016afc6cdc0dc1c6ae23f961 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:13:56 -0700 Subject: [PATCH 101/155] fix(team): keep the team reset schedule when a temp budget increase is set on a member inheriting the team default (#48495) * fix(team): keep the team reset schedule when a temp budget increase is set on a member inheriting the team default Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): expect the temp-only member row to carry the team default cap Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: joshua Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management_endpoints/common_utils.py | 21 ++-- .../spend/test_team_member_spend.py | 106 +++++++++++++++++- .../test_upsert_budget_membership.py | 17 ++- 3 files changed, 127 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index adf89c29279..983c3d24513 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -569,8 +569,8 @@ _TEAM_MEMBER_BUDGET_LIMIT_FIELDS: Final = ( "temp_budget_expiry", ) -_TEMP_BUDGET_FIELDS: Final = frozenset({"temp_budget_increase", "temp_budget_expiry"}) +_BUDGET_COLUMNS: Final = TypeAdapter(dict[str, object]) MEMBER_BUDGET_PATCH_FIELDS: Final = MappingProxyType( { @@ -653,8 +653,8 @@ async def upsert_budget_and_membership( ``shared_budget_ids`` extends that protection to any other row more than one membership points at, which a caller patching several members at once has already counted; a row listed there is cloned rather than written in place. - A patch that only touches the temporary budget pair never copies permanent - limits into a new row, so the member keeps inheriting the live team default. + A patch whose every field already matches the shared row changes nothing and + is a no-op rather than a clone. """ ignore_default_model_budget_clear: Final = ( existing_budget_id is not None @@ -685,7 +685,6 @@ async def upsert_budget_and_membership( is_shared_default: Final = existing_budget_id is not None and ( existing_budget_id == team_default_budget_id or existing_budget_id in (shared_budget_ids or frozenset()) ) - temp_only: Final = frozenset(write_data) <= _TEMP_BUDGET_FIELDS async def _disconnect(): await tx.litellm_teammembership.update( @@ -707,11 +706,15 @@ async def upsert_budget_and_membership( return source_row: Final = ( - await tx.litellm_budgettable.find_unique(where={"budget_id": existing_budget_id}) - if is_shared_default and not temp_only - else None + await tx.litellm_budgettable.find_unique(where={"budget_id": existing_budget_id}) if is_shared_default else None ) - source: Final[Mapping[str, object]] = source_row.model_dump() if source_row is not None else MappingProxyType({}) + source: Final[Mapping[str, object]] = ( + MappingProxyType(_BUDGET_COLUMNS.validate_python(source_row.model_dump())) + if source_row is not None + else MappingProxyType({}) + ) + if is_shared_default and all(source.get(field) == value for field, value in effective_patch.items()): + return create_data: Final[dict[str, object]] = { # mutable-ok: Prisma create payloads are dict-shaped "created_by": user_api_key_dict.user_id or "", @@ -740,7 +743,7 @@ async def upsert_budget_and_membership( create_data.pop("budget_reset_at", None) if not _has_meaningful_budget_limit(create_data): - if existing_budget_id is not None and not temp_only: + if existing_budget_id is not None: await _disconnect() return diff --git a/tests/integration/spend/test_team_member_spend.py b/tests/integration/spend/test_team_member_spend.py index 306138e2bb1..c11dc9db4bf 100644 --- a/tests/integration/spend/test_team_member_spend.py +++ b/tests/integration/spend/test_team_member_spend.py @@ -1,3 +1,4 @@ +import asyncio import os import uuid from collections.abc import Callable, Iterator @@ -7,10 +8,14 @@ from typing import Final import httpx import pytest from integration._support.client import Gateway, Scenario, eventually, object_value, string_value -from integration._support.database import read_rows +from integration._support.database import read_rows, write_rows from pydantic import BaseModel, JsonValue from redis import Redis +from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.utils import PrismaClient, ProxyLogging + def _python_isoformat(value: datetime) -> str: return value.isoformat() @@ -444,9 +449,7 @@ def test_member_update_temp_budget_increase_readmits_a_blocked_member( ) assert added.status_code == 200, added.text default_budget_id: Final = _team_default_budget_id(team) if budget_source == "team-default" else None - default_budget_before: Final = ( - _budget_record(default_budget_id) if default_budget_id is not None else () - ) + default_budget_before: Final = _budget_record(default_budget_id) if default_budget_id is not None else () if default_budget_id is not None: assert len(default_budget_before) == 1, repr(default_budget_before) assert default_budget_before[0].max_budget == 0.05, default_budget_before @@ -503,7 +506,7 @@ def test_member_update_temp_budget_increase_readmits_a_blocked_member( else: assert default_budget_id is not None assert state[0].budget_id != default_budget_id, state - assert state[0].max_budget is None, state + assert state[0].max_budget == 0.05, state default_budget_after: Final = _budget_record(default_budget_id) assert default_budget_after == default_budget_before, repr(default_budget_after) assert default_budget_after[0].max_budget == 0.05, default_budget_after @@ -1024,3 +1027,96 @@ def test_bulk_update_restores_team_default_when_last_member_limit_is_cleared( assert first_observations + second_observations == (provider_model, provider_model), repr( (first_observations, second_observations) ) + + +def _budget_reset_at(budget_id: str) -> str: + rows: Final = read_rows( + 'SELECT budget_reset_at::text AS budget_reset_at FROM "LiteLLM_BudgetTable" WHERE budget_id = %s', (budget_id,) + ) + assert len(rows) == 1, rows + return string_value(rows[0]["budget_reset_at"]) + + +async def _run_scheduled_budget_reset() -> None: + client: Final = PrismaClient( + database_url=os.environ["DATABASE_URL"], proxy_logging_obj=ProxyLogging(user_api_key_cache=UserApiKeyCache()) + ) + await client.connect() + try: + await ResetBudgetJob( + ProxyLogging(user_api_key_cache=UserApiKeyCache()), client + ).reset_budget_for_litellm_budget_table() + finally: + await client.db.disconnect() + + +def test_temp_budget_increase_on_a_team_default_member_keeps_the_team_reset_schedule( + gateway: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + model, provider_model = _priced_model(scenario) + team: Final = scenario.team(models=[model], team_member_budget=5, team_member_budget_duration="30d") + topped_up: Final = scenario.user() + untouched: Final = scenario.user() + for user in (topped_up, untouched): + added = gateway.request( + "POST", "/team/member_add", {"team_id": team, "member": {"user_id": user, "role": "user"}} + ) + assert added.status_code == 200, added.text + default_budget_id: Final = _team_default_budget_id(team) + for user in (topped_up, untouched): + key = scenario.key(team_id=team, user_id=user) + chat = _chat(gateway, model, key, f"spend before top-up {uuid.uuid4().hex}") + assert chat.status_code == 200, chat.text + observed: Final = _observed_models(upstream) + assert observed == (provider_model, provider_model), repr(observed) + for user in (topped_up, untouched): + accrued = eventually( + lambda user=user: _budget_state(team, user), + lambda values: len(values) == 1 and values[0].spend is not None and values[0].spend > 0, + seconds=70, + ) + assert accrued[0].budget_id == default_budget_id, accrued + + expiry: Final = datetime.now(timezone.utc) + timedelta(days=30) + granted: Final = _member_update( + gateway, + { + "team_id": team, + "user_id": topped_up, + "temp_budget_increase": 1, + "temp_budget_expiry": expiry.isoformat(), + }, + ) + assert granted.temp_budget_increase == 1, granted + topped_up_state: Final = _budget_state(team, topped_up) + assert len(topped_up_state) == 1, topped_up_state + topped_up_budget_id: Final = topped_up_state[0].budget_id + assert topped_up_budget_id is not None and topped_up_budget_id != default_budget_id, topped_up_state + assert topped_up_state[0].max_budget == 5, topped_up_state + assert topped_up_state[0].budget_duration == "30d", topped_up_state + assert topped_up_state[0].temp_budget_increase == 1, topped_up_state + assert _budget_reset_at(topped_up_budget_id) == _budget_reset_at(default_budget_id), topped_up_state + assert _budget_state(team, untouched)[0].budget_id == default_budget_id + + write_rows( + "UPDATE \"LiteLLM_BudgetTable\" SET budget_reset_at = NOW() - INTERVAL '1 hour' WHERE budget_id IN (%s, %s)", + (default_budget_id, topped_up_budget_id), + ) + asyncio.run(_run_scheduled_budget_reset()) + for user in (topped_up, untouched): + after_reset = _budget_state(team, user) + assert len(after_reset) == 1 and after_reset[0].spend == 0, (user, after_reset) + assert _budget_reset_at(topped_up_budget_id) == _budget_reset_at(default_budget_id) + + cleared: Final = _member_update( + gateway, + {"team_id": team, "user_id": topped_up, "temp_budget_increase": None, "temp_budget_expiry": None}, + ) + assert cleared.temp_budget_increase is None, cleared + final_state: Final = _budget_state(team, topped_up) + assert len(final_state) == 1, final_state + assert final_state[0].budget_id == topped_up_budget_id, final_state + assert final_state[0].budget_duration == "30d", final_state + assert final_state[0].max_budget == 5, final_state + assert final_state[0].temp_budget_increase is None, final_state diff --git a/tests/unit/proxy/common_utils/test_upsert_budget_membership.py b/tests/unit/proxy/common_utils/test_upsert_budget_membership.py index 1f7154f0a74..4aa1a7f7cb1 100644 --- a/tests/unit/proxy/common_utils/test_upsert_budget_membership.py +++ b/tests/unit/proxy/common_utils/test_upsert_budget_membership.py @@ -267,10 +267,17 @@ async def test_create_from_temp_pair_never_snapshots_team_default(mock_tx, fake_ @pytest.mark.asyncio -async def test_temp_pair_on_shared_default_member_creates_bare_row(mock_tx, fake_user): +async def test_temp_pair_on_shared_default_member_clones_default_with_its_reset_schedule(mock_tx, fake_user): expiry = datetime(2100, 1, 1, tzinfo=timezone.utc) + default_reset_at = datetime(2026, 11, 1, tzinfo=timezone.utc) mock_tx.litellm_budgettable.find_unique = AsyncMock( - return_value=budget_row(budget_id="team-default-budget-1", max_budget=0.4, rpm_limit=10) + return_value=budget_row( + budget_id="team-default-budget-1", + max_budget=0.4, + rpm_limit=10, + budget_duration="30d", + budget_reset_at=default_reset_at, + ) ) await upsert_budget_and_membership( mock_tx, @@ -282,10 +289,14 @@ async def test_temp_pair_on_shared_default_member_creates_bare_row(mock_tx, fake team_default_budget_id="team-default-budget-1", ) - mock_tx.litellm_budgettable.find_unique.assert_not_awaited() mock_tx.litellm_budgettable.update.assert_not_called() + mock_tx.litellm_teammembership.update.assert_not_called() assert stored_budget_row(mock_tx) == { "model_max_budget": "{}", + "max_budget": 0.4, + "rpm_limit": 10, + "budget_duration": "30d", + "budget_reset_at": default_reset_at, "temp_budget_increase": 1.0, "temp_budget_expiry": expiry, } From 833d82bb4ee4886a9d9058a219fddf0d09e700b3 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 26 Oct 2026 20:14:57 -0400 Subject: [PATCH 102/155] fix(mcp): reject revoked browser sessions during gateway authorization (#48494) --- .../mcp_server/discoverable_endpoints.py | 24 +- .../mcp_server/test_discoverable_endpoints.py | 271 ++++++++++++++---- 2 files changed, 228 insertions(+), 67 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index f5c54451006..ca7ad9e3384 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -472,15 +472,15 @@ def redeem_passthrough_authorization_code( return sealed -def _session_cookie_user_id(request: Request) -> str | None: +async def _session_cookie_user_id(request: Request) -> str | None: """The signed-in litellm user for a browser request, or ``None``. Thin wrapper so the aggregate DCR flow's verbs receive the identity as a plain value instead of parsing cookies themselves.""" from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( # noqa: PLC0415 # circular import at module load - user_id_from_session_cookie, + get_authenticated_browser_user_id, ) - return user_id_from_session_cookie(request) + return await get_authenticated_browser_user_id(request) def _redirect_to_litellm_login(request: Request) -> RedirectResponse: @@ -996,17 +996,13 @@ async def _resolve_oauth_authorization_user( enforce_binding: bool, ) -> str | RedirectResponse: """Resolve the authorization subject without replacing denied credentials with cookie grants.""" - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( # noqa: PLC0415 # proxy import cycle - user_id_from_session_cookie, - ) - use_gateway_credential: Final = enforce_binding and await oauth_authorization_uses_gateway_credential(request) request_user_id: Final = ( await authorize_oauth_credential_request(request, mcp_server.server_id) if use_gateway_credential else None ) if use_gateway_credential and request_user_id is None: return _bridge_access_denied_redirect(redirect_uri, state, mcp_server) - user_id: Final = request_user_id or user_id_from_session_cookie(request) + user_id: Final = request_user_id or await _session_cookie_user_id(request) if user_id is None: return _redirect_to_litellm_login(request) if not await _user_can_reach_mcp_server(user_id, mcp_server.server_id): @@ -1024,7 +1020,7 @@ async def _managed_authorization_scope( request ) session_user: Final = ( - _session_cookie_user_id(request) if authorized_user_id is None and not gateway_credential else None + await _session_cookie_user_id(request) if authorized_user_id is None and not gateway_credential else None ) scope_user_id: Final = authorized_user_id or ( await authorize_oauth_credential_request(request, server.server_id) @@ -2359,7 +2355,7 @@ async def authorize_mcp_session( code_challenge=code_challenge, code_challenge_method=code_challenge_method, response_type=response_type, - session_user_id=_session_cookie_user_id(request), + session_user_id=await _session_cookie_user_id(request), resource=resource, scope=scope, ) @@ -2391,7 +2387,7 @@ async def authorize( code_challenge=code_challenge, code_challenge_method=code_challenge_method, response_type=response_type, - session_user_id=_session_cookie_user_id(request), + session_user_id=await _session_cookie_user_id(request), lookup_consent_teams=lookup_consent_teams, ) return aggregate_authorize( @@ -2402,7 +2398,7 @@ async def authorize( code_challenge=code_challenge, code_challenge_method=code_challenge_method, response_type=response_type, - session_user_id=_session_cookie_user_id(request), + session_user_id=await _session_cookie_user_id(request), resource=resource, scope=scope, ) @@ -2560,7 +2556,7 @@ async def authorize_flow(request: Request, flow: str) -> Response: return await describe_connect_flow( request=request, flow_handle=flow, - session_user_id=_session_cookie_user_id(request), + session_user_id=await _session_cookie_user_id(request), lookup_vendor_credential=_vendor_credential_state, lookup_server_reachability=_user_can_reach_mcp_server, ) @@ -2586,7 +2582,7 @@ async def authorize_complete( return await complete_connect_flow( request=request, flow_handle=flow, - session_user_id=_session_cookie_user_id(request), + session_user_id=await _session_cookie_user_id(request), cache=user_api_key_cache, delivery=delivery, team_id=team_id, diff --git a/tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index a43dacaf100..cedfa296515 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -5599,12 +5599,13 @@ def _bridge_server(**overrides): return MCPServer(**fields) -def _bridge_mock_request(): +def _bridge_mock_request(cookie: str | None = None) -> MagicMock: from fastapi import Request - mock_request = MagicMock(spec=Request) + mock_request: Final = MagicMock(spec=Request) mock_request.base_url = "https://litellm.example.com/" mock_request.headers = {} + mock_request.cookies = {"token": cookie} if cookie else {} return mock_request @@ -6524,7 +6525,7 @@ async def test_interactive_bridge_gateway_code_for_another_server_is_rejected_40 @pytest.mark.asyncio -async def test_interactive_bridge_authorize_seals_sso_user_into_state(): +async def test_interactive_bridge_authorize_seals_sso_user_into_state(monkeypatch: pytest.MonkeyPatch) -> None: """On the short-circuit bridge oauth_delegate arm, authorize captures the SSO user from the UI session cookie and seals it (and the target server) into the encrypted OAuth state, so the callback can later mint a user-bound gateway code; it still proceeds to the upstream redirect. @@ -6540,16 +6541,13 @@ async def test_interactive_bridge_authorize_seals_sso_user_into_state(): admitted = UserAPIKeyAuth(user_id="sso-user-42") admitted.mcp_admitted_user_subject = True captured: dict = {} + cookie: Final = _browser_session_cookie(monkeypatch, "sso-user-42") def _capture(**kwargs): captured.update(kwargs) return "mocked_encrypted_state" with ( - patch( - "litellm.proxy._experimental.mcp_server.byok_oauth_endpoints.user_id_from_session_cookie", - return_value="sso-user-42", - ), patch( "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", new=AsyncMock(return_value=admitted), @@ -6565,7 +6563,7 @@ async def test_interactive_bridge_authorize_seals_sso_user_into_state(): ), ): response = await authorize_with_server( - request=_bridge_mock_request(), + request=_bridge_mock_request(cookie), mcp_server=server, client_id="ignored", redirect_uri="http://127.0.0.1:60108/callback", @@ -6581,7 +6579,9 @@ async def test_interactive_bridge_authorize_seals_sso_user_into_state(): @pytest.mark.asyncio @pytest.mark.parametrize("user_can_reach_server", [True, False]) -async def test_bridge_authorize_gates_on_the_egress_server_access_resolver(user_can_reach_server): +async def test_bridge_authorize_gates_on_the_egress_server_access_resolver( + monkeypatch: pytest.MonkeyPatch, user_can_reach_server: bool +) -> None: """The interactive dcr_bridge oauth_delegate authorize admits the signed-in user the way MCP egress will and refuses with an RFC 6749 access_denied redirect when that admitted subject cannot reach the target server, instead of minting an envelope whose every tool request would @@ -6607,14 +6607,11 @@ async def test_bridge_authorize_gates_on_the_egress_server_access_resolver(user_ mock_request = MagicMock(spec=Request) mock_request.base_url = "https://litellm.example.com/" mock_request.headers = {} + mock_request.cookies = {"token": _browser_session_cookie(monkeypatch, "bridge-user-1")} client_redirect = "http://127.0.0.1:60108/callback" try: with ( - patch( - "litellm.proxy._experimental.mcp_server.byok_oauth_endpoints.user_id_from_session_cookie", - return_value="bridge-user-1", - ), patch( "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", new=AsyncMock(return_value=admitted), @@ -6660,7 +6657,9 @@ async def test_bridge_authorize_gates_on_the_egress_server_access_resolver(user_ @pytest.mark.asyncio @pytest.mark.parametrize("reload_status,expect_denial", [(401, True), (500, False), (503, False)]) -async def test_bridge_authorize_reload_failure_denies_or_stays_retryable(reload_status, expect_denial): +async def test_bridge_authorize_reload_failure_denies_or_stays_retryable( + monkeypatch: pytest.MonkeyPatch, reload_status: int, expect_denial: bool +) -> None: """An unknown or deactivated signed-in user denies like a missing grant (fail closed); a DB outage keeps its retryable 503 instead of masquerading as an access denial.""" from urllib.parse import parse_qs, urlparse @@ -6679,13 +6678,10 @@ async def test_bridge_authorize_reload_failure_denies_or_stays_retryable(reload_ mock_request = MagicMock(spec=Request) mock_request.base_url = "https://litellm.example.com/" mock_request.headers = {} + mock_request.cookies = {"token": _browser_session_cookie(monkeypatch, "bridge-user-1")} try: with ( - patch( - "litellm.proxy._experimental.mcp_server.byok_oauth_endpoints.user_id_from_session_cookie", - return_value="bridge-user-1", - ), patch( "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", new=AsyncMock(side_effect=HTTPException(status_code=reload_status, detail="x")), @@ -6728,19 +6724,15 @@ async def test_interactive_bridge_authorize_without_session_redirects_to_login() from litellm.types.mcp import MCPAuth server = _bridge_server(auth_type=MCPAuth.oauth_delegate, client_id="admin-client", registration_url=None) - with patch( - "litellm.proxy._experimental.mcp_server.byok_oauth_endpoints.user_id_from_session_cookie", - return_value=None, - ): - response = await authorize_with_server( - request=_bridge_mock_request(), - mcp_server=server, - client_id="ignored", - redirect_uri="http://127.0.0.1:60108/callback", - state="s", - code_challenge="chal", - code_challenge_method="S256", - ) + response = await authorize_with_server( + request=_bridge_mock_request(), + mcp_server=server, + client_id="ignored", + redirect_uri="http://127.0.0.1:60108/callback", + state="s", + code_challenge="chal", + code_challenge_method="S256", + ) assert "/sso/key/generate" in response.headers["location"] @@ -11997,10 +11989,97 @@ def _s256(verifier: str) -> str: _NATIVE_CLIENT_MASTER_KEY = "sk-test-salt-for-LIT-5874" +def _browser_session_cookie( + monkeypatch: pytest.MonkeyPatch, + user_id: str, + *, + signing_key: str = _NATIVE_CLIENT_MASTER_KEY, + session_key: str | None = None, + expired_session: bool = False, + expired_cookie: bool = False, +) -> str: + import jwt + + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.auth_checks import LITELLM_SESSION_TOKEN_PREFIX + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_bearer_token + + monkeypatch.setattr(proxy_server, "master_key", signing_key) + if proxy_server.prisma_client is None: + monkeypatch.setattr(proxy_server, "prisma_client", object()) + monkeypatch.setattr(proxy_server, "general_settings", {**proxy_server.general_settings, "supported_db_objects": []}) + session: Final = UserAPIKeyAuth( + user_id=user_id, + expires=datetime(2000 if expired_session else 2099, 1, 1, tzinfo=timezone.utc), + ) + key: Final = ( + encrypt_bearer_token(session.model_dump_json(exclude_none=True), LITELLM_SESSION_TOKEN_PREFIX) + if session_key is None + else session_key + ) + return jwt.encode( + {"user_id": user_id, "key": key, "login_method": "sso", "exp": 946684800 if expired_cookie else 9999999999}, + signing_key, + algorithm="HS256", + ) + + +class _BrowserSessionDatabase: + def __init__(self) -> None: + from litellm.proxy._types import UserAPIKeyAuth, hash_token + + self.row: UserAPIKeyAuth | None = UserAPIKeyAuth( + token=hash_token("sk-browser-session"), + user_id="u1", + expires=datetime(2099, 1, 1, tzinfo=timezone.utc), + ) + self.litellm_deprecatedverificationtoken = self + + def __call__(self, **kwargs: object) -> "_BrowserSessionDatabase": + return self + + async def query_first(self, query: str, *args: object) -> dict[str, object] | None: + if self.row is None: + return None + return { + **self.row.model_dump(exclude={"last_refreshed_at"}), + "team_models": None, + "team_blocked": None, + "team_members_with_roles": None, + } + + async def find_first(self, **kwargs: object) -> None: + return None + + +@pytest.fixture +def browser_session_database(monkeypatch: pytest.MonkeyPatch) -> _BrowserSessionDatabase: + import prisma + + from litellm.proxy import proxy_server + from litellm.proxy._types import hash_token + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.utils import PrismaClient + + database: Final = _BrowserSessionDatabase() + monkeypatch.setattr(prisma, "Prisma", database) + for name in ("DATABASE_URL_READ_REPLICA", "IAM_TOKEN_DB_AUTH", "AZURE_POSTGRESQL_AUTH"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr( + proxy_server, + "prisma_client", + PrismaClient("postgresql://local-test@localhost/browser-session", proxy_server.proxy_logging_obj), + ) + cache: Final = UserApiKeyCache() + cache.set_cache(hash_token("sk-browser-session"), database.row) + monkeypatch.setattr(proxy_server, "user_api_key_cache", cache) + return database + + def _native_client_app(monkeypatch): """The unauthenticated discoverable router served over TestClient with a signed UI session cookie available, plus fakes for the two database-backed hooks the native-client flow calls.""" - import jwt from fastapi import FastAPI from fastapi.testclient import TestClient @@ -12029,14 +12108,108 @@ def _native_client_app(monkeypatch): app = FastAPI() app.include_router(router) client = TestClient(app) - session_cookie = jwt.encode( - {"user_id": "u1", "login_method": "username_password", "exp": int(time.time()) + 600}, - _NATIVE_CLIENT_MASTER_KEY, - algorithm="HS256", - ) + session_cookie: Final = _browser_session_cookie(monkeypatch, "u1") return client, session_cookie, minted +@pytest.mark.parametrize( + "authorize_path,resource", + [("/authorize", None), ("/authorize/mcp-session", None), ("/authorize", "http://testserver")], +) +@pytest.mark.parametrize( + "session_state", ["valid", "revoked", "blocked", "expired", "wrong_owner", "missing_key", "no_database"] +) +def test_gateway_authorize_requires_live_browser_session( + monkeypatch: pytest.MonkeyPatch, + browser_session_database: _BrowserSessionDatabase, + authorize_path: str, + resource: str | None, + session_state: str, +) -> None: + from urllib.parse import urlparse + + from litellm.proxy import proxy_server + + client, _, _ = _native_client_app(monkeypatch) + database: Final = browser_session_database + assert database.row is not None + database.row = None if session_state == "revoked" else database.row.model_copy( + update={ + "blocked": session_state == "blocked", + "expires": datetime(2000 if session_state == "expired" else 2099, 1, 1, tzinfo=timezone.utc), + "user_id": "another-user" if session_state == "wrong_owner" else "u1", + } + ) + cookie: Final = _browser_session_cookie( + monkeypatch, "u1", session_key="" if session_state == "missing_key" else "sk-browser-session" + ) + client.cookies.set("token", cookie) + if session_state == "no_database": + monkeypatch.setattr(proxy_server, "prisma_client", None) + registered: Final = client.post("/register", json={"redirect_uris": ["http://127.0.0.1/callback"]}) + assert registered.status_code == 201, registered.text + response: Final = client.get( + authorize_path, + params={ + "client_id": registered.json()["client_id"], + "redirect_uri": "http://127.0.0.1/callback", + "response_type": "code", + "code_challenge": _s256("v" * 43), + "code_challenge_method": "S256", + **({"resource": resource} if resource else {}), + }, + follow_redirects=False, + ) + if session_state == "valid" and resource: + assert response.status_code == 200 and 'name="decision"' in response.text, response.text + else: + assert response.status_code == 303, response.text + assert urlparse(response.headers["location"]).path == ( + "/ui/connect" if session_state == "valid" else "/sso/key/generate" + ) + + +@pytest.mark.parametrize("endpoint", ["flow", "complete"]) +@pytest.mark.parametrize("revoked", [False, True]) +def test_gateway_connect_rechecks_session_after_authorize( + monkeypatch: pytest.MonkeyPatch, + browser_session_database: _BrowserSessionDatabase, + endpoint: str, + revoked: bool, +) -> None: + from urllib.parse import parse_qs, urlparse + + client, _, _ = _native_client_app(monkeypatch) + client.cookies.set("token", _browser_session_cookie(monkeypatch, "u1", session_key="sk-browser-session")) + registered: Final = client.post("/register", json={"redirect_uris": ["http://127.0.0.1/callback"]}) + assert registered.status_code == 201, registered.text + authorized: Final = client.get( + "/authorize/mcp-session", + params={ + "client_id": registered.json()["client_id"], + "redirect_uri": "http://127.0.0.1/callback", + "response_type": "code", + "code_challenge": _s256("v" * 43), + "code_challenge_method": "S256", + }, + follow_redirects=False, + ) + flow: Final = parse_qs(urlparse(authorized.headers["location"]).query)["connect_flow"][0] + if revoked: + browser_session_database.row = None + response: Final = ( + client.get("/authorize/flow", params={"flow": flow}) + if endpoint == "flow" + else client.post("/authorize/complete", data={"flow": flow}, follow_redirects=False) + ) + assert response.status_code == (401 if revoked else 200 if endpoint == "flow" else 303), response.text + if revoked: + assert response.json()["error"] == "login_required" + assert "location" not in response.headers + elif endpoint == "complete": + assert parse_qs(urlparse(response.headers["location"]).query)["code"] + + @pytest.mark.parametrize("authorize_path", ["/authorize/mcp-session", "/authorize"]) def test_gateway_authorize_routes_forward_server_scope(monkeypatch: pytest.MonkeyPatch, authorize_path: str) -> None: from http.cookies import SimpleCookie @@ -13605,11 +13778,7 @@ async def test_identity_bound_authorize_preserves_presented_jwt_permissions( else "https://idp.example.test", ) ) - cookie: Final = jwt.encode( - {"user_id": "cookie-owner", "login_method": "sso", "exp": int(time.time()) + 300}, - master, - algorithm="HS256", - ) + cookie: Final = _browser_session_cookie(monkeypatch, "cookie-owner", signing_key=master) response: Final = await discoverable_endpoints.authorize_with_server( request=_token_request( { @@ -13645,7 +13814,7 @@ async def test_identity_bound_authorize_preserves_presented_jwt_permissions( @pytest.mark.asyncio @pytest.mark.parametrize("credential", ["none", "opaque", "foreign_jwt"]) -@pytest.mark.parametrize("cookie_state", ["allowed", "server_denied", "expired", "missing"]) +@pytest.mark.parametrize("cookie_state", ["allowed", "server_denied", "expired", "missing", "session_expired"]) async def test_identity_bound_authorize_unrelated_bearer_uses_browser_session( jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], monkeypatch: pytest.MonkeyPatch, @@ -13654,8 +13823,6 @@ async def test_identity_bound_authorize_unrelated_bearer_uses_browser_session( ) -> None: from urllib.parse import parse_qs, urlparse - import jwt - from litellm.models.user import LiteLLM_UserTable from litellm.proxy import proxy_server from litellm.proxy._experimental.mcp_server import discoverable_endpoints, mcp_server_manager @@ -13694,14 +13861,12 @@ async def test_identity_bound_authorize_unrelated_bearer_uses_browser_session( if credential == "foreign_jwt" else "unrelated-upstream-bearer" ) - cookie: Final = jwt.encode( - { - "user_id": "cookie-owner", - "login_method": "sso", - "exp": int(time.time()) + (-60 if cookie_state == "expired" else 300), - }, - master, - algorithm="HS256", + cookie: Final = _browser_session_cookie( + monkeypatch, + "cookie-owner", + signing_key=master, + expired_cookie=cookie_state == "expired", + expired_session=cookie_state == "session_expired", ) response: Final = await discoverable_endpoints.authorize_with_server( request=_token_request( From 549e6d4f465ec9cf4ca80d3d7e2541ff61ea29ef Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:17:52 -0700 Subject: [PATCH 103/155] fix(router): keep DB-only deployments priced across price data reloads (#48486) * fix(router): track every router for price data reloads so DB-only deployments keep custom pricing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): register the router for reload replay only once model_list exists, and type the regression test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: joshua Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 7 +++-- .../unit/test_router_model_cost_isolation.py | 30 +++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index ddb4b9bc075..8d95e3e91d4 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1483,15 +1483,16 @@ class Router: if model_list is not None: # set_model_list will build indices automatically self.set_model_list(model_list) - # Track this router so a price data reload can rebuild its deployments' - # cost-map entries from the list it is serving at that moment. - _live_routers.add(self) self.healthy_deployments: list = self.model_list for m in model_list: if "model" in m["litellm_params"]: self.deployment_latency_map[m["litellm_params"]["model"]] = 0 else: self.model_list: list = [] # initialize an empty list - to allow _add_deployment and delete_deployment to work + # Track this router so a price data reload can rebuild its deployments' + # cost-map entries from the list it is serving at that moment, whether + # those deployments came from model_list or were added later from the DB. + _live_routers.add(self) if allowed_fails is not None: self.allowed_fails = allowed_fails diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index f001008ee2c..20b0055f729 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -3081,6 +3081,36 @@ def test_reload_keeps_custom_pricing_configured_on_litellm_params_for_a_db_model _invalidate_model_cost_lowercase_map() +def test_reload_keeps_custom_pricing_of_a_db_model_on_a_router_built_without_a_model_list( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ + A proxy whose config.yaml has no model_list builds its router without one and + takes every deployment from the DB afterwards. A price data reload must restore + that deployment's custom pricing exactly as it does for a config router. + """ + fetched_catalog: Final = copy.deepcopy(litellm.model_cost) + router: Final = Router() + router.add_deployment( + deployment=Deployment( + model_name="db-only-priced", + litellm_params=LiteLLM_Params( + model="openai/gpt-4o", + api_key="sk-fake", + input_cost_per_token=0.000123, + output_cost_per_token=0.000456, + ), + model_info=ModelInfo(id="db-only-priced-id"), + ) + ) + assert litellm.model_cost["db-only-priced-id"]["input_cost_per_token"] == 0.000123 + + _simulate_price_data_reload_with_provider_sets(monkeypatch, fetched_catalog) + + assert litellm.model_cost["db-only-priced-id"]["input_cost_per_token"] == 0.000123 + assert litellm.model_cost["db-only-priced-id"]["output_cost_per_token"] == 0.000456 + + def test_replay_live_router_model_cost_rebuilds_every_live_router(): """ A process can hold more than one Router, so the rebuild has to fan out across From 100529391a249e2f6c24bdc2c198afb0d57e59cf Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:24:44 -0700 Subject: [PATCH 104/155] fix(ui): show the budget period date range instead of "cycle" on team member spend (#48488) * fix(ui): show the budget period date range instead of "cycle" on team member spend Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): derive the budget period start in UTC before showing it in local time Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): keep clock times on sub-day budget periods and restore the total spend lower bound wording Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: joshua Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../src/components/team/MyUserTab.test.tsx | 6 +- .../src/components/team/MyUserTab.tsx | 16 ++-- .../components/team/TeamMemberTab.test.tsx | 29 ++++++- .../src/components/team/TeamMemberTab.tsx | 41 ++++++---- .../src/utils/budgetUtils.test.ts | 77 ++++++++++++++++++- ui/litellm-dashboard/src/utils/budgetUtils.ts | 64 +++++++++++++-- 6 files changed, 198 insertions(+), 35 deletions(-) diff --git a/ui/litellm-dashboard/src/components/team/MyUserTab.test.tsx b/ui/litellm-dashboard/src/components/team/MyUserTab.test.tsx index 24ce626ecb7..eee9fcdadd1 100644 --- a/ui/litellm-dashboard/src/components/team/MyUserTab.test.tsx +++ b/ui/litellm-dashboard/src/components/team/MyUserTab.test.tsx @@ -48,7 +48,7 @@ describe("MyUserTab", () => { expect(screen.getByText("Never resets")).toBeInTheDocument(); }); - it("shows the next reset time when the member budget has a reset window", () => { + it("shows the budget period and next reset when the member budget has a reset window", () => { vi.mocked(useMyTeamMember).mockReturnValue({ data: { user_id: "user-1", @@ -69,6 +69,8 @@ describe("MyUserTab", () => { renderWithProviders(); - expect(screen.getByText("Resets Jul 15, 2026 12:00")).toBeInTheDocument(); + expect(screen.getByText("Current Period Spend (USD)")).toBeInTheDocument(); + expect(screen.getByText("Jun 15 - Jul 15, 2026 (resets Jul 15, 2026 12:00)")).toBeInTheDocument(); + expect(screen.queryByText(/cycle/i)).not.toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/team/MyUserTab.tsx b/ui/litellm-dashboard/src/components/team/MyUserTab.tsx index bf1a2c2c242..8d7583c51e4 100644 --- a/ui/litellm-dashboard/src/components/team/MyUserTab.tsx +++ b/ui/litellm-dashboard/src/components/team/MyUserTab.tsx @@ -1,4 +1,4 @@ -import { describeBudgetResetSchedule } from "@/utils/budgetUtils"; +import { BudgetResetSchedule, describeBudgetResetSchedule } from "@/utils/budgetUtils"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { SimpleTooltip } from "@/components/ui/tooltip"; import { Badge } from "@/components/ui/badge"; @@ -30,6 +30,12 @@ const formatRateLimit = (value: number | null | undefined): string => { return formatNumberWithCommas(value, 0); }; +function describePeriodLine(schedule: BudgetResetSchedule): string { + if (schedule.period) return `${schedule.period} (resets ${schedule.label})`; + if (schedule.resetAt) return `Resets ${schedule.label}`; + return schedule.label; +} + export default function MyUserTab({ teamId }: MyUserTabProps) { const { data, isLoading, error } = useMyTeamMember(teamId); @@ -93,16 +99,14 @@ export default function MyUserTab({ teamId }: MyUserTabProps) {

- {labelWithTooltip("Current Cycle Spend (USD)", budgetReset.tooltip)} + {labelWithTooltip("Current Period Spend (USD)", budgetReset.tooltip)}

${formatNumber(spend, 4)}

of {maxBudget === null ? "Unlimited" : `$${formatNumber(maxBudget, 4)}`}
-
- {budgetReset.resetAt ? `Resets ${budgetReset.label}` : budgetReset.label} -
+
{describePeriodLine(budgetReset)}
 @@ -121,7 +125,7 @@ export default function MyUserTab({ teamId }: MyUserTabProps) { {labelWithTooltip( "Total Spend (USD)", - "Cumulative spend across budget cycles within this team. This is a lower bound: lifetime tracking began 2026-04-21, so cycles completed before then are not included. It is never less than Current Cycle Spend.", + "Cumulative spend within this team across budget periods. This is a lower bound: lifetime tracking began 2026-04-21, so periods that reset before then are not included. It is never less than Current Period Spend.", )}

${formatNumber(totalSpend, 4)}

 diff --git a/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx b/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx index 38a8e4f97d5..9350de8a8fc 100644 --- a/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx @@ -316,6 +316,29 @@ describe("TeamMembersComponent", () => { expect(screen.getByText("Jul 15, 2026 12:00")).toBeInTheDocument(); }); + it("shows the budget period date range instead of a bare reset date when the duration is known", () => { + const teamData = createMockTeamData(); + teamData.team_memberships[0].litellm_budget_table!.budget_duration = "30d"; + renderWithProviders( + , + ); + + expect(screen.getByRole("columnheader", { name: /^Budget Period/ })).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: /^Current Period Spend \(USD\)/ })).toBeInTheDocument(); + expect(screen.getByText("Jun 15 - Jul 15, 2026")).toBeInTheDocument(); + expect(screen.queryByText("Jul 15, 2026 12:00")).not.toBeInTheDocument(); + expect(screen.queryByText(/cycle/i)).not.toBeInTheDocument(); + }); + it("shows Never resets for a member whose budget has no reset window", () => { const teamData = createMockTeamData(); teamData.team_memberships[0].litellm_budget_table!.budget_reset_at = null; @@ -696,7 +719,7 @@ describe("TeamMembersComponent", () => { />, ); - it("resets the member's current cycle spend to $0 after confirming, then refreshes the team", async () => { + it("resets the member's current period spend to $0 after confirming, then refreshes the team", async () => { const user = userEvent.setup(); POST.mockResolvedValue({ data: {} }); renderEditableTab(); @@ -745,7 +768,7 @@ describe("TeamMembersComponent", () => { expect(POST).not.toHaveBeenCalled(); }); - it("disables the reset with a reason on members that have no current cycle spend", async () => { + it("disables the reset with a reason on members that have no current period spend", async () => { const user = userEvent.setup(); renderEditableTab(); @@ -753,7 +776,7 @@ describe("TeamMembersComponent", () => { "reset-member-spend", ); await user.hover(resetButton); - expect(await screen.findByText("No current cycle spend to reset")).toBeInTheDocument(); + expect(await screen.findByText("No current period spend to reset")).toBeInTheDocument(); await user.click(resetButton); expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx b/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx index 319f8c973f6..1ff77f565a5 100644 --- a/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx @@ -102,7 +102,7 @@ export default function TeamMemberTab({ return "0"; }; - const getUserCurrentCycleSpend = (userId: string | null): number => { + const getUserCurrentPeriodSpend = (userId: string | null): number => { if (!userId) return 0; return findMembership(userId)?.spend ?? 0; }; @@ -153,12 +153,16 @@ export default function TeamMemberTab({ return findMembership(userId)?.litellm_budget_table?.budget_reset_at ?? null; }; - const renderBudgetReset = (userId: string | null) => { + const getUserBudgetSchedule = (userId: string | null) => { const budget = findMembership(userId)?.litellm_budget_table; - const schedule = describeBudgetResetSchedule(budget?.budget_duration ?? null, budget?.budget_reset_at ?? null); + return describeBudgetResetSchedule(budget?.budget_duration ?? null, budget?.budget_reset_at ?? null); + }; + + const renderBudgetPeriod = (userId: string | null) => { + const schedule = getUserBudgetSchedule(userId); return ( - {schedule.label} + {schedule.period ?? schedule.label} ); }; @@ -200,22 +204,22 @@ export default function TeamMemberTab({ { title: ( - Current Cycle Spend (USD) - - + Current Period Spend (USD) + + ), key: "spend", numeric: true, - sortValue: (record: Member) => getUserCurrentCycleSpend(record.user_id), - render: (record: Member) => , + sortValue: (record: Member) => getUserCurrentPeriodSpend(record.user_id), + render: (record: Member) => , }, { title: ( Total Spend (USD) - + @@ -263,10 +267,10 @@ export default function TeamMemberTab({ }, }, { - title: "Budget Reset", + title: "Budget Period", key: "budget_reset", sortValue: (record: Member) => getUserBudgetReset(record.user_id), - render: (record: Member) => renderBudgetReset(record.user_id), + render: (record: Member) => renderBudgetPeriod(record.user_id), }, { title: ( @@ -314,10 +318,12 @@ export default function TeamMemberTab({ const resetSpendDisabledReason = (member: Member): string | null => { if (!isProxyAdmin && member.user_id === userId) return "Ask a proxy admin to reset your own spend"; - if (getUserCurrentCycleSpend(member.user_id) <= 0) return "No current cycle spend to reset"; + if (getUserCurrentPeriodSpend(member.user_id) <= 0) return "No current period spend to reset"; return null; }; + const resetSpendPeriod = getUserBudgetSchedule(memberToResetSpend?.user_id ?? null).period; + return ( <> Reset Team Member Spend

- Reset current cycle spend for{" "} + Reset current period spend for{" "} {memberToResetSpend?.user_email || memberToResetSpend?.user_id} in this team to{" "} $0?

- Current cycle spend:{" "} - ${formatNumberWithCommas(getUserCurrentCycleSpend(memberToResetSpend?.user_id ?? null), 4)} + Current period spend + {resetSpendPeriod ? ` (${resetSpendPeriod})` : ""}:{" "} + + ${formatNumberWithCommas(getUserCurrentPeriodSpend(memberToResetSpend?.user_id ?? null), 4)} + . This is the value checked against the member's budget. Total spend and logs are preserved.

diff --git a/ui/litellm-dashboard/src/utils/budgetUtils.test.ts b/ui/litellm-dashboard/src/utils/budgetUtils.test.ts index e33c779bb65..11339d08bc5 100644 --- a/ui/litellm-dashboard/src/utils/budgetUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/budgetUtils.test.ts @@ -2,6 +2,9 @@ import { afterEach, describe, expect, it } from "vitest"; import { budgetPeriodRange, describeBudgetResetSchedule, formatBudgetReset, formatBudgetResetUtc } from "./budgetUtils"; +const localIso = (year: number, monthIndex: number, day: number, hour = 12): string => + new Date(year, monthIndex, day, hour, 0, 0).toISOString(); + const originalTimezone: string | undefined = process.env.TZ; afterEach(() => { @@ -23,15 +26,18 @@ describe("formatBudgetResetUtc", () => { }); describe("describeBudgetResetSchedule", () => { - it("reports Never resets when the budget has no duration and no scheduled reset", () => { + it("reports lifetime spend with no period when the budget has no duration and no scheduled reset", () => { const schedule = describeBudgetResetSchedule(null, null); expect(schedule.label).toBe("Never resets"); expect(schedule.resetAt).toBeNull(); + expect(schedule.period).toBeNull(); + expect(schedule.tooltip).toContain("lifetime spend"); }); it("reports Unknown when a duration is set but no reset has been scheduled yet", () => { const schedule = describeBudgetResetSchedule("30d", null); expect(schedule.label).toBe("Unknown"); + expect(schedule.period).toBeNull(); expect(schedule.tooltip).toContain("every 30d"); }); @@ -41,7 +47,74 @@ describe("describeBudgetResetSchedule", () => { expect(schedule.label).toBe("Jul 15, 2026 12:00"); expect(schedule.resetAt).toBe(resetAt.toISOString()); expect(schedule.tooltip).toContain(Intl.DateTimeFormat().resolvedOptions().timeZone); - expect(schedule.tooltip).toContain("carries into the next cycle"); + expect(schedule.tooltip).toContain("carries into the next period"); + }); + + it("treats 30d as one calendar month back from the reset, like the proxy does", () => { + const schedule = describeBudgetResetSchedule("30d", localIso(2026, 10, 1, 0)); + expect(schedule.period).toBe("Oct 1 - Nov 1, 2026"); + expect(schedule.tooltip).toContain("covers Oct 1 - Nov 1, 2026 and resets every 30d"); + }); + + it("subtracts the exact duration for day and week windows", () => { + expect(describeBudgetResetSchedule("7d", localIso(2026, 6, 15)).period).toBe("Jul 8 - Jul 15, 2026"); + expect(describeBudgetResetSchedule("2w", localIso(2026, 6, 15)).period).toBe("Jul 1 - Jul 15, 2026"); + expect(describeBudgetResetSchedule("weekly", localIso(2026, 6, 15)).period).toBe("Jul 8 - Jul 15, 2026"); + }); + + it("walks the month back in UTC so a non-UTC viewer sees the proxy's window", () => { + const originalTz = process.env.TZ; + process.env.TZ = "America/New_York"; + try { + const schedule = describeBudgetResetSchedule("30d", "2026-03-01T00:00:00Z"); + expect(schedule.period).toBe("Jan 31 - Feb 28, 2026"); + } finally { + process.env.TZ = originalTz; + } + }); + + it("clamps month windows to the last day of a shorter month", () => { + expect(describeBudgetResetSchedule("1mo", localIso(2026, 2, 31)).period).toBe("Feb 28 - Mar 31, 2026"); + }); + + it("spells out both years when the period crosses a year boundary", () => { + expect(describeBudgetResetSchedule("30d", localIso(2027, 0, 10)).period).toBe("Dec 10, 2026 - Jan 10, 2027"); + }); + + it("keeps both clock times when an hourly window crosses midnight", () => { + expect(describeBudgetResetSchedule("1h", localIso(2026, 6, 16, 0)).period).toBe( + "Jul 15, 2026 23:00 - Jul 16, 2026 00:00", + ); + }); + + it("shows seconds for a window shorter than a minute", () => { + expect(describeBudgetResetSchedule("30s", localIso(2026, 6, 15)).period).toBe("Jul 15, 2026 11:59:30 - 12:00:00"); + }); + + it("shows clock times for a window that starts and ends on the same day", () => { + expect(describeBudgetResetSchedule("1h", localIso(2026, 6, 15)).period).toBe("Jul 15, 2026 11:00 - 12:00"); + }); + + it("falls back to a one day window for a duration the proxy would not recognise", () => { + expect(describeBudgetResetSchedule("fortnightly", localIso(2026, 6, 15)).period).toBe("Jul 14 - Jul 15, 2026"); + }); + + it("shows no period when the reset date is unparseable", () => { + const schedule = describeBudgetResetSchedule("30d", "not-a-date"); + expect(schedule.label).toBe("Unknown"); + expect(schedule.period).toBeNull(); + expect(schedule.tooltip).not.toMatch(/NaN|Invalid/); + }); + + it("never describes the window as a cycle", () => { + const schedules = [ + describeBudgetResetSchedule("30d", localIso(2026, 6, 15)), + describeBudgetResetSchedule("30d", null), + describeBudgetResetSchedule(null, null), + ]; + for (const schedule of schedules) { + expect(`${schedule.label} ${schedule.tooltip}`).not.toMatch(/cycle/i); + } }); }); diff --git a/ui/litellm-dashboard/src/utils/budgetUtils.ts b/ui/litellm-dashboard/src/utils/budgetUtils.ts index 23af15eb539..71638551f9f 100644 --- a/ui/litellm-dashboard/src/utils/budgetUtils.ts +++ b/ui/litellm-dashboard/src/utils/budgetUtils.ts @@ -1,4 +1,4 @@ -import dayjs from "dayjs"; +import dayjs, { Dayjs } from "dayjs"; import utc from "dayjs/plugin/utc"; dayjs.extend(utc); @@ -20,10 +20,55 @@ export function formatBudgetResetUtc(iso: string | null | undefined): string | n export interface BudgetResetSchedule { label: string; resetAt: string | null; + period: string | null; tooltip: string; } -const ROLLOVER_NOTE = "Spend above a rollover cap at reset carries into the next cycle."; +const ROLLOVER_NOTE = "Spend above a rollover cap at reset carries into the next period."; + +const DURATION_WORD_ALIASES: Record = { + hourly: "1h", + daily: "24h", + weekly: "7d", + monthly: "30d", +}; + +const DURATION_PATTERN = /^(\d+)(mo|[smhdw])$/; + +// Mirrors the proxy's get_budget_window_start: "30d" and "Nmo" span calendar months, +// everything else is a fixed length, and an unrecognised duration resets at the next midnight. +export function getBudgetPeriodStart(budgetDuration: string, resetAt: Dayjs): Dayjs { + const trimmed = budgetDuration.trim(); + const normalized = DURATION_WORD_ALIASES[trimmed.toLowerCase()] ?? trimmed; + const match = DURATION_PATTERN.exec(normalized); + if (!match) return resetAt.subtract(1, "day"); + const value = Number(match[1]); + switch (match[2]) { + case "mo": + return resetAt.subtract(value, "month"); + case "d": + return value === 30 ? resetAt.subtract(1, "month") : resetAt.subtract(value, "day"); + case "w": + return resetAt.subtract(value, "week"); + case "h": + return resetAt.subtract(value, "hour"); + case "m": + return resetAt.subtract(value, "minute"); + default: + return resetAt.subtract(value, "second"); + } +} + +export function formatBudgetPeriod(start: Dayjs, end: Dayjs): string { + const windowSeconds = end.diff(start, "second"); + if (windowSeconds < 24 * 60 * 60) { + const time = windowSeconds < 60 ? "HH:mm:ss" : "HH:mm"; + const endFormat = start.isSame(end, "day") ? time : `MMM D, YYYY ${time}`; + return `${start.format(`MMM D, YYYY ${time}`)} - ${end.format(endFormat)}`; + } + if (start.isSame(end, "year")) return `${start.format("MMM D")} - ${end.format("MMM D, YYYY")}`; + return `${start.format("MMM D, YYYY")} - ${end.format("MMM D, YYYY")}`; +} export function describeBudgetResetSchedule( budgetDuration: string | null | undefined, @@ -31,26 +76,33 @@ export function describeBudgetResetSchedule( ): BudgetResetSchedule { const resetDate = budgetResetAt ? dayjs(budgetResetAt) : null; if (resetDate?.isValid()) { - const cadence = budgetDuration ? `every ${budgetDuration}` : "on a schedule"; const timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone; const label = resetDate.format("MMM D, YYYY HH:mm"); + const period = budgetDuration + ? formatBudgetPeriod(getBudgetPeriodStart(budgetDuration, resetDate.utc()).local(), resetDate) + : null; + const coverage = period + ? `Current Period Spend covers ${period} and resets every ${budgetDuration}.` + : "Current Period Spend resets on a schedule."; return { label, resetAt: resetDate.toISOString(), - tooltip: `Current Cycle Spend resets ${cadence}. Next reset ${label} (${timeZone}). ${ROLLOVER_NOTE}`, + period, + tooltip: `${coverage} Next reset ${label} (${timeZone}). ${ROLLOVER_NOTE}`, }; } if (!budgetDuration) { return { label: "Never resets", resetAt: null, - tooltip: - "This budget has no reset window, so Current Cycle Spend accumulates for as long as the membership exists.", + period: null, + tooltip: "This budget has no reset window, so this is lifetime spend for as long as the membership exists.", }; } return { label: "Unknown", resetAt: null, + period: null, tooltip: `Resets every ${budgetDuration}, but the next reset time has not been scheduled yet. ${ROLLOVER_NOTE}`, }; } From b5befe2dd0bb0899e5df8f05a20e9840609ada25 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:54:32 -0700 Subject: [PATCH 105/155] test(routing): a team's wildcard serves its keys before a proxy-wide wildcard of the same pattern (#48501) Co-authored-by: joshua Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../routing/test_team_wildcard_precedence.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 tests/integration/routing/test_team_wildcard_precedence.py diff --git a/tests/integration/routing/test_team_wildcard_precedence.py b/tests/integration/routing/test_team_wildcard_precedence.py new file mode 100644 index 00000000000..38fda682129 --- /dev/null +++ b/tests/integration/routing/test_team_wildcard_precedence.py @@ -0,0 +1,61 @@ +"""A team's own wildcard deployment serves that team's keys even when a proxy-wide wildcard of the +same pattern exists, and the proxy-wide one serves every other key (#48490).""" + +import uuid +from typing import Final + +import pytest +from integration._support.client import Gateway, Scenario + +pytestmark: Final = pytest.mark.timeout(120) + + +def _wildcard(scenario: Scenario, pattern: str, *, team_id: str | None) -> str: + identity: Final = f"wildcard-{uuid.uuid4().hex}" + scenario.gateway.post( + "/model/new", + { + "model_name": pattern, + "litellm_params": { + "model": "openai/*", + "api_key": f"provider-key-{identity}", + "api_base": f"{scenario.gateway.upstream_url}/v1", + }, + "model_info": {"id": identity, **({"team_id": team_id} if team_id is not None else {})}, + }, + ) + scenario.cleanups.callback(scenario.delete_model, identity) + return identity + + +def _served_by(gateway: Gateway, key: str, model: str) -> str: + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "wildcard precedence control"}]}, + key=key, + ) + assert response.status_code == 200, response.text + return response.headers["x-litellm-model-id"] + + +def test_a_teams_wildcard_serves_its_keys_before_a_proxy_wide_wildcard_of_the_same_pattern(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + byok_team: Final = scenario.team(models=[]) + other_team: Final = scenario.team(models=[]) + pattern: Final = f"byok-{uuid.uuid4().hex}/*" + team_wildcard: Final = _wildcard(scenario, pattern, team_id=byok_team) + shared_wildcard: Final = _wildcard(scenario, pattern, team_id=None) + model: Final = pattern.replace("*", "gpt-4o-mini") + + served: Final = { + "byok team key": _served_by(gateway, scenario.key(team_id=byok_team), model), + "other team key": _served_by(gateway, scenario.key(team_id=other_team), model), + "key without a team": _served_by(gateway, scenario.key(), model), + } + + assert served == { + "byok team key": team_wildcard, + "other team key": shared_wildcard, + "key without a team": shared_wildcard, + } From 715e9bb08d7567d58beebff640a7b32c4370342d Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 26 Oct 2026 17:57:06 -0700 Subject: [PATCH 106/155] fix(prometheus): count provider-side 401s in LLM failure and deployment metrics (#46668) * fix(prometheus): count provider-side 401s in LLM failure and deployment metrics The invalid virtual key skip no longer applies to an error the LLM provider returned while the deployment was called with its own credential, so a 401 for a bad deployment credential increments litellm_llm_api_failed_requests_metric, litellm_deployment_failure_responses and litellm_deployment_total_requests and marks litellm_deployment_state as a partial outage again. Proxy-gate rejections and a failing credential that belongs to the caller (a rejected per-user credential, an api_key sent with the request, a per-user GitHub Copilot session) keep the skip * fix(prometheus): keep a caller's own Anthropic OAuth token failure out of the deployment metrics A provider 401 on an Anthropic OAuth token the caller forwarded through the proxy is the caller's credential failing, so it stays out of the deployment failure and outage series like a caller-supplied api_key. Adds integration cells for an Anthropic deployment's own key, the forwarded caller OAuth token and a direct SDK call that brings its own key * test(prometheus): pin how a caller's own key or Authorization header on a deployment is counted An SDK Router call that passes its own api_key is counted against the one-off deployment the router builds for that key, so the operator deployment keeps only its own failure. A provider 401 on an Authorization header the caller sends through the proxy counts against the deployment, as on v1.80.12 * fix(prometheus): keep caller-sent provider credentials and token-exchange 401s out of deployment metrics A caller can send its own provider credential in the request body, such as azure_ad_token, azure_scope, the Azure Entra app fields, watsonx token, gigachat_access_token, OCI keys, AWS keys or Vertex credentials, at the top level or inside extra_body. A provider 401 on that credential is the caller's failure, so it stays out of the shared deployment's failure, request, LLM-failed and state series, as on main. A token-exchange deployment trades the caller's identity token on every call, so its 401s stay out as well, matching the router cooldown. * test(prometheus): Final locals, precise helper types and one for per comprehension in the new test helpers * test(prometheus): pin that a provider 401 at a caller-sent api_base counts against its one-off deployment * fix(prometheus): keep a provider 401 on caller-sent headers out of deployment metrics A caller can override the deployment's credential with headers it sends in the request body: headers or provider_specific_header, or an Authorization, api-key, x-api-key, x-goog-api-key or Ocp-Apim-Subscription-Key entry in extra_headers. A provider 401 on such a request is the caller's failure, so it stays out of the shared deployment's failure, request, LLM-failed and state series, as on main. * fix(prometheus): skip a provider 401 only when the caller's headers carry a credential The proxy now records headers, extra_headers or provider_specific_header in credential_fields only when they name a provider credential header (Authorization, api-key, x-api-key, x-goog-api-key, Ocp-Apim-Subscription-Key), the same way api_key is recorded. A caller that sends a plain header such as a trace id still has a provider 401 on the deployment's own key counted against the deployment. * fix(prometheus): stamp credential_fields after the proxy forwards the caller's headers A provider credential header the proxy forwards from the caller's HTTP request (forward_llm_provider_auth_headers + forward_client_headers_to_llm_api) lands in data["headers"] after proxy_server_request is built, so the stamp missed it and a 401 on that caller credential was counted against the deployment. Compute the stamp after the header forwarding steps and pin both forwarded cases. * fix(prometheus): skip a provider 401 when the caller sends any header or an OpenAI organization The proxy now records headers, extra_headers or provider_specific_header in credential_fields whenever the caller sends or the proxy forwards any header there, and records organization when the caller sends one in the body or as a forwarded OpenAI-Organization header. A forwarded header can be a gateway key such as x-portkey-api-key or an OpenAI-Project id, and a wrong organization gets a 401 from OpenAI, so these 401s stay out of the deployment metrics as on main instead of marking a healthy deployment as a partial outage. * fix(prometheus): ignore the user information the proxy adds when stamping caller headers With add_user_information_to_llm_headers on and header forwarding enabled, the proxy adds x-litellm-user_api_key_* headers to the forwarded headers. Those carry the proxy's own key and user details, not the caller's, so they no longer make a request look caller-scoped, and a provider 401 on the deployment's own key on such a request is counted. * fix(prometheus): skip a 401 raised before the provider is called or at the caller's own endpoint A deployment with no key of its own fails with LiteLLM's local "Missing Anthropic API Key" 401 before any provider call, and a caller who sends aws_bedrock_runtime_endpoint (or another provider endpoint field) can get a 401 from their own gateway. Both were counted against the shared deployment and marked it as a partial outage, where main skipped every 401. Count a 401 only once the provider was called (api_call_start_time is set), and treat caller-sent provider endpoint fields like caller credential fields. * fix(prometheus): count the deployment's own 401 on requests that carry only anthropic-version or anthropic-beta The Anthropic SDK sends anthropic-version on every request, and the proxy copies anthropic-version and anthropic-beta into provider_specific_header for Anthropic, Bedrock and Vertex. Both were treated as caller headers, so a 401 on an Anthropic deployment's own key from any Anthropic SDK client stayed out of the deployment metrics. These two headers only pick the API version and beta features. Anthropic answers a bad anthropic-version or an unknown anthropic-beta with a 400, never a 401, so they cannot turn the deployment's own key into a 401 and no longer mark the request as carrying a caller header. * test(prometheus): read and write the owned proxy's own database in the expired-key and MCP cells Since #47213 an owned proxy boots on a private database, so the expired-key update and the MCP guardrail row read now go through the gateway's database_url instead of the worker's. * fix(prometheus): keep credential_fields as main stamps it and give Prometheus its own caller_credential_fields credential_fields tells the Rust OCR bridge which stripped body fields the caller sent. Repurposing it for the Prometheus caller-credential check changed its value (organization and extra_headers added, anthropic-version-only headers dropped). Restore main's stamp unchanged and stamp the Prometheus classification under caller_credential_fields, which only Prometheus reads. * fix(prometheus): skip a provider 401 on a stored credential the caller picks by litellm_credential_name A request body litellm_credential_name swaps the deployment's stored credential for one the caller chose, so a 401 on it is the caller's, not the deployment's. Treat it like the other caller credential fields so it no longer marks a healthy shared deployment as a partial outage. --------- Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- litellm/integrations/prometheus.py | 97 +- litellm/proxy/litellm_pre_call_utils.py | 45 + .../copilot_session_prometheus_worker.py | 162 +++ .../test_prometheus_provider_auth_failure.py | 1074 +++++++++++++++++ ..._prometheus_provider_auth_failure_chaos.py | 147 +++ ...etheus_copilot_session_auth_failure_sdk.py | 99 ++ .../test_prometheus_invalid_key_filtering.py | 441 ++++++- .../unit/proxy/test_litellm_pre_call_utils.py | 216 ++++ 8 files changed, 2220 insertions(+), 61 deletions(-) create mode 100644 tests/integration/_support/copilot_session_prometheus_worker.py create mode 100644 tests/integration/observability/test_prometheus_provider_auth_failure.py create mode 100644 tests/integration/observability/test_prometheus_provider_auth_failure_chaos.py create mode 100644 tests/integration/sdk/test_prometheus_copilot_session_auth_failure_sdk.py diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 19d161b8aa4..bc8344fa686 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -14,6 +14,7 @@ from functools import partial from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast +import openai from pydantic import BaseModel, Field, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict, assert_never @@ -22,11 +23,13 @@ from litellm._internal_context import with_service_target from litellm._logging import print_verbose, verbose_logger from litellm.constants import ( AUTH_OBJECTS_TARGET, + LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, PROMETHEUS_OVERFLOW_SERIES_LABEL_VALUE, PROXY_LLM_PROVIDER_FALLBACK, PROXY_REJECTED_BEFORE_ROUTING_KEY, ) from litellm.exceptions import ( + CallerCredentialAuthenticationError, MalformedRequestFieldError, validate_rate_limit_category, validate_rate_limit_type, @@ -51,6 +54,7 @@ from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_cust from litellm.litellm_core_utils.service_tier_utils import ( get_service_tier_from_standard_logging_payload, ) +from litellm.llms.github_copilot.per_user_auth import github_copilot_user_session_from from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.models.end_user import LiteLLM_EndUserTable from litellm.models.team import LiteLLM_TeamTable @@ -269,7 +273,10 @@ _NO_LOGGED_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) def _logged_fields(value: object) -> Mapping[str, object]: - return _LOGGED_FIELDS.validate_python(value) if isinstance(value, Mapping) else _NO_LOGGED_FIELDS + try: + return _LOGGED_FIELDS.validate_python(value) + except ValidationError: + return _NO_LOGGED_FIELDS def _is_llm_provider(name: str) -> bool: @@ -359,6 +366,87 @@ def _get_proxy_llm_router() -> Router | None: return llm_router +_CALLER_CREDENTIAL_FIELDS: Final = frozenset( + { + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "aws_web_identity_token", + "azure_ad_token", + "azure_password", + "azure_scope", + "azure_username", + "client_id", + "client_secret", + "gigachat_access_token", + "litellm_credential_name", + "oci_fingerprint", + "oci_key", + "oci_key_file", + "oci_tenancy", + "oci_user", + "tenant_id", + "token", + "user_config", + "vertex_ai_credentials", + "vertex_credentials", + "watsonx_credentials", + "wx_credentials", + "zen_api_key", + } +) + + +_CALLER_ENDPOINT_FIELDS: Final = frozenset( + {"aws_bedrock_runtime_endpoint", "aws_sts_endpoint", "deployment_url", "s3_endpoint_url", "sagemaker_base_url"} +) + + +_CALLER_TRANSPORT_FIELDS: Final = frozenset( + {"api_key", "extra_headers", "headers", "organization", "provider_specific_header"} +) + + +def _sent_a_provider_credential_or_endpoint(caller_body: Mapping[str, object]) -> bool: + """True when the caller's request body, or its ``extra_body``, carries a provider credential field or a + provider endpoint field.""" + sent: Final = {*caller_body, *_logged_fields(caller_body.get("extra_body"))} + return not (_CALLER_CREDENTIAL_FIELDS.isdisjoint(sent) and _CALLER_ENDPOINT_FIELDS.isdisjoint(sent)) + + +def _used_a_caller_credential(logged_call: Mapping[str, object]) -> bool: + """True when the provider was called with a credential or at an endpoint that belongs to the caller rather than + the deployment: an api_key, header, OpenAI organization or other provider credential or endpoint field the caller + sent with the request, a stored credential the caller picked by name, the caller's own Anthropic OAuth token, the caller's own GitHub Copilot connection, or the + caller's identity token that a token-exchange deployment trades in on every call.""" + litellm_params: Final = _logged_fields(logged_call.get("litellm_params")) + proxy_server_request: Final = _logged_fields(litellm_params.get("proxy_server_request")) + caller_credential_fields: Final = proxy_server_request.get("caller_credential_fields") + logged_metadata: Final = _logged_fields(_logged_fields(logged_call.get("standard_logging_object")).get("metadata")) + return ( + ( + isinstance(caller_credential_fields, tuple) + and any(field in caller_credential_fields for field in _CALLER_TRANSPORT_FIELDS) + ) + or _sent_a_provider_credential_or_endpoint(_logged_fields(proxy_server_request.get("body"))) + or bool(litellm_params.get("token_exchange_endpoint")) + or logged_metadata.get("used_client_oauth_token") is True + or github_copilot_user_session_from(litellm_params) is not None + ) + + +def _is_deployment_failure(exception: openai.APIError, logged_call: Mapping[str, object]) -> bool: + """True for an error raised while calling the deployment with its own credential, on a request the proxy + did not reject itself. An error LiteLLM raises before it calls the provider, such as a missing deployment key, + is not one.""" + return not ( + isinstance(exception, CallerCredentialAuthenticationError) + or bool(logged_call.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)) + or logged_call.get("api_call_start_time") is None + or _used_a_caller_credential(logged_call) + ) + + def _bounded_requested_model_label(requested_model: object, router_originated: bool = False) -> str | None: """ Bound ``requested_model`` label cardinality: names the router recognizes @@ -2817,6 +2905,8 @@ class PrometheusLogger(CustomLogger): information from various callback function signatures and determines if the request represents an invalid API key attempt that should be filtered from metrics. + An error raised while calling the deployment with its own credential is never skipped. + Args: kwargs: Dictionary potentially containing exception and other data user_api_key_dict: User API key authentication object (currently unused) @@ -2836,6 +2926,11 @@ class PrometheusLogger(CustomLogger): if exception is None and kwargs: exception = kwargs.get("exception") + if isinstance(exception, openai.APIError) and _is_deployment_failure( + exception, _LOGGED_FIELDS.validate_python(kwargs or {}) + ): + return False + if self._is_invalid_api_key_request(status_code, exception=exception): verbose_logger.debug( "Skipping Prometheus metrics for invalid API key request: status_code=%s, exception=%s", diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index b51eea7a876..5f1948d1c1c 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -99,6 +99,8 @@ _CREDENTIAL_HEADER_NAMES: Final = SpecialHeaders.litellm_credential_header_names {"cookie", "proxy-authorization"} ) _TRANSPORT_ONLY_CREDENTIAL_KEYS: Final = frozenset({"provider_specific_header", "headers", "api_key"}) +_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object]) +_REQUEST_ENTRIES: Final = TypeAdapter(tuple[object, ...]) # Matches any header of the form x--session-id (case-insensitive). # Excludes the two explicit litellm headers which are handled with higher priority. @@ -289,6 +291,9 @@ _DESTINATION_ONLY_CALLBACK_VARS: Final[frozenset[str]] = frozenset({CAPTURE_MESS _CALLBACK_CREDENTIAL_KEYS: Final = frozenset(StandardCallbackDynamicParams.__annotations__) | frozenset( {TRUSTED_CALLBACK_VARS_FIELD} ) +_HEADERS_THAT_CANNOT_CHANGE_PROVIDER_AUTH: Final = frozenset( + {*(f"x-litellm-{key}" for key in StandardLoggingUserAPIKeyMetadata.__annotations__), *ANTHROPIC_API_HEADERS} +) service_logger_obj: Final = ServiceLogging() # used for tracking latency on OTEL # Bounded dedup for stale-alias warnings (FIFO eviction when over cap). _MAX_STALE_ALIAS_WARNING_KEYS: Final = 10_000 @@ -1451,6 +1456,44 @@ def clean_headers( return clean_headers +def _request_fields(value: object) -> Mapping[str, object]: + try: + return _REQUEST_FIELDS.validate_python(value) + except PydanticValidationError: + return MappingProxyType({}) + + +def _request_entries(value: object) -> tuple[object, ...]: + if value is None: + return () + try: + return _REQUEST_ENTRIES.validate_python(value) + except PydanticValidationError: + return (value,) + + +def _carries_a_caller_header(headers: object) -> bool: + return any(name.lower() not in _HEADERS_THAT_CANNOT_CHANGE_PROVIDER_AUTH for name in _request_fields(headers)) + + +def _caller_credential_fields(data: Mapping[str, object]) -> tuple[str, ...]: + """Names of the request fields through which the caller can change how the provider authenticates the call: + ``api_key``, ``organization``, and ``headers``, ``extra_headers`` or a ``provider_specific_header`` entry's + ``extra_headers`` that carry any header besides the key and user details the proxy adds under + ``add_user_information_to_llm_headers`` and the ``anthropic-version`` and ``anthropic-beta`` headers.""" + carried: Final = { + "api_key": "api_key" in data, + "extra_headers": _carries_a_caller_header(data.get("extra_headers")), + "headers": _carries_a_caller_header(data.get("headers")), + "organization": "organization" in data, + "provider_specific_header": any( + _carries_a_caller_header(_request_fields(entry).get("extra_headers")) + for entry in _request_entries(data.get("provider_specific_header")) + ), + } + return tuple(name for name, sent in sorted(carried.items()) if sent) + + def _is_credential_header(header: str) -> bool: """Whether `header` carries a caller credential rather than request context.""" return header.lower().removeprefix("x-pass-") in _CREDENTIAL_HEADER_NAMES @@ -2605,6 +2648,8 @@ async def add_litellm_data_to_request( data[_metadata_variable_name]["used_client_oauth_token"] = add_provider_specific_headers_to_request( data=data, headers=_headers ) + caller_fields: Final = _caller_credential_fields(_REQUEST_FIELDS.validate_python(data)) + data["proxy_server_request"]["caller_credential_fields"] = caller_fields # rebind-ok: data is an out-param ## Cache Controls cache_control_header: Final = _headers.get("Cache-Control", None) diff --git a/tests/integration/_support/copilot_session_prometheus_worker.py b/tests/integration/_support/copilot_session_prometheus_worker.py new file mode 100644 index 00000000000..a6b2c5d4692 --- /dev/null +++ b/tests/integration/_support/copilot_session_prometheus_worker.py @@ -0,0 +1,162 @@ +"""One SDK process for the SDK provider-401 metric cells: a provider 401 on a caller's own Copilot session, one on +the router deployment's own key, one on a key the router caller passes and one on a direct call's own key, then a +report of the deployment series on that process's default registry. Run as a script by ``run_copilot_session_worker`` so each process registers the +Prometheus metrics exactly once.""" + +from __future__ import annotations + +import asyncio +import json +import subprocess +import sys +from itertools import chain +from dataclasses import dataclass +from typing import Final + +import litellm +import openai +from litellm.integrations.prometheus import PrometheusLogger +from litellm.llms.github_copilot.per_user_auth import GithubCopilotUserSession +from prometheus_client import REGISTRY +from pydantic import TypeAdapter + +COPILOT_DEPLOYMENT: Final = "copilot-per-user-deployment" +OPERATOR_DEPLOYMENT: Final = "operator-key-deployment" +CALLER_SESSION_TOKEN: Final = "caller-copilot-session-token" +OPERATOR_KEY: Final = "rotated-operator-key" +ROUTER_CALLER_KEY: Final = "rotated-router-caller-key" +DIRECT_CALL: Final = "direct-sdk-call" +DIRECT_KEY: Final = "rotated-direct-call-key" +FAILURES: Final = "litellm_deployment_failure_responses_total" +TOTALS: Final = "litellm_deployment_total_requests_total" +LLM_FAILED: Final = "litellm_llm_api_failed_requests_metric_total" +STATE: Final = "litellm_deployment_state" +DEPLOYMENT_SERIES: Final = frozenset({FAILURES, TOTALS, LLM_FAILED, STATE}) +WORKER_TIMEOUT_SECONDS: Final = 120 + + +@dataclass(frozen=True, slots=True) +class Outcome: + deployment: str + status: int | None + + +@dataclass(frozen=True, slots=True) +class Series: + metric: str + model_id: str + api_base: str + value: float + + +@dataclass(frozen=True, slots=True) +class WorkerReport: + api_base: str + outcomes: tuple[Outcome, ...] + series: tuple[Series, ...] + + def values(self, metric: str, model_id: str) -> tuple[float, ...]: + return tuple(s.value for s in self.series if s.metric == metric and s.model_id == model_id) + + def on_the_endpoint(self, metric: str, model_id: str) -> tuple[float, ...]: + return tuple( + s.value for s in self.series if (s.metric, s.model_id, s.api_base) == (metric, model_id, self.api_base) + ) + + def outside_the_router(self, metric: str) -> tuple[float, ...]: + return tuple(s.value for s in self.series if s.metric == metric and not s.model_id) + + def on_a_one_off_deployment(self, metric: str) -> tuple[float, ...]: + named_ids: Final = ("", COPILOT_DEPLOYMENT, OPERATOR_DEPLOYMENT) + return tuple(s.value for s in self.series if s.metric == metric and s.model_id not in named_ids) + + +_REPORT: Final = TypeAdapter(WorkerReport) + + +def run_copilot_session_worker(api_base: str) -> WorkerReport: + completed: Final = subprocess.run( + (sys.executable, "-P", __file__, api_base), + capture_output=True, + text=True, + timeout=WORKER_TIMEOUT_SECONDS, + check=False, + ) + assert completed.returncode == 0, completed.stderr[-3000:] + return _REPORT.validate_json(completed.stdout.splitlines()[-1]) + + +def _deployment_series() -> tuple[Series, ...]: + samples: Final = tuple(chain.from_iterable(family.samples for family in REGISTRY.collect())) + return tuple( + Series( + metric=sample.name, + model_id=str(sample.labels.get("model_id", "")), + api_base=str(sample.labels.get("api_base", "")), + value=float(sample.value), + ) + for sample in samples + if sample.name in DEPLOYMENT_SERIES + ) + + +def _router(api_base: str) -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": COPILOT_DEPLOYMENT, + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-per-user", + "github_copilot_auth_type": "per_user_oauth", + }, + "model_info": {"id": COPILOT_DEPLOYMENT}, + }, + { + "model_name": OPERATOR_DEPLOYMENT, + "litellm_params": {"model": "openai/gpt-5.6", "api_base": api_base, "api_key": OPERATOR_KEY}, + "model_info": {"id": OPERATOR_DEPLOYMENT}, + }, + ], + num_retries=0, + disable_cooldowns=True, + ) + + +async def _attempt(router: litellm.Router, model: str, **extra: object) -> Outcome: + try: + await router.acompletion(model=model, messages=[{"role": "user", "content": f"{model} worker"}], **extra) + except openai.APIStatusError as error: + return Outcome(deployment=model, status=error.status_code) + return Outcome(deployment=model, status=None) + + +async def _direct_attempt(api_base: str) -> Outcome: + try: + await litellm.acompletion( + model="openai/gpt-5.6", + api_base=api_base, + api_key=DIRECT_KEY, + messages=[{"role": "user", "content": f"{DIRECT_CALL} worker"}], + ) + except openai.APIStatusError as error: + return Outcome(deployment=DIRECT_CALL, status=error.status_code) + return Outcome(deployment=DIRECT_CALL, status=None) + + +async def main(api_base: str) -> None: + litellm.callbacks = [PrometheusLogger()] # test-quality-ok: this worker process exits right after the report + session: Final = GithubCopilotUserSession(token=CALLER_SESSION_TOKEN, api_base=api_base) + router: Final = _router(api_base) + outcomes: Final = ( + await _attempt(router, COPILOT_DEPLOYMENT, github_copilot_user_session=session), + await _attempt(router, OPERATOR_DEPLOYMENT), + await _attempt(router, OPERATOR_DEPLOYMENT, api_key=ROUTER_CALLER_KEY), + await _direct_attempt(api_base), + ) + report: Final = WorkerReport(api_base=api_base, outcomes=outcomes, series=_deployment_series()) + sys.stdout.write(json.dumps(_REPORT.dump_python(report)) + "\n") + + +if __name__ == "__main__": + asyncio.run(main(sys.argv[1])) diff --git a/tests/integration/observability/test_prometheus_provider_auth_failure.py b/tests/integration/observability/test_prometheus_provider_auth_failure.py new file mode 100644 index 00000000000..13d264ca6b0 --- /dev/null +++ b/tests/integration/observability/test_prometheus_provider_auth_failure.py @@ -0,0 +1,1074 @@ +from __future__ import annotations + +import asyncio +import json +import threading +import uuid +from collections import Counter +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path +from typing import Final + +import httpx +import openai +import pytest +import yaml +from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment +from integration._support.database import write_rows +from integration._support.mcp import ENTRY_POINTS, McpCaller, McpPeer, echo_tool, register_mcp, scripted_peer +from integration._support.process import owned_proxy +from integration._support.prometheus_series import Sample, scrape +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +RUN: Final = uuid.uuid4().hex[:8] +REJECTS: Final = "provider-rejects-credential" +HELD_REJECTS: Final = "provider-rejects-credential-when-released" +KEY_NOT_VALID: Final = "provider-key-not-valid" +RATE_LIMITS: Final = "provider-rate-limits" +RECOVERS: Final = "provider-rejects-once" +HEALTHY: Final = "provider-healthy" +AGENT_365_GUARDRAIL: Final = f"agent-365-{RUN}" +CALLER_OAUTH_TOKEN: Final = f"sk-ant-oat01-caller-{RUN}" +REVOKED_GATEWAY_KEY: Final = f"revoked-gateway-key-{RUN}" +UNKNOWN_ORGANIZATION: Final = f"org-unknown-{RUN}" +REVOKED_STORED_KEY: Final = f"sk-revoked-stored-credential-{RUN}" +AZURE_API_VERSION: Final = "2024-02-01" +CALLER_AZURE_CELLS: Final = ( + ("chat", False), + ("embeddings", False), + ("responses", False), + ("messages", False), + ("responses", True), +) +CALLER_CREDENTIAL_CELLS: Final = ( + ("caller-watsonx-token", "token"), + ("caller-gigachat-access-token", "gigachat_access_token"), +) +TOKEN_EXCHANGE_CELLS: Final = ("token-exchange", "token-exchange-credential") +KEYLESS_CELLS: Final = ("keyless", "messages-keyless") +TOKEN_EXCHANGE_SETTINGS: Final = { + "token_exchange_endpoint": "https://login.microsoftonline.com/00000000-0000-4000-8000-0000000c0b01/oauth2/v2.0/token", + "client_id": "synthetic-obo-client-id", + "client_secret": "synthetic-obo-client-secret", +} + +FAILURES: Final = "litellm_deployment_failure_responses_total" +TOTALS: Final = "litellm_deployment_total_requests_total" +LLM_FAILED: Final = "litellm_llm_api_failed_requests_metric_total" +STATE: Final = "litellm_deployment_state" +PROXY_FAILED: Final = "litellm_proxy_failed_requests_metric_total" +SUCCESSES: Final = "litellm_deployment_success_responses_total" +DEPLOYMENT_SERIES: Final = (FAILURES, TOTALS, LLM_FAILED, STATE) + +ENDPOINT_CELLS: Final = ( + "chat", + "chat-stream", + "messages", + "messages-stream", + "responses", + "responses-stream", + "embeddings", +) +REJECTED_CELLS: Final = ( + *ENDPOINT_CELLS, + "sdk-sync", + "sdk-async", + "repeated", + "fallback", + "burst", + "caller-api-key", + "caller-extra-headers", + "caller-headers", + "caller-provider-specific-header", + "plain-extra-headers", + "plain-headers", + "plain-provider-specific-header", + "forwarded-provider-header", + "forwarded-plain-header", + "forwarded-user-information", +) +GATED_CELLS: Final = ("invalid-key", "missing-key", "expired-key", "blocked-key", "no-model-access") +CALLER_SCOPE_CELLS: Final = ("forwarded-gateway-key", "caller-organization", "forwarded-organization") +HEALTHY_CELLS: Final = (*GATED_CELLS, *CALLER_SCOPE_CELLS, "healthy", "key-rpm-limit") +GUARDRAIL_ROWS: Final = ( + "SELECT COALESCE(jsonb_path_query_first(metadata, " + "'$.guardrail_information[*] ? (@.guardrail_name == $name).guardrail_status', " + "jsonb_build_object('name', %s::text)) #>> '{}', 'none') AS status " + 'FROM "LiteLLM_SpendLogs" WHERE api_key = %s AND call_type = %s' +) + + +def name(cell: str) -> str: + return f"prometheus-{cell}-{RUN}" + + +def identity(cell: str) -> str: + return f"{name(cell)}-deployment" + + +def _error(status: int, message: str, kind: str, code: str) -> Reply: + return Reply(status=status, body=json.dumps({"error": {"message": message, "type": kind, "code": code}}).encode()) + + +def _completion() -> Reply: + return Reply( + body=json.dumps( + { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + ).encode() + ) + + +class Provider: + """Answers by upstream model name. `HELD_REJECTS` waits for `release` before rejecting the credential.""" + + def __init__(self, release: threading.Event) -> None: + self._lock: Final = threading.Lock() + self._seen: Final[Counter[str]] = Counter() + self._release: Final = release + + def __call__(self, request: Request) -> Reply: + sent: Final = json.loads(request.body) + model: Final = str(sent.get("model") or sent.get("model_id") or request.target.split("/")[2]) + if request.headers.get("x-portkey-api-key") == REVOKED_GATEWAY_KEY: + return _error(401, "Invalid gateway API key", "invalid_request_error", "invalid_api_key") + if request.headers.get("authorization") == f"Bearer {REVOKED_STORED_KEY}": + return _error(401, "Incorrect API key provided: sk-revok*****", "invalid_request_error", "invalid_api_key") + if request.headers.get("openai-organization") == UNKNOWN_ORGANIZATION: + return _error( + 401, + "OpenAI-Organization header should match organization for API key", + "invalid_request_error", + "mismatched_organization", + ) + with self._lock: + self._seen[model] += 1 + attempt: Final = self._seen[model] + if model == HELD_REJECTS: + assert self._release.wait(timeout=30), "held provider reply was never released" + if model in (REJECTS, HELD_REJECTS) or (model == RECOVERS and attempt == 1): + return _error(401, "Incorrect API key provided: sk-rotat*****", "invalid_request_error", "invalid_api_key") + if model == KEY_NOT_VALID: + return _error(400, "API key not valid. Please pass a valid API key.", "invalid_request_error", "400") + if model == RATE_LIMITS: + return _error(429, "Rate limit reached for requests", "rate_limit_error", "rate_limit_exceeded") + return _completion() + + +def deployment(cell: str, upstream_model: str, api_base: str) -> dict[str, JsonValue]: + return { + "model_name": name(cell), + "litellm_params": { + "model": f"openai/{upstream_model}", + "api_key": f"sk-deployment-credential-{cell}", + "api_base": api_base, + }, + "model_info": {"id": identity(cell)}, + } + + +def anthropic_deployment(cell: str, api_base: str) -> dict[str, JsonValue]: + return { + "model_name": name(cell), + "litellm_params": { + "model": f"anthropic/{REJECTS}", + "api_key": f"sk-ant-api03-deployment-credential-{cell}", + "api_base": api_base, + }, + "model_info": {"id": identity(cell)}, + } + + +def bedrock_deployment(cell: str, opt_in: dict[str, JsonValue]) -> dict[str, JsonValue]: + return { + "model_name": name(cell), + "litellm_params": { + "model": f"bedrock/converse/{REJECTS}", + "aws_access_key_id": "AKIADEPLOYMENTCREDENTIAL", + "aws_secret_access_key": f"deployment-credential-{cell}", + "aws_region_name": "us-east-1", + **opt_in, + }, + "model_info": {"id": identity(cell)}, + } + + +def keyless_anthropic_deployment(cell: str, api_base: str) -> dict[str, JsonValue]: + return { + "model_name": name(cell), + "litellm_params": {"model": f"anthropic/{REJECTS}", "api_base": api_base}, + "model_info": {"id": identity(cell)}, + } + + +def azure_deployment(cell: str, api_base: str, credential: dict[str, JsonValue]) -> dict[str, JsonValue]: + return { + "model_name": name(cell), + "litellm_params": { + "model": f"azure/{REJECTS}", + "api_base": api_base, + "api_version": AZURE_API_VERSION, + **credential, + }, + "model_info": {"id": identity(cell)}, + } + + +def caller_credential_deployment(cell: str, field: str, api_base: str) -> dict[str, JsonValue]: + provider: Final = "watsonx" if field == "token" else "gigachat" + project: Final[dict[str, JsonValue]] = {"project_id": f"project-{RUN}"} if provider == "watsonx" else {} + return { + "model_name": name(cell), + "litellm_params": { + "model": f"{provider}/{REJECTS}", + "api_base": api_base, + field: f"deployment-credential-{cell}", + **project, + }, + "model_info": {"id": identity(cell)}, + } + + +def token_exchange_deployment(cell: str) -> dict[str, JsonValue]: + settings: Final[dict[str, JsonValue]] = ( + {"litellm_credential_name": f"m365-obo-{RUN}"} if cell.endswith("-credential") else {**TOKEN_EXCHANGE_SETTINGS} + ) + return { + "model_name": name(cell), + "litellm_params": {"model": "microsoft_365_copilot/chat", **settings}, + "model_info": {"id": identity(cell)}, + } + + +def caller_azure_cell(opt_in: str, route: str, inside_extra_body: bool) -> str: + return f"{opt_in}-caller-azure-token-{route}{'-extra-body' if inside_extra_body else ''}" + + +def config(directory: Path, upstream: Wire) -> Path: + api_base: Final = f"{upstream.url}/v1" + configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + configuration["model_list"] = [ + *configuration["model_list"], + *(deployment(cell, REJECTS, api_base) for cell in REJECTED_CELLS), + *(deployment(cell, HEALTHY, api_base) for cell in HEALTHY_CELLS), + deployment("held", HELD_REJECTS, api_base), + deployment("key-not-valid", KEY_NOT_VALID, api_base), + deployment("rate-limited", RATE_LIMITS, api_base), + deployment("recovers", RECOVERS, api_base), + anthropic_deployment("anthropic-own-key", upstream.url), + anthropic_deployment("anthropic-sdk-own-key", upstream.url), + anthropic_deployment("caller-oauth", upstream.url), + *(keyless_anthropic_deployment(cell, upstream.url) for cell in KEYLESS_CELLS), + bedrock_deployment( + "deployment-caller-bedrock-endpoint", + {"configurable_clientside_auth_params": ["aws_bedrock_runtime_endpoint"]}, + ), + azure_deployment("azure-own-token", upstream.url, {"azure_ad_token": "deployment-entra-token-azure-own-token"}), + *( + azure_deployment( + caller_azure_cell("deployment", route, inside_extra_body), + upstream.url, + {"configurable_clientside_auth_params": ["azure_ad_token"]}, + ) + for route, inside_extra_body in CALLER_AZURE_CELLS + ), + *(caller_credential_deployment(cell, field, upstream.url) for cell, field in CALLER_CREDENTIAL_CELLS), + *(token_exchange_deployment(cell) for cell in TOKEN_EXCHANGE_CELLS), + { + "model_name": name("caller-api-base"), + "litellm_params": { + "model": f"openai/{REJECTS}", + "api_key": "sk-deployment-credential-caller-api-base", + "api_base": api_base, + "configurable_clientside_auth_params": ["api_base"], + }, + "model_info": {"id": identity("caller-api-base")}, + }, + { + "model_name": name("caller-credential"), + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": f"copilot-per-user-{RUN}", + "github_copilot_auth_type": "per_user_oauth", + }, + "model_info": {"id": identity("caller-credential")}, + }, + { + "model_name": name("caller-credential-name"), + "litellm_params": { + "model": f"openai/{HEALTHY}", + "api_base": api_base, + "litellm_credential_name": f"deployment-key-{RUN}", + }, + "model_info": {"id": identity("caller-credential-name")}, + }, + ] + configuration["credential_list"] = [ + *configuration.get("credential_list", []), + {"credential_name": f"m365-obo-{RUN}", "credential_values": {**TOKEN_EXCHANGE_SETTINGS}, "credential_info": {}}, + { + "credential_name": f"deployment-key-{RUN}", + "credential_values": {"api_key": "sk-deployment-credential-caller-credential-name"}, + "credential_info": {}, + }, + { + "credential_name": f"revoked-key-{RUN}", + "credential_values": {"api_key": REVOKED_STORED_KEY}, + "credential_info": {}, + }, + ] + configuration["guardrails"] = [ + { + "guardrail_name": AGENT_365_GUARDRAIL, + "litellm_params": { + "guardrail": "agent_365", + "mode": "pre_mcp_call", + "default_on": True, + "tenant_id": "00000000-0000-4000-8000-0000000a3650", + "client_id": "synthetic-client-id", + "client_secret": "synthetic-client-secret", + }, + } + ] + configuration["litellm_settings"]["callbacks"] = ["prometheus"] + configuration["litellm_settings"]["model_group_settings"] = { + "forward_client_headers_to_llm_api": [ + name("forwarded-provider-header"), + name("forwarded-plain-header"), + name("forwarded-gateway-key"), + name("forwarded-user-information"), + ] + } + configuration["litellm_settings"]["add_user_information_to_llm_headers"] = True + configuration["general_settings"]["forward_llm_provider_auth_headers"] = True + configuration["general_settings"]["forward_openai_org_id"] = True + configuration.setdefault("router_settings", {}).update({"num_retries": 0, "disable_cooldowns": True}) + path: Final = directory / "prometheus-provider-auth-failure.yaml" + path.write_text(yaml.safe_dump(configuration)) + return path + + +@dataclass(frozen=True, slots=True) +class Rig: + gateway: Gateway + upstream: Wire + + @property + def api_base(self) -> str: + return f"{self.upstream.url}/v1" + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("prometheus-provider-auth-failure") + released: Final = threading.Event() + released.set() + with ( + wire_server(Provider(released)) as upstream, + gateway_from_environment() as shared, + owned_proxy( + shared, + directory, + {"PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": "2"}, + config=config(directory, upstream), + remove_environment=("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN"), + ) as owned, + ): + yield Rig(owned, upstream) + + +def opted_in_config(directory: Path, upstream: Wire) -> Path: + configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + configuration["model_list"] = [ + *configuration["model_list"], + *( + azure_deployment(caller_azure_cell("proxy-wide", route, inside_extra_body), upstream.url, {}) + for route, inside_extra_body in CALLER_AZURE_CELLS + ), + bedrock_deployment("proxy-wide-caller-bedrock-endpoint", {}), + ] + configuration["general_settings"]["allow_client_side_credentials"] = True + configuration["litellm_settings"]["callbacks"] = ["prometheus"] + configuration.setdefault("router_settings", {}).update({"num_retries": 0, "disable_cooldowns": True}) + path: Final = directory / "prometheus-caller-credentials-opted-in.yaml" + path.write_text(yaml.safe_dump(configuration)) + return path + + +@pytest.fixture(scope="module") +def opted_in(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("prometheus-caller-credentials-opted-in") + released: Final = threading.Event() + released.set() + with ( + wire_server(Provider(released)) as upstream, + gateway_from_environment() as shared, + owned_proxy(shared, directory, {}, config=opted_in_config(directory, upstream)) as owned, + ): + yield Rig(owned, upstream) + + +def series(samples: Sequence[Sample], metric: str, **labels: str) -> tuple[Sample, ...]: + return tuple( + sample + for sample in samples + if sample.name == metric and all(sample.labels.get(label) == value for label, value in labels.items()) + ) + + +def values(samples: Sequence[Sample], metric: str, **labels: str) -> list[float]: + return [sample.value for sample in series(samples, metric, **labels)] + + +def until(gateway: Gateway, metric: str, **labels: str) -> tuple[Sample, ...]: + return eventually(lambda: scrape(gateway), lambda samples: bool(series(samples, metric, **labels)), 30) + + +def messages(cell: str) -> list[JsonValue]: + return [{"role": "user", "content": f"{cell} {uuid.uuid4().hex}"}] + + +def send(gateway: Gateway, cell: str, model: str | None = None, key: str | None = None) -> httpx.Response: + asked: Final = model or name(cell) + stream: Final = cell.endswith("-stream") + if cell.startswith("messages"): + anthropic: Final = {"model": asked, "max_tokens": 16, "messages": messages(cell), "stream": stream} + return gateway.request("POST", "/v1/messages", anthropic, key=key) + if cell.startswith("responses"): + return gateway.request("POST", "/v1/responses", {"model": asked, "input": cell, "stream": stream}, key=key) + if cell == "embeddings": + return gateway.request("POST", "/v1/embeddings", {"model": asked, "input": cell}, key=key) + return gateway.request( + "POST", "/v1/chat/completions", {"model": asked, "messages": messages(cell), "stream": stream}, key=key + ) + + +def assert_reached_with_the_deployment_credential(rig: Rig, cell: str, attempts: int = 1) -> None: + reached: Final = rig.upstream.drain() + assert [request.headers["authorization"] for request in reached] == [ + f"Bearer sk-deployment-credential-{cell}" + ] * attempts, [request.target for request in reached] + + +def recorded(samples: Sequence[Sample], **labels: str) -> list[tuple[str, dict[str, str], float]]: + return [(sample.name, dict(sample.labels), sample.value) for sample in series_of(samples, **labels)] + + +def series_of(samples: Sequence[Sample], **labels: str) -> tuple[Sample, ...]: + return tuple( + sample + for sample in samples + if sample.name.startswith("litellm_") + and all(sample.labels.get(label) == value for label, value in labels.items()) + ) + + +def assert_kept_out_of_the_deployment_metrics(rig: Rig, cell: str) -> None: + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name(cell)) + assert {metric: values(samples, metric, model_id=identity(cell)) for metric in DEPLOYMENT_SERIES} == { + metric: [] for metric in DEPLOYMENT_SERIES + }, recorded(samples, model_id=identity(cell)) + + +def assert_recorded_as_a_deployment_failure(rig: Rig, cell: str, status: str, count: float = 1.0) -> None: + samples: Final = until(rig.gateway, FAILURES, model_id=identity(cell)) + observed: Final = recorded(samples, model_id=identity(cell)) + assert values(samples, FAILURES, model_id=identity(cell), exception_status=status) == [count], observed + assert values(samples, TOTALS, model_id=identity(cell)) == [count], observed + assert values(samples, LLM_FAILED, model_id=identity(cell)) == [count], observed + assert values(samples, STATE, model_id=identity(cell)) == [1.0], observed + + +@pytest.mark.parametrize("cell", ENDPOINT_CELLS) +def test_a_provider_401_on_every_endpoint_counts_as_a_deployment_failure_and_a_partial_outage( + rig: Rig, cell: str +) -> None: + rig.upstream.drain() + response: Final = send(rig.gateway, cell) + assert response.status_code == 401, response.text + assert_reached_with_the_deployment_credential(rig, cell) + assert_recorded_as_a_deployment_failure(rig, cell, "401") + + +def test_a_provider_401_seen_by_the_sync_openai_client_is_recorded(rig: Rig) -> None: + rig.upstream.drain() + with ( + openai.OpenAI( + base_url=f"{rig.gateway.client.base_url}/v1", + api_key=rig.gateway.key, + max_retries=0, + http_client=httpx.Client(trust_env=False), + ) as client, + pytest.raises(openai.AuthenticationError, match="Incorrect API key provided"), + ): + client.chat.completions.create(model=name("sdk-sync"), messages=[{"role": "user", "content": "sync"}]) + assert_reached_with_the_deployment_credential(rig, "sdk-sync") + assert_recorded_as_a_deployment_failure(rig, "sdk-sync", "401") + + +def test_a_provider_401_seen_by_the_async_openai_client_is_recorded(rig: Rig) -> None: + async def ask() -> None: + async with openai.AsyncOpenAI( + base_url=f"{rig.gateway.client.base_url}/v1", + api_key=rig.gateway.key, + max_retries=0, + http_client=httpx.AsyncClient(trust_env=False), + ) as client: + await client.chat.completions.create(model=name("sdk-async"), messages=[{"role": "user", "content": "a"}]) + + rig.upstream.drain() + with pytest.raises(openai.AuthenticationError, match="Incorrect API key provided"): + asyncio.run(ask()) + assert_reached_with_the_deployment_credential(rig, "sdk-async") + assert_recorded_as_a_deployment_failure(rig, "sdk-async", "401") + + +def test_each_repeated_provider_401_is_counted_once(rig: Rig) -> None: + rig.upstream.drain() + assert [send(rig.gateway, "repeated").status_code for _ in range(3)] == [401, 401, 401] + assert_reached_with_the_deployment_credential(rig, "repeated", attempts=3) + assert_recorded_as_a_deployment_failure(rig, "repeated", "401", count=3.0) + + +def test_a_provider_401_rescued_by_a_fallback_still_counts_against_the_rejected_deployment(rig: Rig) -> None: + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name("fallback"), "messages": messages("fallback"), "fallbacks": [name("healthy")]}, + ) + assert response.status_code == 200, response.text + assert [request.headers["authorization"] for request in rig.upstream.drain()] == [ + "Bearer sk-deployment-credential-fallback", + "Bearer sk-deployment-credential-healthy", + ] + assert_recorded_as_a_deployment_failure(rig, "fallback", "401") + rescued: Final = eventually( + lambda: scrape(rig.gateway), lambda samples: values(samples, SUCCESSES, model_id=identity("healthy")) == [1.0] + ) + assert values(rescued, FAILURES, model_id=identity("healthy")) == [] + + +def test_a_provider_auth_message_on_a_non_401_status_is_recorded(rig: Rig) -> None: + rig.upstream.drain() + response: Final = send(rig.gateway, "key-not-valid") + assert response.status_code == 400, response.text + assert_reached_with_the_deployment_credential(rig, "key-not-valid") + assert_recorded_as_a_deployment_failure(rig, "key-not-valid", "400") + + +def test_a_provider_429_is_recorded_as_before(rig: Rig) -> None: + rig.upstream.drain() + response: Final = send(rig.gateway, "rate-limited") + assert response.status_code == 429, response.text + assert_reached_with_the_deployment_credential(rig, "rate-limited") + assert_recorded_as_a_deployment_failure(rig, "rate-limited", "429") + + +def test_a_deployment_marked_down_by_a_provider_401_is_healthy_again_after_a_success(rig: Rig) -> None: + rig.upstream.drain() + assert send(rig.gateway, "recovers").status_code == 401 + assert_recorded_as_a_deployment_failure(rig, "recovers", "401") + recovered: Final = send(rig.gateway, "recovers") + assert recovered.status_code == 200, recovered.text + assert len(rig.upstream.drain()) == 2 + eventually( + lambda: scrape(rig.gateway), + lambda samples: values(samples, STATE, model_id=identity("recovers"), api_base=rig.api_base) == [0.0], + ) + + +def test_a_per_caller_credential_rejection_stays_out_of_the_deployment_metrics(rig: Rig) -> None: + rig.upstream.drain() + response: Final = send(rig.gateway, "caller-credential") + assert response.status_code == 401, response.text + assert "Connect GitHub Copilot" in response.text, response.text + assert rig.upstream.drain() == () + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name("caller-credential")) + assert values(samples, FAILURES, model_id=identity("caller-credential")) == [] + assert values(samples, TOTALS, model_id=identity("caller-credential")) == [] + assert values(samples, LLM_FAILED, model_id=identity("caller-credential")) == [] + assert values(samples, STATE, model_id=identity("caller-credential")) == [] + + +def deployment_series(samples: Sequence[Sample]) -> frozenset[tuple[str, tuple[tuple[str, str], ...]]]: + return frozenset((sample.name, sample.identity()) for sample in samples if sample.name in DEPLOYMENT_SERIES) + + +def test_a_provider_401_on_an_api_key_the_caller_sent_stays_out_of_the_deployment_metrics(rig: Rig) -> None: + caller_key: Final = f"sk-caller-supplied-{uuid.uuid4().hex}" + before: Final = deployment_series(scrape(rig.gateway)) + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name("caller-api-key"), "messages": messages("caller-api-key"), "api_key": caller_key}, + ) + assert response.status_code == 401, response.text + assert [request.headers["authorization"] for request in rig.upstream.drain()] == [f"Bearer {caller_key}"] + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name("caller-api-key")) + assert deployment_series(samples) - before == frozenset(), recorded(samples, requested_model=name("caller-api-key")) + + +def test_a_provider_401_on_a_stored_credential_the_caller_names_leaves_a_healthy_deployment_healthy(rig: Rig) -> None: + cell: Final = "caller-credential-name" + rig.upstream.drain() + assert send(rig.gateway, cell).status_code == 200 + assert_reached_with_the_deployment_credential(rig, cell) + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name(cell), "messages": messages(cell), "litellm_credential_name": f"revoked-key-{RUN}"}, + ) + assert response.status_code == 401, response.text + assert [request.headers["authorization"] for request in rig.upstream.drain()] == [f"Bearer {REVOKED_STORED_KEY}"] + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name(cell)) + observed: Final = recorded(samples, model_id=identity(cell)) + assert values(samples, FAILURES, model_id=identity(cell)) == [], observed + assert values(samples, LLM_FAILED, model_id=identity(cell)) == [], observed + assert values(samples, TOTALS, model_id=identity(cell)) == [1.0], observed + assert values(samples, STATE, model_id=identity(cell)) == [0.0], observed + + +def anthropic_messages(gateway: Gateway, cell: str, key: str, headers: dict[str, str]) -> httpx.Response: + body: Final[dict[str, JsonValue]] = {"model": name(cell), "max_tokens": 16, "messages": messages(cell)} + return gateway.request("POST", "/v1/messages", body, key=key, headers=headers) + + +def caller_header(field: str, header: str, value: str) -> JsonValue: + if field == "provider_specific_header": + return {"custom_llm_provider": "openai", "extra_headers": {header: value}} + return {header: value} + + +@pytest.mark.parametrize( + "cell,field", + [ + ("caller-extra-headers", "extra_headers"), + ("caller-headers", "headers"), + ("caller-provider-specific-header", "provider_specific_header"), + ], +) +def test_a_provider_401_on_an_authorization_header_the_caller_sends_stays_out_of_the_deployment_metrics( + rig: Rig, cell: str, field: str +) -> None: + caller_value: Final = f"Bearer sk-{cell}-{uuid.uuid4().hex}" + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name(cell), "messages": messages(cell), field: caller_header(field, "Authorization", caller_value)}, + ) + assert response.status_code == 401, response.text + assert [request.headers["authorization"] for request in rig.upstream.drain()] == [caller_value] + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name(cell)) + observed: Final = recorded(samples, model_id=identity(cell)) + assert values(samples, FAILURES, model_id=identity(cell)) == [], observed + assert values(samples, TOTALS, model_id=identity(cell)) == [], observed + assert values(samples, LLM_FAILED, model_id=identity(cell)) == [], observed + assert values(samples, STATE, model_id=identity(cell)) == [], observed + + +@pytest.mark.parametrize( + "cell,field", + [ + ("plain-extra-headers", "extra_headers"), + ("plain-headers", "headers"), + ("plain-provider-specific-header", "provider_specific_header"), + ], +) +def test_a_provider_401_on_the_deployments_own_key_stays_out_when_the_caller_sends_a_plain_header( + rig: Rig, cell: str, field: str +) -> None: + trace: Final = f"{cell}-{uuid.uuid4().hex}" + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name(cell), "messages": messages(cell), field: caller_header(field, "X-Caller-Trace", trace)}, + ) + assert response.status_code == 401, response.text + reached: Final = rig.upstream.drain() + assert [(request.headers["authorization"], request.headers.get("x-caller-trace")) for request in reached] == [ + (f"Bearer sk-deployment-credential-{cell}", trace) + ] + assert_kept_out_of_the_deployment_metrics(rig, cell) + + +def test_a_provider_401_on_a_provider_credential_header_the_proxy_forwards_stays_out_of_the_deployment_metrics( + rig: Rig, +) -> None: + caller_key: Final = f"AIza-forwarded-{uuid.uuid4().hex}" + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name("forwarded-provider-header"), "messages": messages("forwarded-provider-header")}, + headers={"x-goog-api-key": caller_key}, + ) + assert response.status_code == 401, response.text + assert [request.headers.get("x-goog-api-key") for request in rig.upstream.drain()] == [caller_key] + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name("forwarded-provider-header")) + observed: Final = recorded(samples, model_id=identity("forwarded-provider-header")) + assert values(samples, FAILURES, model_id=identity("forwarded-provider-header")) == [], observed + assert values(samples, TOTALS, model_id=identity("forwarded-provider-header")) == [], observed + assert values(samples, LLM_FAILED, model_id=identity("forwarded-provider-header")) == [], observed + assert values(samples, STATE, model_id=identity("forwarded-provider-header")) == [], observed + + +def test_a_provider_401_on_the_deployments_own_key_stays_out_when_the_proxy_forwards_a_plain_header(rig: Rig) -> None: + trace: Final = f"forwarded-plain-header-{uuid.uuid4().hex}" + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name("forwarded-plain-header"), "messages": messages("forwarded-plain-header")}, + headers={"x-caller-trace": trace}, + ) + assert response.status_code == 401, response.text + assert [ + (request.headers["authorization"], request.headers.get("x-caller-trace")) for request in rig.upstream.drain() + ] == [("Bearer sk-deployment-credential-forwarded-plain-header", trace)] + assert_kept_out_of_the_deployment_metrics(rig, "forwarded-plain-header") + + +def test_a_provider_401_on_the_deployments_own_key_counts_when_the_proxy_forwards_only_its_own_user_information( + rig: Rig, +) -> None: + rig.upstream.drain() + response: Final = send(rig.gateway, "forwarded-user-information") + assert response.status_code == 401, response.text + reached: Final = rig.upstream.drain() + assert [ + (request.headers["authorization"], "x-litellm-user_api_key_hash" in request.headers) for request in reached + ] == [("Bearer sk-deployment-credential-forwarded-user-information", True)] + assert_recorded_as_a_deployment_failure(rig, "forwarded-user-information", "401") + + +def test_a_provider_401_on_a_gateway_key_the_proxy_forwards_stays_out_of_a_healthy_deployments_metrics( + rig: Rig, +) -> None: + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name("forwarded-gateway-key"), "messages": messages("forwarded-gateway-key")}, + headers={"x-portkey-api-key": REVOKED_GATEWAY_KEY}, + ) + assert response.status_code == 401, response.text + assert [ + (request.headers["authorization"], request.headers.get("x-portkey-api-key")) for request in rig.upstream.drain() + ] == [("Bearer sk-deployment-credential-forwarded-gateway-key", REVOKED_GATEWAY_KEY)] + assert_kept_out_of_the_deployment_metrics(rig, "forwarded-gateway-key") + + +@pytest.mark.parametrize( + "cell,body,headers", + [ + ("caller-organization", {"organization": UNKNOWN_ORGANIZATION}, {}), + ("forwarded-organization", {}, {"OpenAI-Organization": UNKNOWN_ORGANIZATION}), + ], +) +def test_a_provider_401_on_an_organization_the_caller_sends_stays_out_of_a_healthy_deployments_metrics( + rig: Rig, cell: str, body: dict[str, JsonValue], headers: dict[str, str] +) -> None: + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", "/v1/chat/completions", {"model": name(cell), "messages": messages(cell), **body}, headers=headers + ) + assert response.status_code == 401, response.text + assert [ + (request.headers["authorization"], request.headers.get("openai-organization")) + for request in rig.upstream.drain() + ] == [(f"Bearer sk-deployment-credential-{cell}", UNKNOWN_ORGANIZATION)] + assert_kept_out_of_the_deployment_metrics(rig, cell) + + +@pytest.mark.parametrize( + ("cell", "headers"), + [ + ("anthropic-own-key", {}), + ("anthropic-sdk-own-key", {"anthropic-version": "2023-06-01", "anthropic-beta": "tools-2024-04-04"}), + ], + ids=["no-client-headers", "anthropic-sdk-headers"], +) +def test_a_provider_401_on_an_anthropic_deployments_own_key_is_recorded( + rig: Rig, cell: str, headers: dict[str, str] +) -> None: + with rig.gateway.scenario() as scenario: + key: Final = scenario.key(models=[name(cell)]) + rig.upstream.drain() + response: Final = anthropic_messages(rig.gateway, cell, key, headers) + assert response.status_code == 401, response.text + assert [request.headers.get("x-api-key") for request in rig.upstream.drain()] == [ + f"sk-ant-api03-deployment-credential-{cell}" + ] + assert_recorded_as_a_deployment_failure(rig, cell, "401") + + +def test_a_provider_401_on_an_anthropic_oauth_token_the_caller_sent_stays_out_of_the_deployment_metrics( + rig: Rig, +) -> None: + with rig.gateway.scenario() as scenario: + key: Final = scenario.key(models=[name("caller-oauth")]) + rig.upstream.drain() + response: Final = anthropic_messages( + rig.gateway, + "caller-oauth", + CALLER_OAUTH_TOKEN, + {"x-litellm-api-key": key}, + ) + assert response.status_code == 401, response.text + assert [request.headers.get("authorization") for request in rig.upstream.drain()] == [ + f"Bearer {CALLER_OAUTH_TOKEN}" + ] + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name("caller-oauth")) + assert {metric: values(samples, metric, model_id=identity("caller-oauth")) for metric in DEPLOYMENT_SERIES} == { + metric: [] for metric in DEPLOYMENT_SERIES + }, recorded(samples, requested_model=name("caller-oauth")) + + +def test_a_provider_401_on_an_azure_deployments_own_ad_token_is_recorded(rig: Rig) -> None: + rig.upstream.drain() + response: Final = send(rig.gateway, "azure-own-token") + assert response.status_code == 401, response.text + assert [request.headers.get("authorization") for request in rig.upstream.drain()] == [ + "Bearer deployment-entra-token-azure-own-token" + ] + assert_recorded_as_a_deployment_failure(rig, "azure-own-token", "401") + + +def caller_azure_request(cell: str, route: str, credential: dict[str, JsonValue]) -> tuple[str, dict[str, JsonValue]]: + if route == "embeddings": + return "/v1/embeddings", {"model": name(cell), "input": cell, **credential} + if route == "responses": + return "/v1/responses", {"model": name(cell), "input": cell, **credential} + if route == "messages": + return "/v1/messages", {"model": name(cell), "max_tokens": 16, "messages": messages(cell), **credential} + return "/v1/chat/completions", {"model": name(cell), "messages": messages(cell), **credential} + + +@pytest.mark.parametrize( + ("route", "inside_extra_body"), + CALLER_AZURE_CELLS, + ids=[f"{route}{'-extra-body' if inside_extra_body else ''}" for route, inside_extra_body in CALLER_AZURE_CELLS], +) +@pytest.mark.parametrize("opt_in", ("deployment", "proxy-wide")) +def test_a_provider_401_on_an_azure_ad_token_the_caller_sent_stays_out_of_the_deployment_metrics( + request: pytest.FixtureRequest, opt_in: str, route: str, inside_extra_body: bool +) -> None: + opted: Final[Rig] = request.getfixturevalue("rig" if opt_in == "deployment" else "opted_in") + cell: Final = caller_azure_cell(opt_in, route, inside_extra_body) + token: Final = f"caller-entra-token-{uuid.uuid4().hex}" + credential: Final[dict[str, JsonValue]] = ( + {"extra_body": {"azure_ad_token": token}} if inside_extra_body else {"azure_ad_token": token} + ) + opted.upstream.drain() + response: Final = opted.gateway.request("POST", *caller_azure_request(cell, route, credential)) + assert response.status_code == 401, response.text + assert [reached.headers.get("authorization") for reached in opted.upstream.drain()] == [ + None if inside_extra_body else f"Bearer {token}" + ] + samples: Final = until(opted.gateway, PROXY_FAILED, requested_model=name(cell)) + assert {metric: values(samples, metric, model_id=identity(cell)) for metric in DEPLOYMENT_SERIES} == { + metric: [] for metric in DEPLOYMENT_SERIES + }, recorded(samples, requested_model=name(cell)) + + +@pytest.mark.parametrize(("cell", "field"), CALLER_CREDENTIAL_CELLS, ids=[cell for cell, _ in CALLER_CREDENTIAL_CELLS]) +def test_a_provider_401_on_a_provider_credential_the_caller_sent_stays_out_of_the_deployment_metrics( + rig: Rig, cell: str, field: str +) -> None: + caller_credential: Final = f"caller-credential-{uuid.uuid4().hex}" + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", "/v1/chat/completions", {"model": name(cell), "messages": messages(cell), field: caller_credential} + ) + assert response.status_code == 401, response.text + assert [reached.headers.get("authorization") for reached in rig.upstream.drain()] == [f"Bearer {caller_credential}"] + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name(cell)) + assert {metric: values(samples, metric, model_id=identity(cell)) for metric in DEPLOYMENT_SERIES} == { + metric: [] for metric in DEPLOYMENT_SERIES + }, recorded(samples, requested_model=name(cell)) + + +@pytest.mark.parametrize("cell", TOKEN_EXCHANGE_CELLS) +def test_a_token_exchange_deployment_rejecting_a_caller_without_an_identity_token_stays_out_of_the_deployment_metrics( + rig: Rig, cell: str +) -> None: + rig.upstream.drain() + response: Final = send(rig.gateway, cell) + assert response.status_code == 401, response.text + assert "IdP-issued access token" in response.text, response.text + assert rig.upstream.drain() == () + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name(cell)) + assert {metric: values(samples, metric, model_id=identity(cell)) for metric in DEPLOYMENT_SERIES} == { + metric: [] for metric in DEPLOYMENT_SERIES + }, recorded(samples, requested_model=name(cell)) + + +@pytest.mark.parametrize("cell", KEYLESS_CELLS) +def test_a_missing_deployment_key_reported_before_the_provider_is_called_stays_out_of_the_deployment_metrics( + rig: Rig, cell: str +) -> None: + rig.upstream.drain() + response: Final = send(rig.gateway, cell) + assert response.status_code == 401, response.text + assert "Missing Anthropic API Key" in response.text, response.text + assert rig.upstream.drain() == () + assert_kept_out_of_the_deployment_metrics(rig, cell) + + +@pytest.mark.parametrize("opt_in", ("deployment", "proxy-wide")) +def test_a_provider_401_at_a_bedrock_endpoint_the_caller_sends_stays_out_of_the_deployment_metrics( + request: pytest.FixtureRequest, opt_in: str +) -> None: + opted: Final[Rig] = request.getfixturevalue("rig" if opt_in == "deployment" else "opted_in") + cell: Final = f"{opt_in}-caller-bedrock-endpoint" + opted.upstream.drain() + response: Final = opted.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name(cell), "messages": messages(cell), "aws_bedrock_runtime_endpoint": opted.upstream.url}, + ) + assert response.status_code == 401, response.text + assert [reached.target for reached in opted.upstream.drain()] == [f"/model/{REJECTS}/converse"] + assert_kept_out_of_the_deployment_metrics(opted, cell) + + +def test_a_provider_401_at_an_api_base_the_caller_sends_counts_against_a_one_off_deployment(rig: Rig) -> None: + caller_api_base: Final = f"{rig.upstream.url}/caller-endpoint-{RUN}/v1" + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name("caller-api-base"), "messages": messages("caller-api-base"), "api_base": caller_api_base}, + ) + assert response.status_code == 401, response.text + assert [(reached.target, reached.headers["authorization"]) for reached in rig.upstream.drain()] == [ + (f"/caller-endpoint-{RUN}/v1/chat/completions", "Bearer sk-deployment-credential-caller-api-base") + ] + samples: Final = until(rig.gateway, FAILURES, api_base=caller_api_base) + one_off: Final = {sample.labels["model_id"] for sample in series(samples, FAILURES, api_base=caller_api_base)} + assert len(one_off) == 1 and identity("caller-api-base") not in one_off, recorded(samples, api_base=caller_api_base) + assert values(samples, FAILURES, api_base=caller_api_base, exception_status="401") == [1.0] + assert values(samples, TOTALS, api_base=caller_api_base) == [1.0] + assert values(samples, STATE, api_base=caller_api_base) == [1.0] + assert values(samples, LLM_FAILED, model_id=next(iter(one_off))) == [1.0] + assert {metric: values(samples, metric, model_id=identity("caller-api-base")) for metric in DEPLOYMENT_SERIES} == { + metric: [] for metric in DEPLOYMENT_SERIES + } + + +def _gated_key(scenario: Scenario, cell: str) -> str | None: + if cell == "invalid-key": + return f"sk-not-a-virtual-key-{uuid.uuid4().hex}" + if cell == "missing-key": + return None + if cell == "no-model-access": + return scenario.key(models=[name("healthy")]) + key: Final = scenario.key(models=[name(cell)]) + if cell == "expired-key": + write_rows( + "UPDATE \"LiteLLM_VerificationToken\" SET expires = now() - interval '1 hour' WHERE token = %s", + (sha256(key.encode()).hexdigest(),), + database_url=scenario.gateway.database_url, + ) + return key + scenario.gateway.post("/key/block", {"key": key}) + return key + + +def _gated(gateway: Gateway, cell: str, key: str | None) -> httpx.Response: + if key is None: + return gateway.client.post("/v1/chat/completions", json={"model": name(cell), "messages": messages(cell)}) + return send(gateway, cell, key=key) + + +def recorded_before(cell: str) -> list[tuple[str, str, str, float]]: + if cell in ("invalid-key", "expired-key"): + return [] + return sorted([(FAILURES, "", "", 1.0), (TOTALS, "", "", 1.0), (LLM_FAILED, identity(cell), "", 1.0)]) + + +def deployment_rows(samples: Sequence[Sample], cell: str) -> list[tuple[str, str, str, float]]: + return sorted( + (sample.name, sample.labels.get("model_id", ""), sample.labels.get("api_base", ""), sample.value) + for sample in samples + if sample.name in DEPLOYMENT_SERIES + and name(cell) in (sample.labels.get("requested_model"), sample.labels.get("model")) + ) + + +@pytest.mark.parametrize("cell", GATED_CELLS) +def test_a_request_the_proxy_rejects_itself_never_marks_the_deployment_down_and_is_recorded_as_before( + rig: Rig, cell: str +) -> None: + with rig.gateway.scenario() as scenario: + key: Final = _gated_key(scenario, cell) + rig.upstream.drain() + response: Final = _gated(rig.gateway, cell, key) + assert response.status_code == (403 if cell == "no-model-access" else 401), response.text + assert rig.upstream.drain() == () + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name(cell)) + assert deployment_rows(samples, cell) == recorded_before(cell), recorded(samples, requested_model=name(cell)) + + +def test_a_key_rate_limit_rejected_at_the_proxy_is_still_counted_as_before(rig: Rig) -> None: + with rig.gateway.scenario() as scenario: + key: Final = scenario.key(models=[name("key-rpm-limit")], rpm_limit=1) + rig.upstream.drain() + admitted: Final = send(rig.gateway, "key-rpm-limit", key=key) + assert admitted.status_code == 200, admitted.text + limited: Final = send(rig.gateway, "key-rpm-limit", key=key) + assert limited.status_code == 429, limited.text + assert len(rig.upstream.drain()) == 1 + samples: Final = until(rig.gateway, FAILURES, requested_model=name("key-rpm-limit"), exception_status="429") + assert values(samples, FAILURES, requested_model=name("key-rpm-limit"), exception_status="429") == [1.0] + eventually( + lambda: scrape(rig.gateway), + lambda observed: ( + values(observed, STATE, model_id=identity("key-rpm-limit"), api_base=rig.api_base) == [0.0] + ), + ) + + +@pytest.fixture(scope="module") +def mcp_peer() -> Iterator[McpPeer]: + with scripted_peer(echo_tool("add")) as peer: + yield peer + + +def test_an_mcp_tool_call_the_proxy_rejects_for_the_caller_stays_out_of_the_llm_failure_metrics( + rig: Rig, mcp_peer: McpPeer +) -> None: + alias: Final = "a365" + RUN + with rig.gateway.scenario() as scenario: + server: Final = register_mcp(scenario, mcp_peer, alias) + key: Final = scenario.key(object_permission={"mcp_servers": [server]}) + hashed: Final = sha256(key.encode()).hexdigest() + outcomes: Final = { + entry: McpCaller(rig.gateway, key, entry, alias).call(f"{alias}-add", {"entry": entry}, server) + for entry in ENTRY_POINTS + } + assert all( + outcome.error is not None and "Agent 365 guardrail rejected" in outcome.raw for outcome in outcomes.values() + ), {entry: outcome.raw for entry, outcome in outcomes.items()} + eventually( + lambda: rig.gateway.rows(GUARDRAIL_ROWS, (AGENT_365_GUARDRAIL, hashed, "call_mcp_tool")), + lambda rows: [row["status"] for row in rows] == ["guardrail_intervened"] * len(ENTRY_POINTS), + seconds=70, + ) + samples: Final = scrape(rig.gateway) + assert values(samples, LLM_FAILED, hashed_api_key=hashed) == [] + assert values(samples, FAILURES, hashed_api_key=hashed) == [] + assert values(samples, TOTALS, hashed_api_key=hashed) == [] diff --git a/tests/integration/observability/test_prometheus_provider_auth_failure_chaos.py b/tests/integration/observability/test_prometheus_provider_auth_failure_chaos.py new file mode 100644 index 00000000000..a1fe1cf4f33 --- /dev/null +++ b/tests/integration/observability/test_prometheus_provider_auth_failure_chaos.py @@ -0,0 +1,147 @@ +import threading +import uuid +from collections.abc import Iterator, Sequence +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import httpx +import psutil +from integration._support.client import Gateway, eventually +from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.prometheus_series import Sample, scrape +from integration._support.wire import Wire, wire_server +from test_prometheus_provider_auth_failure import ( + FAILURES, + LLM_FAILED, + STATE, + TOTALS, + Provider, + config, + identity, + name, + send, + values, +) + +BURST: Final = 12 +ENDPOINTS: Final = ("chat", "chat-stream", "messages", "responses") + + +@dataclass(frozen=True, slots=True) +class ChaosRig: + owned: OwnedProxy + upstream: Wire + release: threading.Event + + +@contextmanager +def _rig(gateway: Gateway, tmp_path: Path) -> Iterator[ChaosRig]: + release: Final = threading.Event() + prom_dir: Final = tmp_path / "prom" + prom_dir.mkdir() + with ( + wire_server(Provider(release)) as upstream, + owned_proxy_process( + gateway, + tmp_path, + {"PROMETHEUS_MULTIPROC_DIR": str(prom_dir)}, + config=config(tmp_path, upstream), + workers=2, + ) as owned, + ): + try: + yield ChaosRig(owned, upstream, release) + finally: + release.set() + + +def _status(gateway: Gateway, cell: str, model: str, key: str | None = None) -> int: + try: + return send(gateway, cell, model=model, key=key).status_code + except httpx.HTTPError: + return -1 + + +def _holds_a_connection_to(pid: int, port: int) -> bool: + return any(connection.raddr[1:] == (port,) for connection in psutil.Process(pid).net_connections(kind="tcp")) + + +def _workers_calling_upstream(rig: ChaosRig) -> frozenset[int]: + port: Final = int(rig.upstream.url.rsplit(":", 1)[1]) + return frozenset(pid for pid in rig.owned.worker_pids() if _holds_a_connection_to(pid, port)) + + +def _counted(samples: Sequence[Sample], cell: str) -> tuple[float, float, float]: + return ( + sum(values(samples, FAILURES, model_id=identity(cell))), + sum(values(samples, TOTALS, model_id=identity(cell))), + sum(values(samples, LLM_FAILED, model_id=identity(cell))), + ) + + +def test_a_burst_of_provider_401s_and_proxy_rejects_across_two_workers_counts_only_the_provider_401s( + gateway: Gateway, tmp_path: Path +) -> None: + with _rig(gateway, tmp_path) as rig: + rig.release.set() + workers: Final = eventually(rig.owned.worker_pids, lambda pids: len(pids) == 2) + invalid: Final = f"sk-not-a-virtual-key-{uuid.uuid4().hex}" + calls: Final[tuple[tuple[str, str, str | None], ...]] = tuple( + (ENDPOINTS[index % len(ENDPOINTS)], name("burst"), None) for index in range(BURST) + ) + tuple(("chat", name("invalid-key"), invalid) for _ in range(BURST)) + + def call(cell: str, model: str, key: str | None) -> int: + return _status(rig.owned.gateway, cell, model, key) + + with ThreadPoolExecutor(max_workers=8) as pool: + statuses: Final = tuple(pool.map(call, *zip(*calls, strict=True))) + assert statuses == (401,) * (2 * BURST), statuses + assert len(rig.upstream.drain()) == BURST + samples: Final = eventually( + lambda: scrape(rig.owned.gateway), + lambda observed: _counted(observed, "burst") == (BURST, BURST, BURST), + seconds=70, + return_last_on_timeout=True, + ) + assert _counted(samples, "burst") == (BURST, BURST, BURST), (workers, _counted(samples, "burst")) + assert set(values(samples, STATE, model_id=identity("burst"))) == {1.0} + assert values(samples, FAILURES, requested_model=name("invalid-key")) == [] + assert values(samples, LLM_FAILED, model=name("invalid-key")) == [] + + +def test_a_worker_killed_while_provider_401s_are_in_flight_loses_no_answered_failure( + gateway: Gateway, tmp_path: Path +) -> None: + with _rig(gateway, tmp_path) as rig: + eventually(rig.owned.worker_pids, lambda pids: len(pids) == 2) + with ThreadPoolExecutor(max_workers=8) as pool: + futures: Final = tuple(pool.submit(_status, rig.owned.gateway, "chat", name("held")) for _ in range(BURST)) + eventually(lambda: rig.upstream.received.qsize() >= 8, bool, seconds=30) + victim: Final = min(eventually(lambda: _workers_calling_upstream(rig), bool)) + with rig.owned.replacing_worker(victim) as kill: + kill() + rig.release.set() + statuses: Final = tuple(future.result() for future in futures) + reached: Final = len(rig.upstream.drain()) + answered: Final = statuses.count(401) + lost: Final = statuses.count(-1) + assert answered + lost == BURST and lost > 0, statuses + samples: Final = eventually( + lambda: scrape(rig.owned.gateway), + lambda observed: answered <= _counted(observed, "held")[0] <= reached, + seconds=70, + return_last_on_timeout=True, + ) + counted: Final = _counted(samples, "held") + assert answered <= counted[0] <= reached and counted[0] == counted[1] == counted[2], (statuses, counted) + follow_up: Final = _status(rig.owned.gateway, "chat", name("held")) + assert follow_up == 401 + recounted: Final = eventually( + lambda: _counted(scrape(rig.owned.gateway), "held"), + lambda observed: observed[0] == counted[0] + 1, + seconds=70, + ) + assert recounted == (counted[0] + 1,) * 3 diff --git a/tests/integration/sdk/test_prometheus_copilot_session_auth_failure_sdk.py b/tests/integration/sdk/test_prometheus_copilot_session_auth_failure_sdk.py new file mode 100644 index 00000000000..1b5e990fe41 --- /dev/null +++ b/tests/integration/sdk/test_prometheus_copilot_session_auth_failure_sdk.py @@ -0,0 +1,99 @@ +"""A provider 401 on a caller's own per-user GitHub Copilot session is the caller's credential failing, so the +shared Copilot deployment keeps no failure, request or outage series for it, while a provider 401 on the router +deployment's own key, on a key a router caller passes or on the key a direct SDK call brings is counted as a failed +request, the router caller's key against its own one-off deployment.""" + +from __future__ import annotations + +import json +from collections.abc import Iterator +from typing import Final + +import pytest +from integration._support.copilot_session_prometheus_worker import ( + CALLER_SESSION_TOKEN, + COPILOT_DEPLOYMENT, + DEPLOYMENT_SERIES, + DIRECT_CALL, + DIRECT_KEY, + FAILURES, + LLM_FAILED, + OPERATOR_DEPLOYMENT, + OPERATOR_KEY, + ROUTER_CALLER_KEY, + STATE, + TOTALS, + Outcome, + WorkerReport, + run_copilot_session_worker, +) +from integration._support.wire import Reply, Request, wire_server + + +@pytest.fixture(scope="module") +def ran() -> Iterator[tuple[WorkerReport, tuple[Request, ...]]]: + def reject(request: Request) -> Reply: + assert request.method == "POST", request.method + return Reply( + status=401, + body=json.dumps({"error": {"message": "bad credentials", "type": "authentication_error"}}).encode(), + ) + + with wire_server(reject) as upstream: + report: Final = run_copilot_session_worker(upstream.url) + yield report, upstream.drain() + + +def test_every_call_reaches_the_provider_with_its_own_credential_and_fails_with_its_401( + ran: tuple[WorkerReport, tuple[Request, ...]], +) -> None: + report, received = ran + assert report.outcomes == ( + Outcome(deployment=COPILOT_DEPLOYMENT, status=401), + Outcome(deployment=OPERATOR_DEPLOYMENT, status=401), + Outcome(deployment=OPERATOR_DEPLOYMENT, status=401), + Outcome(deployment=DIRECT_CALL, status=401), + ) + assert [r.headers.get("authorization") for r in received] == [ + f"Bearer {CALLER_SESSION_TOKEN}", + f"Bearer {OPERATOR_KEY}", + f"Bearer {ROUTER_CALLER_KEY}", + f"Bearer {DIRECT_KEY}", + ] + + +def test_a_provider_401_on_a_callers_copilot_session_stays_out_of_the_shared_deployment_metrics( + ran: tuple[WorkerReport, tuple[Request, ...]], +) -> None: + report, _ = ran + assert {s.metric for s in report.series if s.model_id == COPILOT_DEPLOYMENT} & DEPLOYMENT_SERIES == set() + + +def test_a_provider_401_on_the_deployments_own_key_counts_as_a_deployment_failure( + ran: tuple[WorkerReport, tuple[Request, ...]], +) -> None: + report, _ = ran + assert report.values(FAILURES, OPERATOR_DEPLOYMENT) == (1.0,) + assert report.values(TOTALS, OPERATOR_DEPLOYMENT) == (1.0,) + assert report.values(LLM_FAILED, OPERATOR_DEPLOYMENT) == (1.0,) + assert report.on_the_endpoint(STATE, OPERATOR_DEPLOYMENT) == (1.0,) + + +def test_a_provider_401_on_the_key_a_direct_sdk_call_brings_is_counted_as_a_failed_request( + ran: tuple[WorkerReport, tuple[Request, ...]], +) -> None: + report, _ = ran + assert report.outside_the_router(FAILURES) == (1.0,) + assert report.outside_the_router(TOTALS) == (1.0,) + assert report.outside_the_router(LLM_FAILED) == (1.0,) + + +def test_a_provider_401_on_a_key_a_router_caller_passes_counts_against_its_own_one_off_deployment( + ran: tuple[WorkerReport, tuple[Request, ...]], +) -> None: + report, _ = ran + assert report.on_a_one_off_deployment(FAILURES) == (1.0,) + assert report.on_a_one_off_deployment(TOTALS) == (1.0,) + assert report.on_a_one_off_deployment(LLM_FAILED) == (1.0,) + assert report.values(FAILURES, OPERATOR_DEPLOYMENT) == (1.0,) + assert report.values(TOTALS, OPERATOR_DEPLOYMENT) == (1.0,) diff --git a/tests/unit/integrations/test_prometheus_invalid_key_filtering.py b/tests/unit/integrations/test_prometheus_invalid_key_filtering.py index 9e8348e860a..453fd28636d 100644 --- a/tests/unit/integrations/test_prometheus_invalid_key_filtering.py +++ b/tests/unit/integrations/test_prometheus_invalid_key_filtering.py @@ -2,19 +2,63 @@ Unit tests for Prometheus invalid API key request filtering. Tests the 401 detection helpers, that LLM-level metrics skip invalid API key -requests, and that the proxy-level failed request counter still records them. +requests rejected at the proxy gate while recording a provider's 401, and that +the proxy-level failed request counter still records them. """ +from collections.abc import Iterator +from datetime import datetime +from typing import Final from unittest.mock import Mock, patch import pytest from fastapi import HTTPException from prometheus_client import REGISTRY +from prometheus_client.samples import Sample +import litellm +from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL from litellm.integrations.prometheus import PrometheusLogger +from litellm.llms.github_copilot.per_user_auth import GithubCopilotUserSession from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth +PROVIDER_CREDENTIAL_FIELDS = [ + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "aws_web_identity_token", + "azure_ad_token", + "azure_password", + "azure_scope", + "azure_username", + "client_id", + "client_secret", + "gigachat_access_token", + "oci_fingerprint", + "oci_key", + "oci_key_file", + "oci_tenancy", + "oci_user", + "tenant_id", + "token", + "user_config", + "vertex_ai_credentials", + "vertex_credentials", + "watsonx_credentials", + "wx_credentials", + "zen_api_key", +] +PROVIDER_ENDPOINT_FIELDS = [ + "aws_bedrock_runtime_endpoint", + "aws_sts_endpoint", + "deployment_url", + "s3_endpoint_url", + "sagemaker_base_url", +] +CALLED_THE_PROVIDER: Final = {"api_call_start_time": datetime(2026, 10, 19, 12, 0, 0)} + + @pytest.fixture(scope="function") def prometheus_logger(): """Create a PrometheusLogger instance for testing.""" @@ -48,18 +92,13 @@ class TestExtractStatusCode: (ExceptionWithStatusCode, 401, 401), ], ) - def test_extract_from_exception( - self, prometheus_logger, exception_class, code_value, expected - ): + def test_extract_from_exception(self, prometheus_logger, exception_class, code_value, expected): exception = exception_class(code_value) assert prometheus_logger._extract_status_code(exception=exception) == expected def test_extract_from_kwargs(self, prometheus_logger): exception = ExceptionWithCode("401") - assert ( - prometheus_logger._extract_status_code(kwargs={"exception": exception}) - == 401 - ) + assert prometheus_logger._extract_status_code(kwargs={"exception": exception}) == 401 def test_extract_from_enum_values(self, prometheus_logger): enum_values = Mock(status_code="401") @@ -79,50 +118,271 @@ class TestInvalidAPIKeyDetection: ], ) def test_status_code_detection(self, prometheus_logger, status_code, expected): - assert ( - prometheus_logger._is_invalid_api_key_request(status_code=status_code) - == expected - ) + assert prometheus_logger._is_invalid_api_key_request(status_code=status_code) == expected def test_auth_error_message_detection(self, prometheus_logger): exception = AssertionError( "LiteLLM Virtual Key expected. Received=invalid-key-12345, expected to start with 'sk-'." ) - assert ( - prometheus_logger._is_invalid_api_key_request( - status_code=None, exception=exception - ) - is True - ) + assert prometheus_logger._is_invalid_api_key_request(status_code=None, exception=exception) is True def test_non_auth_exception_not_detected(self, prometheus_logger): exception = ValueError("Some other error") - assert ( - prometheus_logger._is_invalid_api_key_request( - status_code=None, exception=exception - ) - is False - ) + assert prometheus_logger._is_invalid_api_key_request(status_code=None, exception=exception) is False class TestSkipMetricsValidation: """Test high-level validation method that orchestrates detection and extraction.""" - def test_skip_for_401_exception(self, prometheus_logger): - """Test full flow: extraction -> detection -> skip decision.""" - exception = ExceptionWithCode("401") - assert ( - prometheus_logger._should_skip_metrics_for_invalid_key(exception=exception) - is True - ) + @pytest.mark.parametrize( + "exception,proxy_gate,expected", + [ + (ExceptionWithCode("401"), True, True), + (AssertionError("expected to start with 'sk-'"), True, True), + (ExceptionWithStatusCode(429), True, False), + (ExceptionWithCode("401"), False, True), + (AssertionError("expected to start with 'sk-'"), False, True), + ( + litellm.AuthenticationError( + message="Incorrect API key provided: sk-rotat*****", + llm_provider="openai", + model="gpt-4o-mini", + ), + False, + False, + ), + ( + litellm.AuthenticationError( + message="API key not valid. Please pass a valid API key.", + llm_provider="gemini", + model="gemini-2.5-flash", + ), + False, + False, + ), + ( + litellm.AuthenticationError( + message="Authentication Error, Invalid proxy server token passed.", + llm_provider="openai", + model="gpt-4o-mini", + ), + True, + True, + ), + ( + litellm.CallerCredentialAuthenticationError( + message="GitHub Copilot connection is missing for this user", + llm_provider="github_copilot", + model="gpt-4o", + ), + False, + True, + ), + ( + HTTPException(status_code=401, detail="caller did not present a token"), + False, + True, + ), + ( + litellm.BadRequestError( + message="API key not valid. Please pass a valid API key.", + llm_provider="gemini", + model="gemini-2.5-flash", + ), + False, + False, + ), + ], + ids=[ + "proxy-gate-401", + "proxy-gate-auth-message", + "proxy-gate-429", + "unmarked-proxy-401", + "unmarked-auth-message", + "provider-401", + "provider-auth-message", + "proxy-gate-litellm-auth-error", + "caller-credential-401", + "unmarked-http-exception-401", + "provider-400-auth-message", + ], + ) + def test_skip_except_for_provider_errors_the_proxy_did_not_raise( + self, prometheus_logger, exception, proxy_gate, expected + ): + kwargs: Final[dict[str, object]] = {"exception": exception, **CALLED_THE_PROVIDER} + if proxy_gate: + kwargs[LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL] = True + assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is expected - def test_skip_for_auth_error_message(self, prometheus_logger): - """Test full flow: exception message -> detection -> skip decision.""" - exception = AssertionError("expected to start with 'sk-'") - assert ( - prometheus_logger._should_skip_metrics_for_invalid_key(exception=exception) - is True + @pytest.mark.parametrize( + "logged_call,expected", + [ + ({"litellm_params": {"proxy_server_request": {"caller_credential_fields": ("api_key",)}}}, True), + ( + { + "litellm_params": { + "github_copilot_user_session": GithubCopilotUserSession( + token="caller-session-token", api_base="https://api.githubcopilot.com" + ) + } + }, + True, + ), + ({"standard_logging_object": {"metadata": {"used_client_oauth_token": True}}}, True), + ({"litellm_params": {"proxy_server_request": {"caller_credential_fields": ("headers",)}}}, True), + ( + { + "litellm_params": { + "proxy_server_request": {"caller_credential_fields": ("provider_specific_header",)} + } + }, + True, + ), + ({"litellm_params": {"proxy_server_request": {"caller_credential_fields": ()}}}, False), + ({"litellm_params": {"proxy_server_request": {"caller_credential_fields": "api_key"}}}, False), + ({"standard_logging_object": {"metadata": {"used_client_oauth_token": False}}}, False), + ({"litellm_params": {"proxy_server_request": {"body": {"azure_ad_token": "caller-token"}}}}, True), + ( + { + "litellm_params": { + "proxy_server_request": {"body": {"extra_body": {"azure_ad_token": "caller-token"}}} + } + }, + True, + ), + ( + { + "litellm_params": { + "azure_ad_token": "deployment-token", + "proxy_server_request": {"body": {"model": "azure-deployment"}}, + } + }, + False, + ), + ({"litellm_params": {"proxy_server_request": {"body": {"extra_body": "azure_ad_token"}}}}, False), + ( + { + "litellm_params": { + "proxy_server_request": {"body": {"litellm_credential_name": "caller-credential"}} + } + }, + True, + ), + ( + { + "litellm_params": { + "litellm_credential_name": "deployment-credential", + "proxy_server_request": {"body": {"model": "openai-deployment"}}, + } + }, + False, + ), + ({"litellm_params": {"proxy_server_request": {"caller_credential_fields": ("extra_headers",)}}}, True), + ({"litellm_params": {"proxy_server_request": {"caller_credential_fields": ("organization",)}}}, True), + ( + { + "litellm_params": { + "proxy_server_request": {"credential_fields": ("headers",), "caller_credential_fields": ()} + } + }, + False, + ), + ( + { + "litellm_params": { + "proxy_server_request": {"body": {"extra_headers": {"Authorization": "Bearer sk-caller"}}} + } + }, + False, + ), + ], + ids=[ + "caller-supplied-api-key", + "caller-copilot-session", + "caller-anthropic-oauth-token", + "caller-supplied-headers", + "caller-supplied-provider-specific-header", + "no-caller-credential-fields", + "malformed-credential-fields", + "deployment-anthropic-key", + "caller-supplied-azure-ad-token", + "caller-supplied-azure-ad-token-in-extra-body", + "deployment-azure-ad-token", + "malformed-extra-body", + "caller-named-credential", + "deployment-named-credential", + "caller-supplied-extra-headers", + "caller-supplied-organization", + "headers-that-cannot-change-provider-auth", + "extra-headers-not-stamped-by-the-proxy", + ], + ) + def test_skip_a_provider_401_on_a_credential_the_caller_owns(self, prometheus_logger, logged_call, expected): + exception: Final = litellm.AuthenticationError( + message="Incorrect API key provided: sk-calle*****", + llm_provider="openai", + model="gpt-4o-mini", ) + kwargs: Final = {"exception": exception, **CALLED_THE_PROVIDER, **logged_call} + assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is expected + + @pytest.mark.parametrize( + "logged_call,expected", + [(CALLED_THE_PROVIDER, False), ({}, True), ({"api_call_start_time": None}, True)], + ids=["provider-401", "401-before-the-provider-call", "blank-provider-call-start"], + ) + def test_skip_a_401_raised_before_the_provider_was_called(self, prometheus_logger, logged_call, expected): + exception: Final = litellm.AuthenticationError( + message="Missing Anthropic API Key - A call is being made to anthropic but no key is set", + llm_provider="anthropic", + model="claude-haiku-4-5", + ) + kwargs: Final = {"exception": exception, **logged_call} + assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is expected + + @pytest.mark.parametrize("field", PROVIDER_CREDENTIAL_FIELDS + PROVIDER_ENDPOINT_FIELDS) + @pytest.mark.parametrize("inside_extra_body", [False, True], ids=["body", "extra-body"]) + def test_skip_a_provider_401_on_a_provider_credential_field_the_caller_sent( + self, prometheus_logger, field, inside_extra_body + ): + sent: Final = {field: "caller-credential"} + body: Final = {"model": "shared-deployment", **({"extra_body": sent} if inside_extra_body else sent)} + kwargs: Final = { + "exception": litellm.AuthenticationError(message="Unauthorized", llm_provider="azure", model="gpt-4o"), + "litellm_params": {"proxy_server_request": {"body": body}}, + **CALLED_THE_PROVIDER, + } + assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is True + + @pytest.mark.parametrize("field", PROVIDER_CREDENTIAL_FIELDS + PROVIDER_ENDPOINT_FIELDS) + def test_count_a_provider_401_on_a_provider_credential_field_the_deployment_owns(self, prometheus_logger, field): + kwargs: Final = { + "exception": litellm.AuthenticationError(message="Unauthorized", llm_provider="azure", model="gpt-4o"), + **CALLED_THE_PROVIDER, + "litellm_params": { + field: "deployment-credential", + "proxy_server_request": {"body": {"model": "shared-deployment"}}, + }, + } + assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is False + + @pytest.mark.parametrize( + "token_exchange_endpoint,expected", + [("https://login.example.com/oauth2/v2.0/token", True), ("", False), (None, False)], + ids=["token-exchange-deployment", "blank-endpoint", "no-endpoint"], + ) + def test_skip_a_provider_401_on_a_token_exchange_deployment( + self, prometheus_logger, token_exchange_endpoint, expected + ): + kwargs: Final = { + "exception": litellm.AuthenticationError( + message="Unauthorized", llm_provider="microsoft_365_copilot", model="copilot" + ), + "litellm_params": {"token_exchange_endpoint": token_exchange_endpoint, "client_secret": "deployment"}, + **CALLED_THE_PROVIDER, + } + assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is expected def test_no_skip_for_valid_request(self, prometheus_logger): assert prometheus_logger._should_skip_metrics_for_invalid_key() is False @@ -147,19 +407,13 @@ class TestAsyncHooks: ), ], ) - async def test_post_call_failure_hook_counts_401_without_key_hash( - self, prometheus_logger, exception - ): + async def test_post_call_failure_hook_counts_401_without_key_hash(self, prometheus_logger, exception): unauthenticated = UserAPIKeyAuth(request_route="/v1/chat/completions") unauthenticated.api_key = "notakeyatall" with ( - patch.object( - prometheus_logger, "litellm_proxy_failed_requests_metric" - ) as mock_failed, - patch.object( - prometheus_logger, "litellm_proxy_total_requests_metric" - ) as mock_total, + patch.object(prometheus_logger, "litellm_proxy_failed_requests_metric") as mock_failed, + patch.object(prometheus_logger, "litellm_proxy_total_requests_metric") as mock_total, ): await prometheus_logger.async_post_call_failure_hook( request_data={"model": "test-model"}, @@ -176,9 +430,7 @@ class TestAsyncHooks: mock_total.labels.return_value.inc.assert_called_once() @pytest.mark.asyncio - async def test_post_call_failure_hook_keeps_resolved_identity_labels_for_401( - self, prometheus_logger - ): + async def test_post_call_failure_hook_keeps_resolved_identity_labels_for_401(self, prometheus_logger): expired_key = UserAPIKeyAuth( api_key="sk-expired", key_alias="expired-alias", @@ -191,9 +443,7 @@ class TestAsyncHooks: code=401, ) - with patch.object( - prometheus_logger, "litellm_proxy_failed_requests_metric" - ) as mock_failed: + with patch.object(prometheus_logger, "litellm_proxy_failed_requests_metric") as mock_failed: await prometheus_logger.async_post_call_failure_hook( request_data={"model": "test-model"}, original_exception=exception, @@ -207,7 +457,7 @@ class TestAsyncHooks: assert failed_labels["team"] == "team-1" @pytest.mark.asyncio - async def test_log_failure_event_skips_401(self, prometheus_logger): + async def test_log_failure_event_skips_401_rejected_at_the_proxy_gate(self, prometheus_logger): exception = ExceptionWithCode("401") kwargs = { "model": "test-model", @@ -220,20 +470,91 @@ class TestAsyncHooks: }, "exception": exception, "litellm_params": {}, + LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: True, } with ( - patch.object( - prometheus_logger, "litellm_llm_api_failed_requests_metric" - ) as mock_failed, - patch.object( - prometheus_logger, "set_llm_deployment_failure_metrics" - ) as mock_deployment, + patch.object(prometheus_logger, "litellm_llm_api_failed_requests_metric") as mock_failed, + patch.object(prometheus_logger, "set_llm_deployment_failure_metrics") as mock_deployment, ): - await prometheus_logger.async_log_failure_event( kwargs=kwargs, response_obj=None, start_time=None, end_time=None ) mock_failed.labels.assert_not_called() mock_deployment.assert_not_called() + + +def _provider_auth_failure_kwargs() -> dict[str, object]: + exception: Final = litellm.AuthenticationError( + message="Incorrect API key provided: sk-rotat*****", + llm_provider="openai", + model="gpt-4o-mini", + ) + return { + "model": "gpt-4o-mini", + "standard_logging_object": { + "metadata": { + "user_api_key_hash": "test-key", + "user_api_key_alias": "test-alias", + "user_api_key_user_id": "test-user", + "user_api_key_team_id": "test-team", + "user_api_key_team_alias": "test-team-alias", + }, + "model_group": "rotated-credential", + "model_id": "deployment-1", + "api_base": "https://api.openai.com/v1", + "custom_llm_provider": "openai", + }, + "exception": exception, + "litellm_params": {"custom_llm_provider": "openai", "metadata": {}}, + **CALLED_THE_PROVIDER, + } + + +def _metric_values(name: str, model_id: str) -> list[float]: + return [ + sample.value + for sample in _registry_samples() + if sample.name == name and sample.labels.get("model_id") == model_id + ] + + +def _registry_samples() -> Iterator[Sample]: + for metric in REGISTRY.collect(): + yield from metric.samples + + +class TestProviderAuthFailures: + """A 401 raised by the provider is an upstream failure and is recorded.""" + + @pytest.mark.asyncio + async def test_log_failure_event_records_a_provider_401(self, prometheus_logger): + await prometheus_logger.async_log_failure_event( + kwargs=_provider_auth_failure_kwargs(), + response_obj=None, + start_time=None, + end_time=None, + ) + + assert _metric_values("litellm_llm_api_failed_requests_metric_total", "deployment-1") == [1.0] + assert _metric_values("litellm_deployment_failure_responses_total", "deployment-1") == [1.0] + assert _metric_values("litellm_deployment_state", "deployment-1") == [1.0] + + def test_deployment_failure_metrics_record_a_provider_401(self, prometheus_logger): + prometheus_logger.set_llm_deployment_failure_metrics(_provider_auth_failure_kwargs()) + + failures: Final = [ + sample for sample in _registry_samples() if sample.name == "litellm_deployment_failure_responses_total" + ] + assert [ + ( + sample.labels["model_id"], + sample.labels["exception_status"], + sample.labels["api_provider"], + sample.value, + ) + for sample in failures + ] == [("deployment-1", "401", "openai", 1.0)] + assert _metric_values("litellm_deployment_total_requests_total", "deployment-1") == [1.0] + assert _metric_values("litellm_deployment_state", "deployment-1") == [1.0] diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index ed7817bd660..e03fd9850c1 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -927,6 +927,222 @@ async def test_add_litellm_data_to_request_body_snapshot_excludes_proxy_server_r assert snapshot_body["messages"] == [{"role": "user", "content": "hello"}] +FORWARD_PROVIDER_AUTH_HEADERS: Final = { + "forward_llm_provider_auth_headers": True, + "forward_client_headers_to_llm_api": True, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "sent,request_headers,general_settings,expected", + [ + ({"headers": {"Authorization": "Bearer sk-caller"}}, {}, {}, ("headers",)), + ({"headers": {"X-Trace-Id": "trace-1"}}, {}, {}, ("headers",)), + ({"headers": {}}, {}, {}, ()), + ({"extra_headers": {"API-Key": "caller-azure-key"}}, {}, {}, ("extra_headers",)), + ({"extra_headers": {"OpenAI-Project": "proj_caller"}}, {}, {}, ("extra_headers",)), + ( + {"provider_specific_header": {"custom_llm_provider": "openai", "extra_headers": {"x-api-key": "caller"}}}, + {}, + {}, + ("provider_specific_header",), + ), + ( + { + "provider_specific_header": [ + {"custom_llm_provider": "anthropic", "extra_headers": {"anthropic-beta": "tools"}}, + {"custom_llm_provider": "gemini", "extra_headers": {"X-Goog-Api-Key": "caller"}}, + ] + }, + {}, + {}, + ("provider_specific_header",), + ), + ( + { + "provider_specific_header": { + "custom_llm_provider": "anthropic", + "extra_headers": {"x-caller-trace": "x"}, + } + }, + {}, + {}, + ("provider_specific_header",), + ), + ( + { + "provider_specific_header": { + "custom_llm_provider": "anthropic", + "extra_headers": {"anthropic-version": "2023-06-01", "anthropic-beta": "tools-2024-04-04"}, + } + }, + {}, + {}, + (), + ), + ({"headers": {"Anthropic-Version": "2023-06-01"}}, {}, {}, ()), + ({"organization": "org-caller"}, {}, {}, ("organization",)), + ({"headers": "authorization", "provider_specific_header": "x-api-key"}, {}, {}, ()), + ( + {"api_key": "sk-caller", "headers": {"Ocp-Apim-Subscription-Key": "caller"}}, + {}, + {}, + ("api_key", "headers"), + ), + ({}, {"anthropic-beta": "tools-2024-04-04"}, {}, ()), + ({}, {"anthropic-version": "2023-06-01", "anthropic-beta": "tools-2024-04-04"}, {}, ()), + ({}, {"openai-organization": "org-caller"}, {"forward_openai_org_id": True}, ("organization",)), + ( + {}, + {"x-litellm-api-key": "sk-virtual", "authorization": "Bearer sk-ant-oat01-caller"}, + {}, + ("provider_specific_header",), + ), + ( + {}, + {"authorization": "Bearer sk-virtual", "x-goog-api-key": "AIza-caller"}, + FORWARD_PROVIDER_AUTH_HEADERS, + ("headers",), + ), + ( + {}, + {"authorization": "Bearer sk-virtual", "x-portkey-api-key": "caller-gateway-key"}, + FORWARD_PROVIDER_AUTH_HEADERS, + ("headers",), + ), + ({}, {"authorization": "Bearer sk-virtual"}, FORWARD_PROVIDER_AUTH_HEADERS, ()), + ], + ids=[ + "authorization-in-headers", + "plain-headers", + "empty-headers", + "api-key-in-extra-headers", + "openai-project-in-extra-headers", + "credential-in-provider-specific-header", + "credential-in-one-provider-specific-header-entry", + "plain-provider-specific-header", + "anthropic-version-and-beta-in-provider-specific-header", + "anthropic-version-in-headers", + "organization", + "malformed-header-fields", + "api-key-and-credential-headers", + "proxy-forwarded-anthropic-beta", + "proxy-forwarded-anthropic-sdk-headers", + "proxy-forwarded-openai-organization", + "proxy-forwarded-anthropic-oauth-token", + "proxy-forwarded-provider-auth-header", + "proxy-forwarded-gateway-key", + "proxy-forwarding-nothing", + ], +) +async def test_caller_credential_fields_name_every_field_that_carries_a_caller_header_or_credential( + sent: dict[str, object], + request_headers: dict[str, str], + general_settings: dict[str, object], + expected: tuple[str, ...], +) -> None: + updated: Final = await _request_data_added_by_the_proxy(sent, request_headers, general_settings) + + assert updated["proxy_server_request"]["caller_credential_fields"] == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_headers,expected", + [({}, ()), ({"x-caller-trace": "trace-1"}, ("headers",))], + ids=["user-information-only", "user-information-and-caller-header"], +) +async def test_caller_credential_fields_ignore_the_user_information_the_proxy_adds_to_forwarded_headers( + monkeypatch: pytest.MonkeyPatch, request_headers: dict[str, str], expected: tuple[str, ...] +) -> None: + monkeypatch.setattr(litellm, "add_user_information_to_llm_headers", True) + + updated: Final = await _request_data_added_by_the_proxy({}, request_headers, FORWARD_PROVIDER_AUTH_HEADERS) + + assert updated["headers"]["x-litellm-user_api_key_user_id"] == "test-user" + assert updated["proxy_server_request"]["caller_credential_fields"] == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "sent,request_headers,general_settings,expected", + [ + ({"headers": {"Anthropic-Version": "2023-06-01"}}, {}, {}, ("headers",)), + ( + { + "provider_specific_header": { + "custom_llm_provider": "anthropic", + "extra_headers": {"anthropic-version": "2023-06-01"}, + } + }, + {}, + {}, + ("provider_specific_header",), + ), + ({"organization": "org-caller"}, {}, {}, ()), + ({"extra_headers": {"API-Key": "caller-azure-key"}}, {}, {}, ()), + ( + {"api_key": "sk-caller", "headers": {"Ocp-Apim-Subscription-Key": "caller"}}, + {}, + {}, + ("api_key", "headers"), + ), + ({}, {"openai-organization": "org-caller"}, {"forward_openai_org_id": True}, ()), + ( + {}, + {"authorization": "Bearer sk-virtual", "x-goog-api-key": "AIza-caller"}, + FORWARD_PROVIDER_AUTH_HEADERS, + (), + ), + ], + ids=[ + "anthropic-version-in-headers", + "anthropic-version-in-provider-specific-header", + "organization", + "api-key-in-extra-headers", + "api-key-and-credential-headers", + "proxy-forwarded-openai-organization", + "proxy-forwarded-provider-auth-header", + ], +) +async def test_credential_fields_name_the_stripped_fields_the_caller_sent( + sent: dict[str, object], + request_headers: dict[str, str], + general_settings: dict[str, object], + expected: tuple[str, ...], +) -> None: + updated: Final = await _request_data_added_by_the_proxy(sent, request_headers, general_settings) + + assert updated["proxy_server_request"]["credential_fields"] == expected + + +async def _request_data_added_by_the_proxy( + sent: dict[str, object], request_headers: dict[str, str], general_settings: dict[str, object] +) -> dict: + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + + request_mock: Final = MagicMock(spec=Request) + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.url.path = "/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"content-type": "application/json", **request_headers} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + data: Final = {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hello"}], **sent} + + return await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", user_id="test-user"), + proxy_config=MagicMock(), + general_settings=dict(general_settings), + version="test-version", + ) + + def test_initial_snapshot_refresh_clears_a_previous_guardrail_checkpoint() -> None: from litellm.integrations.shadow_eval_logger import GuardrailRequestSnapshot from litellm.litellm_core_utils.litellm_logging import Logging From 11779b260e4f6ef6fdd7e7add735baef4fd10e3b Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 26 Oct 2026 20:58:27 -0400 Subject: [PATCH 107/155] fix(ui): include endpoint-qualified models in Decisions Playground (#48504) --- .../playground/llm_calls/system_one.test.ts | 21 ++++++++++++++++++- .../playground/llm_calls/system_one.ts | 12 ++++++++--- .../src/lib/decisionModels.ts | 2 +- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.test.ts index 90e9482c89c..6fb1f8c8b8d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.test.ts @@ -207,18 +207,37 @@ describe("fetchDecisionModels", () => { vi.unstubAllGlobals(); }); - it("lists the key's decision models from the proxy that answers the request, sorted by name", async () => { + it("lists the key's models with decision modes or endpoints from the request proxy, sorted by name", async () => { mockFetch.mockResolvedValueOnce( modelGroupInfo([ { model_group: "pplx-decider", mode: "evaluation" }, { model_group: "gpt-5.5", mode: "chat" }, { model_group: "jev-latest", mode: "evaluation" }, { model_group: "unknown-mode", mode: null }, + { model_group: "native-decision", mode: "decisions" }, + { + model_group: "chat-decisions", + mode: "chat", + supported_endpoints: ["/v1/chat/completions", "/v1/decisions"], + }, + { model_group: "chat-systemone", mode: "chat", supported_endpoints: ["/v1/systemone"] }, + { model_group: "custom-decision", mode: "custom", supported_endpoints: ["/v1/decisions"] }, + { model_group: "endpoint-only", supported_endpoints: ["/v1/decisions"] }, + { model_group: "chat-only", mode: "chat", supported_endpoints: ["/v1/chat/completions"] }, + { model_group: "no-endpoints", mode: "chat", supported_endpoints: [] }, + { model_group: "null-endpoints", mode: "chat", supported_endpoints: null }, + { model_group: "missing-mode" }, + { model_group: "custom-mode", mode: "custom" }, ]), ); await expect(fetchDecisionModels("virtual-key", "https://custom.example.com/")).resolves.toEqual([ + "chat-decisions", + "chat-systemone", + "custom-decision", + "endpoint-only", "jev-latest", + "native-decision", "pplx-decider", ]); expect(mockFetch).toHaveBeenCalledWith( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts index a4030029903..035a05729ba 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts @@ -2,7 +2,7 @@ import { z } from "zod"; import { getGatewayBaseUrl, getGlobalLitellmHeaderName, getProxyBaseUrl } from "@/components/networking"; import { withRequiredHeaders } from "@/components/llm_calls/request_headers"; import { createApiClient } from "@/lib/http/client"; -import { isDecisionMode } from "@/lib/decisionModels"; +import { isDecisionEntry } from "@/lib/decisionModels"; import { systemOneResponseSchema, type PlaygroundRequest, @@ -26,7 +26,13 @@ export type PlaygroundDecisionRequest = | { endpoint: "/v1/decisions"; payload: OpenAIDecisionsRequest }; const modelGroupInfoSchema = z.object({ - data: z.array(z.object({ model_group: z.string(), mode: z.string().nullish() })), + data: z.array( + z.object({ + model_group: z.string(), + mode: z.string().nullish(), + supported_endpoints: z.array(z.string()).nullish(), + }), + ), }); function proxyClient(customBaseUrl?: string, getDefaultBaseUrl: () => string = getProxyBaseUrl) { @@ -56,7 +62,7 @@ export async function fetchDecisionModels( }); return modelGroupInfoSchema .parse(body) - .data.filter((group) => isDecisionMode(group.mode)) + .data.filter(isDecisionEntry) .map((group) => group.model_group) .sort((a, b) => a.localeCompare(b)); } diff --git a/ui/litellm-dashboard/src/lib/decisionModels.ts b/ui/litellm-dashboard/src/lib/decisionModels.ts index 1808b13c62b..15c632fe055 100644 --- a/ui/litellm-dashboard/src/lib/decisionModels.ts +++ b/ui/litellm-dashboard/src/lib/decisionModels.ts @@ -38,7 +38,7 @@ export const displayMode = (mode: string | null | undefined): string | undefined export const expandModeFilterValues = (modes: readonly string[]): readonly string[] => modes.flatMap((mode) => (mode === DECISIONS_MODE ? DECISION_MODE_VALUES : [mode])); -const isDecisionEntry = (entry: CostMapEntry): boolean => +export const isDecisionEntry = (entry: Pick): boolean => isDecisionMode(entry.mode) || (entry.supported_endpoints ?? []).some((endpoint) => DECISION_ENDPOINTS.has(endpoint)); const parsedEntries = (costMap: unknown): ReadonlyArray => { From 45183428a54043186a401e419b8c0b09df682169 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 26 Oct 2026 21:00:39 -0400 Subject: [PATCH 108/155] feat(ui): persist model group alias display ordering (#48500) --- ui/litellm-dashboard/eslint-suppressions.json | 3 - .../ModelGroupAliasPanel.integration.test.tsx | 1 + .../panels/ModelGroupAliasPanel.tsx | 5 +- .../model_group_alias_ordering.test.ts | 76 ++++++ .../components/model_group_alias_ordering.ts | 57 +++++ ..._group_alias_settings.integration.test.tsx | 218 ++++++++++++++++++ .../model_group_alias_settings.test.tsx | 66 ------ .../components/model_group_alias_settings.tsx | 186 ++++++++++++++- 8 files changed, 533 insertions(+), 79 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/model_group_alias_ordering.test.ts create mode 100644 ui/litellm-dashboard/src/components/model_group_alias_ordering.ts create mode 100644 ui/litellm-dashboard/src/components/model_group_alias_settings.integration.test.tsx delete mode 100644 ui/litellm-dashboard/src/components/model_group_alias_settings.test.tsx diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 32777b43504..724c49eb3b3 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1730,9 +1730,6 @@ "src/components/model_group_alias_settings.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "react-hooks/set-state-in-effect": { - "count": 1 } }, "src/components/model_info_view.tsx": { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelGroupAliasPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelGroupAliasPanel.integration.test.tsx index 227a34335a3..36a9f92dfd4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelGroupAliasPanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelGroupAliasPanel.integration.test.tsx @@ -10,6 +10,7 @@ vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ vi.mock("@/components/networking", () => ({ getCallbacksCall: vi.fn(), + getProxyBaseUrl: () => "https://proxy.test", getRouterSettingsCall: vi.fn(), setCallbacksCall: vi.fn(), })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelGroupAliasPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelGroupAliasPanel.tsx index 025923e08e9..e6c720f38cf 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelGroupAliasPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelGroupAliasPanel.tsx @@ -2,7 +2,7 @@ import { useEffect, useState } from "react"; import ModelGroupAliasSettings from "@/components/model_group_alias_settings"; -import { getCallbacksCall, getRouterSettingsCall } from "@/components/networking"; +import { getCallbacksCall, getProxyBaseUrl, getRouterSettingsCall } from "@/components/networking"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; export default function ModelGroupAliasPanel() { @@ -22,6 +22,7 @@ export default function ModelGroupAliasPanel() { : null; const managedByConfig = currentAliasOwnership?.managedByConfig ?? false; const ownershipLoading = Boolean(accessToken && !currentAliasOwnership); + const preferenceScope = userID ? JSON.stringify([getProxyBaseUrl(), userID]) : null; useEffect(() => { if (!accessToken) { @@ -62,6 +63,8 @@ export default function ModelGroupAliasPanel() { return ( ({ + id: aliasName, + aliasName, + targetModelGroup, +}); +const names = (aliases: readonly AliasItem[]): string[] => aliases.map((alias) => alias.aliasName); + +describe("orderAliases", () => { + const aliases = Object.freeze([row("Zebra", "alpha10"), row("alpha10", "beta"), row("alpha2", "Alpha2")]); + + it.each<{ sortBy: AliasDisplayPreferences["sortBy"]; descending: boolean; expected: string[] }>([ + { sortBy: "aliasName", descending: false, expected: ["alpha2", "alpha10", "Zebra"] }, + { sortBy: "aliasName", descending: true, expected: ["Zebra", "alpha10", "alpha2"] }, + { sortBy: "targetModelGroup", descending: false, expected: ["alpha2", "Zebra", "alpha10"] }, + { sortBy: "targetModelGroup", descending: true, expected: ["alpha10", "Zebra", "alpha2"] }, + ])("sorts $sortBy with descending=$descending without changing the input", ({ sortBy, descending, expected }) => { + expect(names(orderAliases(aliases, { ...DEFAULT_ALIAS_DISPLAY, sortBy, descending }))).toEqual(expected); + expect(names(aliases)).toEqual(["Zebra", "alpha10", "alpha2"]); + }); + + it.each([false, true])( + "breaks equal-target ties by alias name regardless of input order, descending=%s", + (descending) => { + const tied = [row("beta", "shared"), row("Alpha", "SHARED"), row("gamma", "shared")]; + const preferences: AliasDisplayPreferences = { ...DEFAULT_ALIAS_DISPLAY, sortBy: "targetModelGroup", descending }; + expect(names(orderAliases(tied, preferences))).toEqual(["Alpha", "beta", "gamma"]); + expect(names(orderAliases([...tied].reverse(), preferences))).toEqual(["Alpha", "beta", "gamma"]); + }, + ); + + it.each([ + { descending: false, remaining: ["alpha", "beta", "subscription/a", "subscription/c"] }, + { descending: true, remaining: ["beta", "alpha", "subscription/c", "subscription/a"] }, + ])("keeps ordered pins first and alias-prefix matches last, descending=$descending", ({ descending, remaining }) => { + const grouped = [ + row("subscription/c", "c"), + row("beta", "b"), + row("ordinary", "y"), + row("subscription/a", "a"), + row("alpha", "subscription/x"), + row("subscription/z", "z"), + ]; + const preferences = { + ...DEFAULT_ALIAS_DISPLAY, + descending, + pinned: ["missing", "subscription/z", "ordinary"], + lastPrefix: "subscription/", + }; + expect(names(orderAliases(grouped, preferences))).toEqual(["subscription/z", "ordinary", ...remaining]); + }); +}); + +describe("movePinnedAlias", () => { + const pinned = Object.freeze(["first", "middle", "last"]); + + it.each<{ name: string; direction: -1 | 1; expected: string[] }>([ + { name: "middle", direction: -1, expected: ["middle", "first", "last"] }, + { name: "middle", direction: 1, expected: ["first", "last", "middle"] }, + { name: "first", direction: -1, expected: ["first", "middle", "last"] }, + { name: "last", direction: 1, expected: ["first", "middle", "last"] }, + { name: "missing", direction: -1, expected: ["first", "middle", "last"] }, + { name: "missing", direction: 1, expected: ["first", "middle", "last"] }, + ])("moves $name in direction $direction without changing the input", ({ name, direction, expected }) => { + expect(movePinnedAlias(pinned, name, direction)).toEqual(expected); + expect(pinned).toEqual(["first", "middle", "last"]); + }); +}); diff --git a/ui/litellm-dashboard/src/components/model_group_alias_ordering.ts b/ui/litellm-dashboard/src/components/model_group_alias_ordering.ts new file mode 100644 index 00000000000..cbec78b63cb --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_group_alias_ordering.ts @@ -0,0 +1,57 @@ +import { z } from "zod"; +import { storageKey } from "@/lib/storage"; + +export interface AliasItem { + id: string; + aliasName: string; + targetModelGroup: string; +} + +const aliasDisplayFields = { + sortBy: z.enum(["aliasName", "targetModelGroup"]), + descending: z.boolean(), + pinned: z.array(z.string()).transform((names) => [...new Set(names)]), + lastPrefix: z.string(), +}; +const aliasDisplaySchema = z.object(aliasDisplayFields); + +export type AliasDisplayPreferences = z.infer; + +export const DEFAULT_ALIAS_DISPLAY: AliasDisplayPreferences = { + sortBy: "aliasName", + descending: false, + pinned: [], + lastPrefix: "", +}; + +export const aliasDisplayStorage = (scope: string) => + storageKey("local", `litellm_alias_display_v1_${scope}`, aliasDisplaySchema, DEFAULT_ALIAS_DISPLAY); + +const alphabetical = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }); + +export function orderAliases(aliases: readonly AliasItem[], preferences: AliasDisplayPreferences): AliasItem[] { + const ranks = new Map(preferences.pinned.map((name, index) => [name, index])); + const group = (name: string): number => { + if (ranks.has(name)) return 0; + return preferences.lastPrefix && name.startsWith(preferences.lastPrefix) ? 2 : 1; + }; + return [...aliases].sort((a, b) => { + const groupDifference = group(a.aliasName) - group(b.aliasName); + if (groupDifference) return groupDifference; + if (ranks.has(a.aliasName) && ranks.has(b.aliasName)) { + return ranks.get(a.aliasName)! - ranks.get(b.aliasName)!; + } + const fieldDifference = alphabetical.compare(a[preferences.sortBy], b[preferences.sortBy]); + return fieldDifference * (preferences.descending ? -1 : 1) || a.aliasName.localeCompare(b.aliasName); + }); +} + +export function movePinnedAlias(pinned: readonly string[], name: string, direction: -1 | 1): string[] { + const index = pinned.indexOf(name); + const target = index + direction; + if (index < 0 || target < 0 || target >= pinned.length) return [...pinned]; + return pinned.map((value, position) => { + if (position === index) return pinned[target]; + return position === target ? name : value; + }); +} diff --git a/ui/litellm-dashboard/src/components/model_group_alias_settings.integration.test.tsx b/ui/litellm-dashboard/src/components/model_group_alias_settings.integration.test.tsx new file mode 100644 index 00000000000..816f5629567 --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_group_alias_settings.integration.test.tsx @@ -0,0 +1,218 @@ +/* @vitest-environment jsdom */ +import { act, fireEvent, render, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { setCallbacksCall } from "@/components/networking"; +import { aliasDisplayStorage } from "./model_group_alias_ordering"; +import ModelGroupAliasSettings from "./model_group_alias_settings"; + +vi.mock("@/components/networking", () => ({ + setCallbacksCall: vi.fn(), +})); + +const BLOCKED_STORAGE = { + getItem: () => null, + setItem: () => { + throw new Error("storage blocked"); + }, +}; + +const aliases = { + "fast-model": "claude-haiku", + "smart-model": { model: "claude-sonnet", hidden: true }, +}; + +const displayAliases = { + zeta: "standard", + "subscription/zeta": "standard", + tester: "standard", + alpha: "subscription/fast", + reviewer: { model: "subscription/reasoning", hidden: true }, + "subscription/alpha": "standard", +}; + +const rowNames = () => + within(screen.getByRole("table")) + .getAllByRole("row") + .slice(1) + .map((row) => within(row).getAllByRole("cell")[0].textContent); + +describe("ModelGroupAliasSettings", () => { + beforeEach(() => { + vi.clearAllMocks(); + localStorage.clear(); + vi.mocked(setCallbacksCall).mockResolvedValue({}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); + + it("locks config-owned aliases while keeping their names and targets visible", async () => { + render( + , + ); + + expect( + screen.getByText( + "These aliases are defined in config.yaml and are read-only here. Edit the config file to change them.", + ), + ).toBeInTheDocument(); + expect(screen.getByPlaceholderText("e.g., gpt-4o")).toBeDisabled(); + expect(screen.getByPlaceholderText("e.g., gpt-4o-mini-openai")).toBeDisabled(); + expect(screen.getByRole("button", { name: "Add Alias" })).toBeDisabled(); + expect(screen.getByRole("button", { name: "Edit alias fast-model" })).toBeDisabled(); + expect(screen.getByRole("button", { name: "Delete alias fast-model" })).toBeDisabled(); + expect(screen.getByRole("button", { name: "Edit alias smart-model" })).toBeDisabled(); + expect(screen.getByRole("button", { name: "Delete alias smart-model" })).toBeDisabled(); + + expect(await screen.findByText("fast-model")).toBeInTheDocument(); + expect(screen.getByText("claude-haiku")).toBeInTheDocument(); + expect(screen.getByText("smart-model")).toBeInTheDocument(); + expect(screen.getByText("claude-sonnet")).toBeInTheDocument(); + + await userEvent.setup().click(screen.getByRole("button", { name: "Delete alias fast-model" })); + expect(setCallbacksCall).not.toHaveBeenCalled(); + }); + + it("keeps aliases editable by default and saves the current string representation after delete", async () => { + const user = userEvent.setup(); + render( + , + ); + + expect(await screen.findByText("fast-model")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Edit alias fast-model" })).toBeEnabled(); + expect(screen.getByRole("button", { name: "Delete alias fast-model" })).toBeEnabled(); + + await user.click(screen.getByRole("button", { name: "Delete alias fast-model" })); + + expect(setCallbacksCall).toHaveBeenCalledWith("test-token", { + router_settings: { model_group_alias: { "smart-model": "claude-sonnet" } }, + }); + }); + + it("keeps ordered pins above sorted groups across remounts without changing routing", async () => { + const user = userEvent.setup(); + const originalAliases = structuredClone(displayAliases); + const props = { + preferenceScope: "test-user-proxy", + accessToken: "test-token", + initialModelGroupAlias: displayAliases, + managedByConfig: true, + }; + const view = render(); + + await user.click(screen.getByRole("button", { name: "Pin alias tester" })); + await user.click(screen.getByRole("button", { name: "Pin alias reviewer" })); + await user.click(screen.getByRole("button", { name: "Move reviewer up" })); + fireEvent.change(screen.getByLabelText("Place aliases with this prefix last"), { + target: { value: "subscription/" }, + }); + expect(rowNames()).toEqual(["reviewer", "tester", "alpha", "zeta", "subscription/alpha", "subscription/zeta"]); + + await user.click(screen.getByRole("button", { name: "Alias Name" })); + expect(rowNames()).toEqual(["reviewer", "tester", "zeta", "alpha", "subscription/zeta", "subscription/alpha"]); + expect(screen.getByRole("columnheader", { name: "Alias Name" })).toHaveAttribute("aria-sort", "descending"); + await user.click(screen.getByRole("button", { name: "Target Model Group" })); + expect(rowNames()).toEqual(["reviewer", "tester", "zeta", "alpha", "subscription/alpha", "subscription/zeta"]); + await user.click(screen.getByRole("button", { name: "Target Model Group" })); + expect(rowNames()).toEqual(["reviewer", "tester", "alpha", "zeta", "subscription/alpha", "subscription/zeta"]); + view.unmount(); + render(); + expect(rowNames()).toEqual(["reviewer", "tester", "alpha", "zeta", "subscription/alpha", "subscription/zeta"]); + expect(screen.getByLabelText("Place aliases with this prefix last")).toHaveValue("subscription/"); + expect(screen.getByRole("button", { name: "Move reviewer up" })).toBeDisabled(); + expect(setCallbacksCall).not.toHaveBeenCalled(); + expect(displayAliases).toEqual(originalAliases); + + await user.click(screen.getByRole("button", { name: "Reset display order" })); + expect(rowNames()).toEqual(["alpha", "reviewer", "subscription/alpha", "subscription/zeta", "tester", "zeta"]); + expect(screen.getByRole("columnheader", { name: "Alias Name" })).toHaveAttribute("aria-sort", "ascending"); + }); + + it.each([false, true])( + "preserves pins through loading and successful edits (storage blocked: %s)", + async (blocked) => { + if (blocked) vi.stubGlobal("localStorage", BLOCKED_STORAGE); + const user = userEvent.setup(); + const props = { preferenceScope: "test-user-proxy", accessToken: "test-token" }; + const view = render(); + await user.click(screen.getByRole("button", { name: "Pin alias tester" })); + view.rerender(); + view.rerender(); + expect(rowNames()[0]).toBe("tester"); + await user.click(screen.getByRole("button", { name: "Edit alias tester" })); + const editRow = screen.getByRole("row", { name: /Save Cancel$/ }); + fireEvent.change(within(editRow).getAllByRole("textbox")[0], { target: { value: "renamed" } }); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.mocked(setCallbacksCall).mockRejectedValueOnce(new Error("save failed")); + await user.click(screen.getByRole("button", { name: "Save" })); + await user.click(screen.getByRole("button", { name: "Cancel" })); + expect(screen.getByRole("button", { name: "Unpin alias tester" })).toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Edit alias tester" })); + const retryRow = screen.getByRole("row", { name: /Save Cancel$/ }); + fireEvent.change(within(retryRow).getAllByRole("textbox")[0], { target: { value: "renamed" } }); + const pendingSave = Promise.withResolvers(); + vi.mocked(setCallbacksCall).mockReturnValueOnce(pendingSave.promise); + await user.click(screen.getByRole("button", { name: "Save" })); + fireEvent.change(screen.getByLabelText("Place aliases with this prefix last"), { + target: { value: "subscription/" }, + }); + await act(async () => pendingSave.resolve()); + expect(screen.getByLabelText("Place aliases with this prefix last")).toHaveValue("subscription/"); + expect(rowNames()[0]).toBe("renamed"); + expect(screen.getByRole("button", { name: "Unpin alias renamed" })).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Delete alias renamed" })); + fireEvent.change(screen.getByPlaceholderText("e.g., gpt-4o"), { target: { value: "renamed" } }); + fireEvent.change(screen.getByPlaceholderText("e.g., gpt-4o-mini-openai"), { target: { value: "standard" } }); + await user.click(screen.getByRole("button", { name: "Add Alias" })); + expect(rowNames()[0]).toBe("alpha"); + expect(screen.getByRole("button", { name: "Pin alias renamed" })).toBeInTheDocument(); + }, + ); + + it("falls back from corrupt preferences and isolates preferences by scope", async () => { + const user = userEvent.setup(); + localStorage.setItem(aliasDisplayStorage("test-user-proxy").name, '{"sortBy":"unknown"}'); + const props = { accessToken: "test-token", initialModelGroupAlias: displayAliases }; + const view = render(); + expect(rowNames()[0]).toBe("alpha"); + await user.click(screen.getByRole("button", { name: "Pin alias tester" })); + view.rerender(); + expect(rowNames()[0]).toBe("alpha"); + view.rerender(); + expect(rowNames()[0]).toBe("tester"); + view.rerender(); + expect(screen.getByRole("button", { name: "Pin alias tester" })).toBeDisabled(); + }); + + it("keeps display changes usable when browser storage refuses writes", async () => { + const user = userEvent.setup(); + vi.stubGlobal("localStorage", BLOCKED_STORAGE); + render( + , + ); + await user.click(screen.getByRole("button", { name: "Pin alias tester" })); + expect(rowNames()[0]).toBe("tester"); + expect(screen.getByRole("status")).toHaveTextContent("could not be saved"); + await user.click(screen.getByRole("button", { name: "Unpin alias tester" })); + expect(rowNames()[0]).toBe("alpha"); + expect(setCallbacksCall).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/model_group_alias_settings.test.tsx b/ui/litellm-dashboard/src/components/model_group_alias_settings.test.tsx deleted file mode 100644 index fe59e6eb139..00000000000 --- a/ui/litellm-dashboard/src/components/model_group_alias_settings.test.tsx +++ /dev/null @@ -1,66 +0,0 @@ -/* @vitest-environment jsdom */ -import { render, screen } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { setCallbacksCall } from "@/components/networking"; -import ModelGroupAliasSettings from "./model_group_alias_settings"; - -vi.mock("@/components/networking", () => ({ - setCallbacksCall: vi.fn(), -})); - -const aliases = { - "fast-model": "claude-haiku", - "smart-model": { model: "claude-sonnet", hidden: true }, -}; - -describe("ModelGroupAliasSettings", () => { - beforeEach(() => { - vi.clearAllMocks(); - vi.mocked(setCallbacksCall).mockResolvedValue({}); - }); - - afterEach(() => { - vi.restoreAllMocks(); - }); - - it("locks config-owned aliases while keeping their names and targets visible", async () => { - render(); - - expect( - screen.getByText( - "These aliases are defined in config.yaml and are read-only here. Edit the config file to change them.", - ), - ).toBeInTheDocument(); - expect(screen.getByPlaceholderText("e.g., gpt-4o")).toBeDisabled(); - expect(screen.getByPlaceholderText("e.g., gpt-4o-mini-openai")).toBeDisabled(); - expect(screen.getByRole("button", { name: "Add Alias" })).toBeDisabled(); - expect(screen.getByRole("button", { name: "Edit alias fast-model" })).toBeDisabled(); - expect(screen.getByRole("button", { name: "Delete alias fast-model" })).toBeDisabled(); - expect(screen.getByRole("button", { name: "Edit alias smart-model" })).toBeDisabled(); - expect(screen.getByRole("button", { name: "Delete alias smart-model" })).toBeDisabled(); - - expect(await screen.findByText("fast-model")).toBeInTheDocument(); - expect(screen.getByText("claude-haiku")).toBeInTheDocument(); - expect(screen.getByText("smart-model")).toBeInTheDocument(); - expect(screen.getByText("claude-sonnet")).toBeInTheDocument(); - - await userEvent.setup().click(screen.getByRole("button", { name: "Delete alias fast-model" })); - expect(setCallbacksCall).not.toHaveBeenCalled(); - }); - - it("keeps aliases editable by default and saves the current string representation after delete", async () => { - const user = userEvent.setup(); - render(); - - expect(await screen.findByText("fast-model")).toBeInTheDocument(); - expect(screen.getByRole("button", { name: "Edit alias fast-model" })).toBeEnabled(); - expect(screen.getByRole("button", { name: "Delete alias fast-model" })).toBeEnabled(); - - await user.click(screen.getByRole("button", { name: "Delete alias fast-model" })); - - expect(setCallbacksCall).toHaveBeenCalledWith("test-token", { - router_settings: { model_group_alias: { "smart-model": "claude-sonnet" } }, - }); - }); -}); diff --git a/ui/litellm-dashboard/src/components/model_group_alias_settings.tsx b/ui/litellm-dashboard/src/components/model_group_alias_settings.tsx index d3cebc76e98..4a5e7fce9ba 100644 --- a/ui/litellm-dashboard/src/components/model_group_alias_settings.tsx +++ b/ui/litellm-dashboard/src/components/model_group_alias_settings.tsx @@ -1,9 +1,30 @@ -import React, { useState, useEffect } from "react"; +import React, { useState, useEffect, useMemo, useRef } from "react"; +import { + ColumnDef, + TableOptions, + Updater, + functionalUpdate, + getCoreRowModel, + useReactTable, +} from "@tanstack/react-table"; +import { ArrowDown, ArrowUp, Pin, PinOff } from "lucide-react"; import { PlusCircleIcon, PencilIcon, TrashIcon, ChevronDownIcon, ChevronRightIcon } from "@heroicons/react/outline"; import { setCallbacksCall } from "./networking"; import { Card, CardTitle } from "@/components/ui/card"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import { toast } from "@/lib/toast"; +import { readStorage, useStoredValue } from "@/lib/storage"; +import { DataTableSortHeader } from "@/components/shared/DataTable/DataTableSortHeader"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { + AliasDisplayPreferences, + AliasItem, + aliasDisplayStorage, + DEFAULT_ALIAS_DISPLAY, + movePinnedAlias, + orderAliases, +} from "./model_group_alias_ordering"; type ModelGroupAliasValue = string | { model: string; hidden?: boolean }; @@ -13,12 +34,30 @@ interface ModelGroupAliasSettingsProps { onAliasUpdate?: (updatedAlias: { [key: string]: string }) => void; managedByConfig?: boolean; ownershipLoading?: boolean; + preferenceScope: string | null; } -interface AliasItem { - id: string; - aliasName: string; - targetModelGroup: string; +const ALIAS_COLUMNS: ColumnDef[] = [ + { accessorKey: "aliasName", sortDescFirst: false }, + { accessorKey: "targetModelGroup", sortDescFirst: false }, +]; + +function useAliasDisplayPreferences(preferenceScope: string | null) { + const preferenceKey = useMemo(() => aliasDisplayStorage(preferenceScope ?? ""), [preferenceScope]); + const [storedPreferences, storePreferences] = useStoredValue(preferenceKey); + const [unsavedPreferences, setUnsavedPreferences] = useState(null); + const unsavedRef = useRef(null); + const preferences = preferenceScope === null ? DEFAULT_ALIAS_DISPLAY : unsavedPreferences ?? storedPreferences; + const updatePreferences = (updater: Updater) => { + if (preferenceScope === null) return; + const next = functionalUpdate(updater, unsavedRef.current ?? readStorage(preferenceKey)); + storePreferences(next); + const saved = JSON.stringify(readStorage(preferenceKey)) === JSON.stringify(next); + // rebind-ok: Async saves need the latest fallback when browser storage refuses writes. + unsavedRef.current = saved ? null : next; + setUnsavedPreferences(unsavedRef.current); + }; + return { preferences, updatePreferences, unsavedPreferences }; } const ModelGroupAliasSettings: React.FC = ({ @@ -27,12 +66,39 @@ const ModelGroupAliasSettings: React.FC = ({ onAliasUpdate, managedByConfig = false, ownershipLoading = false, + preferenceScope, }) => { + "use no memo"; + const [aliases, setAliases] = useState([]); const [newAlias, setNewAlias] = useState({ aliasName: "", targetModelGroup: "" }); const [editingAlias, setEditingAlias] = useState(null); const writesDisabled = managedByConfig || ownershipLoading; const [isExpanded, setIsExpanded] = useState(true); + const { preferences, updatePreferences, unsavedPreferences } = useAliasDisplayPreferences(preferenceScope); + const displayAliases = useMemo(() => orderAliases(aliases, preferences), [aliases, preferences]); + const visiblePins = preferences.pinned.filter((name) => aliases.some((alias) => alias.aliasName === name)); + const sorting = [{ id: preferences.sortBy, desc: preferences.descending }]; + const tableOptions: TableOptions = { + data: displayAliases, + columns: ALIAS_COLUMNS, + getCoreRowModel: getCoreRowModel(), + manualSorting: true, + enableSorting: preferenceScope !== null, + enableSortingRemoval: false, + enableMultiSort: false, + state: { sorting }, + onSortingChange: (updater) => { + const selected = functionalUpdate(updater, sorting)[0]; + updatePreferences({ + ...preferences, + sortBy: selected?.id === "targetModelGroup" ? "targetModelGroup" : "aliasName", + descending: selected?.desc ?? false, + }); + }, + }; + const table = useReactTable(tableOptions); + const sortDirection = preferences.descending ? "descending" : "ascending"; useEffect(() => { const aliasArray = Object.entries(initialModelGroupAlias).map(([aliasName, value], index) => ({ @@ -128,8 +194,15 @@ const ModelGroupAliasSettings: React.FC = ({ } const updatedAliases = aliases.map((alias) => (alias.id === editingAlias.id ? editingAlias : alias)); + const originalName = aliases.find((alias) => alias.id === editingAlias.id)?.aliasName; if (await saveAliasesToBackend(updatedAliases)) { + if (originalName !== editingAlias.aliasName) { + updatePreferences((current) => ({ + ...current, + pinned: current.pinned.map((name) => (name === originalName ? editingAlias.aliasName : name)), + })); + } setAliases(updatedAliases); setEditingAlias(null); toast.success("Alias updated successfully"); @@ -144,8 +217,13 @@ const ModelGroupAliasSettings: React.FC = ({ if (writesDisabled) return; const updatedAliases = aliases.filter((alias) => alias.id !== aliasId); + const deletedName = aliases.find((alias) => alias.id === aliasId)?.aliasName; if (await saveAliasesToBackend(updatedAliases)) { + updatePreferences((current) => ({ + ...current, + pinned: current.pinned.filter((name) => name !== deletedName), + })); setAliases(updatedAliases); toast.success("Alias deleted successfully"); } @@ -235,18 +313,57 @@ const ModelGroupAliasSettings: React.FC = ({

Manage Existing Aliases

+
+
+ + updatePreferences({ ...preferences, lastPrefix: event.target.value })} + /> +
+ +
+

+ {unsavedPreferences + ? "Display order could not be saved in this browser. Your changes apply until you leave this page." + : "Display preferences are saved for your account in this browser. Pinned aliases stay first in your chosen order."} +

- Alias Name - Target Model Group + + + + + + + Display order Actions - {aliases.map((alias) => ( + {displayAliases.map((alias) => ( {editingAlias && !writesDisabled && editingAlias.id === alias.id ? ( <> @@ -276,6 +393,7 @@ const ModelGroupAliasSettings: React.FC = ({ className="w-full px-2 py-1 border border-border rounded-md text-sm" /> +
+ {visiblePins.includes(alias.aliasName) && + ([-1, 1] as const).map((direction) => ( + + ))} +
+