From 7bd342d7eadba4e9ebbc393c44d36719bb29c7f6 Mon Sep 17 00:00:00 2001 From: paveln83 <349044532+paveln83@users.noreply.github.com> Date: Tue, 3 Nov 2026 01:56:08 -0800 Subject: [PATCH 001/122] fix(proxy): let a config license unlock enterprise settings and spell "environment" right in license errors (#49745) * fix(proxy): spell "environment" right in the allowed_ips and AWS KMS V2 license errors * fix(proxy): read general_settings.litellm_license before the allowed_ips and enforced_params gates * fix(proxy): read the config license before callbacks and gates, keeping a valid env license * fix(proxy): keep an invalid config license reported and sync license data with the license in use A config license that fails no longer overwrites the license string when the env license is not valid either, so /health/license still reports has_license. A failed boot-time env license is not re-checked. Both license writes go through one helper that also refreshes premium_user_data, so /health/license and the seat limits follow the license in use. * fix(proxy): keep the env license's boot result when a config license fails, without a second license check --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 38 +++- litellm/secret_managers/aws_secret_manager.py | 2 +- .../proxy/proxy_server/test_proxy_config.py | 212 ++++++++++++++++++ ...test_proxy_server_endpoints_and_startup.py | 6 +- .../test_aws_secret_manager.py | 14 ++ 5 files changed, 261 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 424e68ae3bc..6fcf789dc1b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1185,6 +1185,27 @@ server_root_path: Final = get_server_root_path() _license_check = LicenseCheck() premium_user: bool = _license_check.is_premium() premium_user_data: Optional["EnterpriseLicenseData"] = _license_check.airgapped_license_data + + +def _set_premium_user(is_premium: bool) -> None: + global premium_user, premium_user_data + premium_user = is_premium # rebind-ok: config load can switch the license in use + premium_user_data = _license_check.airgapped_license_data # rebind-ok: tracks the license in use + + +def _use_config_license(config_license: str, env_license_is_premium: bool) -> bool: + env_license: Final = _license_check.license_str + env_license_data: Final = _license_check.airgapped_license_data + _license_check.license_str = config_license + if _license_check.is_premium(): + return True + if not env_license_is_premium: + return False + _license_check.license_str = env_license + _license_check.airgapped_license_data = env_license_data + return True + + global_max_parallel_request_retries_env: Final[str | None] = os.getenv("LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES") proxy_state: Final = ProxyState() SENSITIVE_DATA_MASKER: Final = SensitiveDataMasker() @@ -6686,7 +6707,6 @@ class ProxyConfig: def _load_environment_variables(self, config: dict): ## ENVIRONMENT VARIABLES - global premium_user environment_variables: Final = config.get("environment_variables", None) if environment_variables: for key, value in environment_variables.items(): @@ -6726,7 +6746,7 @@ class ProxyConfig: # check if litellm_license in general_settings if "LITELLM_LICENSE" in environment_variables: _license_check.license_str = os.getenv("LITELLM_LICENSE", None) - premium_user = _license_check.is_premium() + _set_premium_user(_license_check.is_premium()) def _set_config_environment_variable(self, key: str, value: str) -> None: if key in os.environ and os.environ[key] != value: @@ -6778,7 +6798,6 @@ class ProxyConfig: prompt_injection_detection_obj, \ redis_usage_cache, \ store_model_in_db, \ - premium_user, \ open_telemetry_logger, \ health_check_details, \ proxy_batch_polling_interval, \ @@ -6791,6 +6810,12 @@ class ProxyConfig: self._load_environment_variables(config=config) + config_license: Final = _GENERAL_SETTINGS_VIEW.validate_python( + _GENERAL_SETTINGS_VIEW.validate_python(config).get("general_settings") or {} + ).get("litellm_license") + if isinstance(config_license, str): + _set_premium_user(_use_config_license(config_license, env_license_is_premium=premium_user)) + ## Coordination Redis (before cache init, so the explicit block wins) coordination_redis_cache: Final = self._init_coordination_redis(config=config) if coordination_redis_cache is not None: @@ -7367,7 +7392,7 @@ class ProxyConfig: allowed_ips: Final = general_settings.get("allowed_ips", None) if allowed_ips is not None and premium_user is False: raise ValueError( - "allowed_ips is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your envionment." + "allowed_ips is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your environment." ) ## BUDGET RESCHEDULER ## proxy_budget_rescheduler_min_time = general_settings.get( @@ -7423,11 +7448,6 @@ class ProxyConfig: if general_settings.get("enforced_params") is not None and premium_user is not True: raise ValueError("Trying to use `enforced_params`" + CommonProxyErrors.not_premium_user.value) - # check if litellm_license in general_settings - if "litellm_license" in general_settings: - _license_check.license_str = general_settings["litellm_license"] - premium_user = _license_check.is_premium() - router_params: Final[dict] = { "cache_responses": litellm.cache is not None, # cache if user passed in cache values } diff --git a/litellm/secret_managers/aws_secret_manager.py b/litellm/secret_managers/aws_secret_manager.py index 88642ae141e..c87edc4b1dd 100644 --- a/litellm/secret_managers/aws_secret_manager.py +++ b/litellm/secret_managers/aws_secret_manager.py @@ -70,7 +70,7 @@ class AWSKeyManagementService_V2: is_litellm_license_in_env = True if is_litellm_license_in_env is False: raise ValueError( - "AWSKeyManagementService V2 is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your envionment." + "AWSKeyManagementService V2 is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your environment." ) def load_aws_kms(self, use_aws_kms: bool | None): diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index 55226fc629b..712b4134c05 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -9,6 +9,7 @@ Pins covered: from __future__ import annotations import asyncio +import base64 import json import logging import os @@ -25,11 +26,15 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import padding, rsa +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, RSAPublicKey from prisma.errors import DataError from pydantic import JsonValue, TypeAdapter, ValidationError import litellm from litellm._logging import turn_on_json +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.proxy._types import CommonProxyErrors, ConfigGeneralSettings from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.config_resolvers.settings_store import SettingsStore @@ -2671,6 +2676,213 @@ async def test_load_config_logs_disabled_budget_reservation_once(tmp_path, monke assert [record.levelno for record in records] == ([logging.INFO] if setting == "true" else []) +@pytest.mark.asyncio +async def test_load_config_refuses_allowed_ips_without_a_license(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + config_file: Final = tmp_path / "allowed_ips.yaml" + config_file.write_text('model_list: []\nlitellm_settings: {}\ngeneral_settings:\n allowed_ips: ["127.0.0.1"]\n') + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + + expected_message: Final = ( + "allowed_ips is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your environment." + ) + with pytest.raises(ValueError, match=f"^{re.escape(expected_message)}$"): + await ProxyConfig().load_config(router=None, config_file_path=str(config_file)) + + +def _signed_test_license(private_key: RSAPrivateKey, max_users: int) -> str: + message: Final = json.dumps({"expiration_date": "9999-12-31", "user_id": "test", "max_users": max_users}).encode() + signature: Final = private_key.sign( + message, + padding.PSS(mgf=padding.MGF1(hashes.SHA256()), salt_length=padding.PSS.MAX_LENGTH), + hashes.SHA256(), + ) + return base64.b64encode(message + b"." + signature).decode() + + +def _write_config(path: Path, general_settings: dict[str, object], litellm_settings: dict[str, object]) -> str: + path.write_text( + json.dumps({"model_list": [], "litellm_settings": litellm_settings, "general_settings": general_settings}) + ) + return str(path) + + +def _boot_proxy( + monkeypatch: pytest.MonkeyPatch, + public_key: RSAPublicKey, + env_license: str | None, + license_server_urls: list[str], + server_accepted_license: str | None = None, +) -> None: + from litellm.proxy import proxy_server + + def _license_server(request: httpx.Request) -> httpx.Response: + license_server_urls.append(str(request.url)) + return httpx.Response( + 200, json={"verify": server_accepted_license is not None and server_accepted_license in str(request.url)} + ) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.setattr("litellm.proxy.proxy_server._license_check.public_key", public_key) + monkeypatch.setattr("litellm.proxy.proxy_server._license_check.airgapped_license_data", None) + monkeypatch.setattr( + "litellm.proxy.proxy_server._license_check.http_handler", + HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_license_server))), + ) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + if env_license is None: + monkeypatch.delenv("LITELLM_LICENSE", raising=False) + else: + monkeypatch.setenv("LITELLM_LICENSE", env_license) + monkeypatch.setattr("litellm.proxy.proxy_server._license_check.license_str", env_license) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", proxy_server._license_check.is_premium()) + monkeypatch.setattr( + "litellm.proxy.proxy_server.premium_user_data", proxy_server._license_check.airgapped_license_data + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "gated_config", + [ + {"general_settings": {"allowed_ips": ["127.0.0.1"]}}, + {"general_settings": {"enforced_params": ["user"]}}, + {"litellm_settings": {"callbacks": ["openai_moderations"]}}, + ], +) +async def test_load_config_lets_a_config_license_unlock_its_enterprise_gates( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, gated_config: dict[str, dict[str, list[str]]] +): + from litellm.proxy import proxy_server + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + license_key: Final = _signed_test_license(private_key, max_users=10) + config_file: Final = _write_config( + tmp_path / "licensed.yaml", + {"litellm_license": license_key, **gated_config.get("general_settings", {})}, + dict(gated_config.get("litellm_settings", {})), + ) + _boot_proxy(monkeypatch, private_key.public_key(), env_license=None, license_server_urls=[]) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + assert proxy_server.premium_user is True + + +@pytest.mark.asyncio +async def test_load_config_keeps_a_valid_env_license_when_the_config_license_is_invalid( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy import proxy_server + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + config_file: Final = _write_config( + tmp_path / "stale_license.yaml", {"litellm_license": "not-a-valid-license", "allowed_ips": ["127.0.0.1"]}, {} + ) + _boot_proxy( + monkeypatch, + private_key.public_key(), + env_license=_signed_test_license(private_key, max_users=5), + license_server_urls=[], + ) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + assert proxy_server.premium_user is True + assert proxy_server._license_check.user_seat_limit() == 5 + + +@pytest.mark.asyncio +async def test_load_config_prefers_the_config_license_when_both_licenses_are_valid( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_license_endpoint + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + config_file: Final = _write_config( + tmp_path / "licensed.yaml", {"litellm_license": _signed_test_license(private_key, max_users=100)}, {} + ) + _boot_proxy( + monkeypatch, + private_key.public_key(), + env_license=_signed_test_license(private_key, max_users=5), + license_server_urls=[], + ) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + assert proxy_server._license_check.user_seat_limit() == 100 + assert (await health_license_endpoint(user_api_key_dict=UserAPIKeyAuth()))["limits"]["max_users"] == 100 + + +@pytest.mark.asyncio +async def test_load_config_still_reports_an_invalid_config_license_as_configured( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_license_endpoint + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + config_file: Final = _write_config(tmp_path / "stale_license.yaml", {"litellm_license": "not-a-valid-license"}, {}) + _boot_proxy(monkeypatch, private_key.public_key(), env_license=None, license_server_urls=[]) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + license_status: Final = await health_license_endpoint(user_api_key_dict=UserAPIKeyAuth()) + assert (license_status["has_license"], license_status["license_type"]) == (True, "community") + + +@pytest.mark.asyncio +async def test_load_config_does_not_recheck_an_env_license_that_failed_at_boot( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy import proxy_server + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + config_file: Final = _write_config(tmp_path / "stale_license.yaml", {"litellm_license": "not-a-valid-license"}, {}) + license_server_urls: Final[list[str]] = [] + _boot_proxy(monkeypatch, private_key.public_key(), "also-not-a-valid-license", license_server_urls) + asked_at_boot: Final = len(license_server_urls) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + assert proxy_server.premium_user is False + assert [url for url in license_server_urls[asked_at_boot:] if "also-not-a-valid-license" in url] == [] + assert any("not-a-valid-license" in url for url in license_server_urls[asked_at_boot:]) + + +@pytest.mark.asyncio +async def test_load_config_keeps_the_boot_verdict_of_a_server_checked_env_license( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy import proxy_server + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + config_file: Final = _write_config( + tmp_path / "stale_license.yaml", {"litellm_license": "not-a-valid-license", "allowed_ips": ["127.0.0.1"]}, {} + ) + license_server_urls: Final[list[str]] = [] + _boot_proxy( + monkeypatch, + private_key.public_key(), + "server-checked-license", + license_server_urls, + server_accepted_license="server-checked-license", + ) + asked_at_boot: Final = len(license_server_urls) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + assert proxy_server.premium_user is True + assert proxy_server._license_check.license_str == "server-checked-license" + assert [url for url in license_server_urls[asked_at_boot:] if "server-checked-license" in url] == [] + + @pytest.mark.asyncio @pytest.mark.parametrize( "setting", ["vector_store_deny_by_default", "search_tool_deny_by_default", "claude_code_marketplace_auth_required"] diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 82db8820447..9914247bc47 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -4117,7 +4117,11 @@ async def test_load_environment_variables_litellm_license_and_edge_cases(): mock_license_check = MagicMock() mock_license_check.is_premium.return_value = True - with patch("litellm.proxy.proxy_server._license_check", mock_license_check): + with ( + patch("litellm.proxy.proxy_server._license_check", mock_license_check), + patch("litellm.proxy.proxy_server.premium_user", False), + patch("litellm.proxy.proxy_server.premium_user_data", None), + ): with patch.dict(os.environ, {}, clear=False): # Call the method under test proxy_config._load_environment_variables(test_config_with_license) diff --git a/tests/unit/secret_managers/test_aws_secret_manager.py b/tests/unit/secret_managers/test_aws_secret_manager.py index d9922b43ec8..e16da7a0844 100644 --- a/tests/unit/secret_managers/test_aws_secret_manager.py +++ b/tests/unit/secret_managers/test_aws_secret_manager.py @@ -1,4 +1,6 @@ import base64 +import re +from typing import Final import pytest from botocore.stub import Stubber @@ -34,3 +36,15 @@ def test_decrypt_value_turns_only_boolean_literals_into_bools(monkeypatch, plain assert decrypted == expected assert type(decrypted) is type(expected) + + +def test_kms_v2_without_a_license_tells_the_user_to_set_one(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AWS_REGION_NAME", "us-west-2") + monkeypatch.delenv("LITELLM_LICENSE", raising=False) + monkeypatch.delenv("LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE", raising=False) + + expected_message: Final = ( + "AWSKeyManagementService V2 is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your environment." + ) + with pytest.raises(ValueError, match=f"^{re.escape(expected_message)}$"): + AWSKeyManagementService_V2() From ff182c194e0b5f782e4a7d3405361fed11466416 Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Tue, 3 Nov 2026 02:02:54 -0800 Subject: [PATCH 002/122] fix(logging): keep the caller's call_type when chat completions are bridged to the Responses API (#49710) * fix(logging): keep the caller's call_type when chat completions are bridged to the Responses API * fix(logging): log bridged chat completions with the chat response they returned * fix(logging): log bridged text completions and /v1/messages as served, with the request's options Bridged rows were translated without the request's options, so structured output and replay fields could differ from what the client got. Text completions and non-streamed /v1/messages through the chat bridge also kept a Responses API object or the encoded id. * fix(logging): redact reasoning summaries on bridged rows and log bridged text completions as text completions Redaction now drops reasoning_items on chat messages and deltas, so a bridged row keeps no reasoning summary when message logging is off. A non-streamed bridged text completion logs the text completion the client got, priced like the chat response it came from. The bridge handler's input TypedDict fields are ReadOnly. * fix(logging): redact bridged text completions for opted-out callbacks and pin the sync logged shape and usage ack * test(integration): look up bridged chat spend rows by the id the client got Bridged chat spend rows are keyed by the client-facing id and logged with the caller's call type, so the helpers that decoded the old resp_ key or pinned call_type responses now assert the new contract --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../handler.py | 22 +- .../transformation.py | 4 - litellm/litellm_core_utils/litellm_logging.py | 75 +++- litellm/litellm_core_utils/redact_messages.py | 52 ++- .../_mantle_gpt_prompt_cache_support.py | 11 +- .../test_bedrock_mantle_claude_chat_wire.py | 2 +- .../providers/test_openai_chat_wire.py | 17 +- .../spend/test_responses_bridge_spend_once.py | 4 +- ...responses_transformation_transformation.py | 425 +++++++++++++++++- .../test_redact_messages.py | 51 +++ 10 files changed, 587 insertions(+), 76 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 76d17ea9693..f6879c313de 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -11,7 +11,7 @@ from typing import ( cast, # noqa: TID251 # optional_params reaches the bridge as an untyped dict ) -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm.types.llms.openai import ResponsesAPIResponse @@ -20,15 +20,15 @@ if TYPE_CHECKING: class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict): - model: str - messages: list - optional_params: dict - litellm_params: dict - headers: dict - model_response: "ModelResponse" - logging_obj: "LiteLLMLoggingObj" - custom_llm_provider: str - encoding: object + model: ReadOnly[str] + messages: ReadOnly[list] + optional_params: ReadOnly[dict[str, object]] + litellm_params: ReadOnly[dict[str, object]] + headers: ReadOnly[dict] + model_response: ReadOnly["ModelResponse"] + logging_obj: ReadOnly["LiteLLMLoggingObj"] + custom_llm_provider: ReadOnly[str] + encoding: ReadOnly[object] def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str: @@ -184,6 +184,7 @@ class ResponsesToCompletionBridgeHandler: custom_llm_provider: Final = validated_kwargs["custom_llm_provider"] if kwargs.get("stream") is True and "stream" not in optional_params: optional_params = {**optional_params, "stream": True} + logging_obj.record_bridged_chat_request(optional_params, litellm_params) request_data: Final = self.transformation_handler.transform_request( model=model, @@ -285,6 +286,7 @@ class ResponsesToCompletionBridgeHandler: custom_llm_provider: Final = validated_kwargs["custom_llm_provider"] if kwargs.get("stream") is True and "stream" not in optional_params: optional_params = {**optional_params, "stream": True} + logging_obj.record_bridged_chat_request(optional_params, litellm_params) try: request_data: Final = self.transformation_handler.transform_request( diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index c03419436d6..8dd4d87ae13 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1235,10 +1235,6 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): api_model: Final = model - from litellm.types.utils import CallTypes - - setattr(litellm_logging_obj, "call_type", CallTypes.responses.value) - sanitized_litellm_params: Final = self._build_sanitized_litellm_params(litellm_params) request_data: Final = { diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 22c5f051d5c..97c019a2180 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -19,7 +19,7 @@ from datetime import datetime as dt_object from enum import Enum from functools import lru_cache from types import MappingProxyType, TracebackType -from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, cast +from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, TypeVar, cast from httpx import Response, Timeout from openai import AsyncStream as OpenAIAsyncStream @@ -99,6 +99,7 @@ from litellm.litellm_core_utils.hidden_params import ( get_hidden_params, served_from_cache, set_hidden_param, + set_hidden_params, ) from litellm.litellm_core_utils.internal_call_metadata import ( MODEL_ACCESS_GROUP_METADATA_KEY, @@ -124,6 +125,7 @@ from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import ( diagnose_zero_cost, zero_cost_warning, ) +from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import LiteLLMResponseObjectHandler from litellm.litellm_core_utils.logging_utils import ( truncate_base64_in_messages, truncate_base64_in_messages_async, @@ -144,7 +146,7 @@ from litellm.litellm_core_utils.served_output_texts import ( from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.llms.base_llm.search.transformation import SearchResponse -from litellm.responses.utils import ResponseAPILoggingUtils +from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils from litellm.types.agents import LiteLLMSendMessageResponse from litellm.types.containers.main import ContainerObject from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse @@ -302,6 +304,11 @@ class _EnterpriseAlertingLoggers(NamedTuple): smtp_email: type[CustomLogger] +class BridgedChatRequest(NamedTuple): + optional_params: Mapping[str, object] + litellm_params: Mapping[str, object] + + @functools.cache def _enterprise_alerting_loggers() -> _EnterpriseAlertingLoggers: try: @@ -582,6 +589,7 @@ _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object]) _SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object]) _GOOGLE_USAGE_FIELDS: Final = frozenset(UsageMetadata.__annotations__) _REDACTED_RESULT: Final = TypeAdapter(object) +_LoggedResponse: Final = TypeVar("_LoggedResponse") _GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information" @@ -875,6 +883,7 @@ class Logging(LiteLLMLoggingBaseClass): # once that response is priced, the same way a non-streamed response is logged self.client_facing_stream_model: str | None = None self.client_facing_response_id: str | None = None + self.bridged_chat_request: BridgedChatRequest | None = None self.zero_cost_warned: bool = False self.llm_caching_handler: LLMCachingHandler | None = None @@ -3235,6 +3244,40 @@ class Logging(LiteLLMLoggingBaseClass): return response return response.model_copy(update={"model": model}) + def _as_served_chat_response(self, response: _LoggedResponse) -> _LoggedResponse | ModelResponse: + request: Final = self.bridged_chat_request + if request is None or not isinstance(response, ResponsesAPIResponse): + return response + return self._bridged_chat_response(response, request) + + def _as_served_response( + self, response: _LoggedResponse + ) -> _LoggedResponse | ModelResponse | TextCompletionResponse: + request: Final = self.bridged_chat_request + if request is None or not isinstance(response, ResponsesAPIResponse): + return response + served: Final = self._bridged_chat_response(response, request) + if self.call_type not in (CallTypes.text_completion.value, CallTypes.atext_completion.value): # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped + return served + return self._as_text_completion_priced_as_chat(served) + + def _as_text_completion_priced_as_chat(self, served: ModelResponse) -> TextCompletionResponse: + text_completion: Final = LiteLLMResponseObjectHandler.convert_chat_to_text_completion( + served, TextCompletionResponse() + ) + set_hidden_params(text_completion, dict(get_hidden_params(served) or {})) + return text_completion + + def _bridged_chat_response(self, response: ResponsesAPIResponse, request: BridgedChatRequest) -> ModelResponse: + translated: Final = self._translate_responses_api_response_to_model_response(response, request) + served: Final = translated.model_copy( + update={ + "id": ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(response.id) + } + ) + set_hidden_params(served, {**(get_hidden_params(translated) or {}), **(get_hidden_params(response) or {})}) + return served + def _with_continued_response_id( self, response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse, @@ -3316,6 +3359,8 @@ class Logging(LiteLLMLoggingBaseClass): ) result = self.normalize_endpoint_logging_result(result) + if self.stream is not True: + result = self._as_served_response(result) # rebind-ok: bridged chat calls log the served response logging_result: Final = self.normalize_logging_result(result=result) if cache_hit is None and _served_from_cache_at_no_charge(logging_result): @@ -3670,7 +3715,9 @@ class Logging(LiteLLMLoggingBaseClass): ) self._add_continued_partial_spend(complete_streaming_response) self._merge_hidden_params_from_response_into_metadata(complete_streaming_response) - logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response) + logged_streaming_response: Final = self._client_facing_streaming_response( + self._as_served_chat_response(complete_streaming_response) + ) self.model_call_details["complete_streaming_response"] = logged_streaming_response self.capture_usage_before_logging(logged_streaming_response, start_time, end_time) ## STANDARDIZED LOGGING PAYLOAD @@ -4268,7 +4315,9 @@ class Logging(LiteLLMLoggingBaseClass): await self._prepare_baseline_cache_estimate(complete_streaming_response) - logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response) + logged_streaming_response: Final = self._client_facing_streaming_response( + self._as_served_chat_response(complete_streaming_response) + ) self.model_call_details["async_complete_streaming_response"] = logged_streaming_response ## STANDARDIZED LOGGING PAYLOAD @@ -5227,6 +5276,13 @@ class Logging(LiteLLMLoggingBaseClass): row is built from this response rather than re-parsed from the served body.""" self.bridged_model_response = response + def record_bridged_chat_request( + self, optional_params: Mapping[str, object], litellm_params: Mapping[str, object] + ) -> None: + self.bridged_chat_request = BridgedChatRequest( + MappingProxyType(dict(optional_params)), MappingProxyType(dict(litellm_params)) + ) + def record_bridged_responses_api_response(self, response: ResponsesAPIResponse) -> None: self.record_bridged_model_response(self._translate_responses_api_response_to_model_response(response)) @@ -5298,6 +5354,9 @@ class Logging(LiteLLMLoggingBaseClass): ): result = result.response if isinstance(result, ResponsesAPIResponse): + bridged_request: Final = self.bridged_chat_request + if bridged_request is not None: + return self._bridged_chat_response(result, bridged_request) return self._translate_responses_api_response_to_model_response(result) raw_body: Final[object] = result @@ -5349,7 +5408,9 @@ class Logging(LiteLLMLoggingBaseClass): set_response_cost_in_hidden_params(result, reported_cost) return result - def _translate_responses_api_response_to_model_response(self, result: ResponsesAPIResponse) -> ModelResponse: + def _translate_responses_api_response_to_model_response( + self, result: ResponsesAPIResponse, request: BridgedChatRequest | None = None + ) -> ModelResponse: """ Convert a Responses API response into a ModelResponse for spend_logs. @@ -5372,8 +5433,8 @@ class Logging(LiteLLMLoggingBaseClass): logging_obj=self, request_data={}, messages=[], - optional_params={}, - litellm_params={}, + optional_params={} if request is None else dict(request.optional_params), + litellm_params={} if request is None else dict(request.litellm_params), encoding=litellm.encoding, ) translated.id = served_id or translated.id diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 078ec18e2db..9da0e35bdda 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -10,7 +10,7 @@ import asyncio import copy import inspect -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import replace from typing import TYPE_CHECKING, Any, Final @@ -107,7 +107,13 @@ def redacted_response_obj_for_custom_logger(response_obj: object, custom_logger: def _is_redactable_response(response_obj: object) -> bool: return isinstance( - response_obj, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse) + response_obj, + ( + litellm.ModelResponse, + litellm.TextCompletionResponse, + litellm.ResponsesAPIResponse, + litellm.EmbeddingResponse, + ), ) or (isinstance(response_obj, dict) and ("choices" in response_obj or "output" in response_obj)) @@ -208,6 +214,7 @@ def _redact_choice_content(choice): choice.message.reasoning_content = REDACTED_BY_LITELLM if hasattr(choice.message, "thinking_blocks"): choice.message.thinking_blocks = None + _drop_field_when_present(choice.message, "reasoning_items") _redact_provider_specific_fields(choice.message) _redact_tool_calls(getattr(choice.message, "tool_calls", None)) _redact_function_call(getattr(choice.message, "function_call", None)) @@ -218,12 +225,14 @@ def _redact_choice_content(choice): choice.delta.reasoning_content = REDACTED_BY_LITELLM if hasattr(choice.delta, "thinking_blocks"): choice.delta.thinking_blocks = None + _drop_field_when_present(choice.delta, "reasoning_items") _redact_provider_specific_fields(choice.delta) _redact_tool_calls(getattr(choice.delta, "tool_calls", None)) _redact_function_call(getattr(choice.delta, "function_call", None)) elif isinstance(choice, litellm.utils.TextChoices): _redact_field_when_present(choice, "text") _redact_field_when_present(choice, "reasoning_content") + _drop_field_when_present(choice, "logprobs") def _as_mapping(value: object) -> Mapping[str, object] | None: @@ -284,6 +293,11 @@ def _redact_field_when_present(item: object, field: str) -> None: setattr(item, field, REDACTED_BY_LITELLM) +def _drop_field_when_present(item: object, field: str) -> None: + if getattr(item, field, None) is not None: + setattr(item, field, None) + + def _redact_responses_api_output(output_items): """Helper to redact ResponsesAPIResponse output items.""" for output_item in output_items: @@ -440,32 +454,22 @@ def _redact_tool_calls_dict(message: Mapping[str, object]) -> None: function_call["arguments"] = REDACTED_BY_LITELLM -def _redact_model_response_dict_choices(choices, redacted_str: str): +def _redact_model_response_dict_choices(choices: Iterable[object], redacted_str: str) -> None: for choice in choices: if isinstance(choice, dict): _redact_provider_specific_fields_dict(choice) if choice.get("token_ids") is not None: choice["token_ids"] = None - if "message" in choice and isinstance(choice["message"], dict): - if choice["message"].get("content") is not None: - choice["message"]["content"] = redacted_str - if choice["message"].get("reasoning_content") is not None: - choice["message"]["reasoning_content"] = redacted_str - if "thinking_blocks" in choice["message"]: - choice["message"]["thinking_blocks"] = None - if "audio" in choice["message"]: - choice["message"]["audio"] = None - _redact_tool_calls_dict(choice["message"]) - elif "delta" in choice and isinstance(choice["delta"], dict): - if choice["delta"].get("content") is not None: - choice["delta"]["content"] = redacted_str - if choice["delta"].get("reasoning_content") is not None: - choice["delta"]["reasoning_content"] = redacted_str - if "thinking_blocks" in choice["delta"]: - choice["delta"]["thinking_blocks"] = None - if "audio" in choice["delta"]: - choice["delta"]["audio"] = None - _redact_tool_calls_dict(choice["delta"]) + if ( + body := next((choice[key] for key in ("message", "delta") if isinstance(choice.get(key), dict)), None) + ) is not None: + for field in ("content", "reasoning_content"): + if body.get(field) is not None: + body[field] = redacted_str + for field in ("thinking_blocks", "reasoning_items", "audio"): + if field in body: + body[field] = None + _redact_tool_calls_dict(body) elif choice.get("text") is not None: choice["text"] = redacted_str else: @@ -588,7 +592,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons return {"text": REDACTED_BY_LITELLM} _result: Final = copy.deepcopy(result) - if isinstance(_result, litellm.ModelResponse): + if isinstance(_result, (litellm.ModelResponse, litellm.TextCompletionResponse)): if hasattr(_result, "choices") and _result.choices is not None: for choice in _result.choices: _redact_choice_content(choice) diff --git a/tests/integration/providers/_mantle_gpt_prompt_cache_support.py b/tests/integration/providers/_mantle_gpt_prompt_cache_support.py index 8f497fea74f..96261bab8b2 100644 --- a/tests/integration/providers/_mantle_gpt_prompt_cache_support.py +++ b/tests/integration/providers/_mantle_gpt_prompt_cache_support.py @@ -475,10 +475,6 @@ def row_key(request_id: str) -> str: return upstream_id_of(request_id.split("_cache_hit", 1)[0]) -def caller_sees_upstream_id(endpoint: Endpoint, *, stream: bool) -> bool: - return (endpoint, stream) != ("messages", True) - - def usage_of(payload: Mapping[str, JsonValue]) -> dict[str, JsonValue]: usage: Final = payload.get("usage") return JSON.validate_python(usage) if isinstance(usage, dict) else {} @@ -733,10 +729,9 @@ def marked_cell(gateway: Gateway, endpoint: Endpoint, *, stream: bool) -> None: assert_answered(outcome, marker) assert_wire(received, expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True), streaming=stream) assert_usage(endpoint, outcome.usage) - row: Final = success_row(name, marker) + row: Final = success_row(name, marker, outcome.response_id) assert_priced_row(row, GPT) - if caller_sees_upstream_id(endpoint, stream=stream): - assert row_key(str(row["request_id"])) == outcome.response_id, (row, outcome.response_id) + assert row_key(str(row["request_id"])) == outcome.response_id, (row, outcome.response_id) if not stream: assert math.isclose(float(outcome.headers["x-litellm-response-cost"]), expected_spend(GPT), rel_tol=1e-9), ( outcome.headers @@ -901,7 +896,7 @@ def assert_marked_sdk_cell( only_received(wire), expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True), streaming=stream ) assert_usage(endpoint, outcome.usage) - assert_priced_row(success_row(name, marker), GPT) + assert_priced_row(success_row(name, marker, outcome.response_id), GPT) HOSTILE_OPTIONS: Final[dict[str, JsonValue]] = { diff --git a/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py b/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py index 32b6c4f1690..3652e49fe8a 100644 --- a/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py @@ -1211,6 +1211,6 @@ def test_non_claude_mantle_id_with_a_bare_bedrock_twin_keeps_its_route_and_spend cost: Final = _mantle_cost(gateway, _LUNA) assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(cost), response.text _assert_success_row( - _spend_row(_BY_CALL_ID, response.headers["x-litellm-call-id"]), model, _LUNA, "responses", cost + _spend_row(_BY_CALL_ID, response.headers["x-litellm-call-id"]), model, _LUNA, "acompletion", cost ) assert [(request.method, request.target) for request in wire.drain()] == [("POST", _RESPONSES_PATH)] diff --git a/tests/integration/providers/test_openai_chat_wire.py b/tests/integration/providers/test_openai_chat_wire.py index 69295485380..5ba631ac776 100644 --- a/tests/integration/providers/test_openai_chat_wire.py +++ b/tests/integration/providers/test_openai_chat_wire.py @@ -1,4 +1,3 @@ -import base64 import json import uuid from collections.abc import Callable @@ -547,14 +546,8 @@ def test_azure_gpt_6_bridged_no_cache_function_requests_each_reach_provider_and_ lambda found: len(found) == 2, seconds=70, ) - by_response_id: Final = { - (decoded := base64.b64decode(string_value(row["request_id"]).removeprefix("resp_")).decode()) - .rsplit("response_id:", 1)[1]: (decoded, row) - for row in rows - } - for response_id in response_ids: - decoded, row = by_response_id[response_id] - assert decoded.startswith("litellm:custom_llm_provider:azure;model_id:"), rows + assert sorted(string_value(row["request_id"]) for row in rows) == sorted(response_ids), rows + for row in rows: assert ( string_value(row["status"]), string_value(row["cache_hit"]), @@ -601,11 +594,7 @@ def test_azure_gpt_6_bridged_function_requests_without_cache_field_still_hit_cac seconds=70, ) priced, cached = rows - priced_request: Final = base64.b64decode( - string_value(priced["request_id"]).removeprefix("resp_") - ).decode() - assert priced_request.startswith("litellm:custom_llm_provider:azure;model_id:"), rows - assert priced_request.endswith(";response_id:resp_cached"), rows + assert string_value(priced["request_id"]) == "resp_cached", rows assert ( priced["status"], priced["cache_hit"], diff --git a/tests/integration/spend/test_responses_bridge_spend_once.py b/tests/integration/spend/test_responses_bridge_spend_once.py index 9eddd832ca0..cc2a0af58a7 100644 --- a/tests/integration/spend/test_responses_bridge_spend_once.py +++ b/tests/integration/spend/test_responses_bridge_spend_once.py @@ -1,5 +1,4 @@ import json -from base64 import b64decode from collections.abc import Callable from hashlib import sha256 from typing import Final @@ -100,8 +99,7 @@ def test_bridged_chat_completion_above_log_offload_threshold_logs_and_charges_on call_rows: Final = [row for row in rows if row["litellm_call_id"] == call_id] assert len(call_rows) == 1, rows assert float(str(call_rows[0]["spend"])) == pytest.approx(EXPECTED_SPEND), rows - logged_id: Final = b64decode(str(call_rows[0]["request_id"]).removeprefix("resp_")).decode() - assert logged_id.startswith("litellm:") and f";response_id:{upstream_id}" in logged_id, rows + assert call_rows[0]["request_id"] == upstream_id, rows key_spend: Final = eventually( lambda: read_rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', (digest,)), diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 9313e3d7039..2d02468fd8c 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -1,9 +1,13 @@ +import asyncio import copy import datetime import json import os +import queue import unittest -from collections.abc import Mapping, Sequence +from collections.abc import AsyncIterator, Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType from typing import TYPE_CHECKING, Final, List, Literal, Optional, Tuple, cast, get_args from unittest.mock import ANY, MagicMock, Mock, patch @@ -24,15 +28,15 @@ import litellm from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, ) -from litellm.types.llms.openai import AllMessageValues, REASONING_EFFORT +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.types.llms.openai import AllMessageValues, REASONING_EFFORT, ResponsesAPIResponse +from litellm.types.utils import ModelResponse, StandardLoggingPayload, TextCompletionResponse if TYPE_CHECKING: from openai.types.responses import ResponseOutputItem from openai.types.responses.response_reasoning_item import ResponseReasoningItem - from litellm.types.llms.openai import ResponsesAPIResponse - from litellm.types.utils import ModelResponse - def test_completion_over_responses_maps_anthropic_search_tool_to_openai_tool_search() -> None: response_body: Final = { @@ -191,6 +195,417 @@ def test_auto_summary_sends_the_level_the_target_accepts( assert request_body["reasoning"] == {"effort": "low", "summary": expected_summary} +@dataclass(frozen=True, slots=True) +class _LoggedSuccess: + call_type: str + payload: StandardLoggingPayload + response_obj: object + complete_streaming_response: object + + +class _LoggedCallTypes(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.seen_by_caller: Final = MappingProxyType( + { + "completion": queue.SimpleQueue[_LoggedSuccess](), + "acompletion": queue.SimpleQueue[_LoggedSuccess](), + } + ) + + @staticmethod + def _logged(kwargs: Mapping[str, object], response_obj: object, streaming_key: str) -> _LoggedSuccess: + return _LoggedSuccess( + call_type=str(kwargs["call_type"]), + payload=cast(StandardLoggingPayload, kwargs["standard_logging_object"]), + response_obj=response_obj, + complete_streaming_response=kwargs.get(streaming_key), + ) + + def log_success_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + self.seen_by_caller["completion"].put(self._logged(kwargs, response_obj, "complete_streaming_response")) + + async def async_log_success_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + self.seen_by_caller["acompletion"].put( + self._logged(kwargs, response_obj, "async_complete_streaming_response") + ) + + +_GET_WEATHER_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, +} + +_BRIDGED_PROVIDERS: Final = pytest.mark.parametrize( + ("model", "url", "provider_kwargs"), + [ + ("openai/gpt-5.3-codex", "https://api.openai.com/v1/responses", {}), + ( + "azure/gpt-6-sol", + "https://example.invalid/openai/responses", + {"api_base": "https://example.invalid", "api_version": "2025-04-01-preview"}, + ), + ("bedrock_mantle/openai.gpt-5.6-terra", _MANTLE_RESPONSES_URL, {"aws_region_name": "us-east-1"}), + ], + ids=["openai-responses-only", "azure-tools", "bedrock-mantle-tools"], +) + + +_PONG_MESSAGE: Final[Mapping[str, object]] = { + "type": "message", + "id": "msg_logged_call_type", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "pong", "annotations": []}], +} + + +def _searched_responses_body(model: str) -> Mapping[str, object]: + return { + "id": "resp_logged_call_type", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": model.rsplit("/", 1)[-1], + "output": [ + { + "type": "web_search_call", + "id": "ws_logged_call_type", + "status": "completed", + "action": {"type": "search", "query": "weather"}, + }, + _PONG_MESSAGE, + ], + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + } + + +def _responses_sse(response_body: Mapping[str, object]) -> str: + events: Final = ( + {"type": "response.created", "response": {**response_body, "status": "in_progress", "output": [], "usage": None}}, + { + "type": "response.output_text.delta", + "item_id": "msg_logged_call_type", + "output_index": 1, + "content_index": 0, + "delta": "pong", + }, + {"type": "response.completed", "response": response_body}, + ) + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) + + +def _bridged_request(model: str, provider_kwargs: Mapping[str, object], stream: bool) -> Mapping[str, object]: + return { + "model": model, + "messages": [{"role": "user", "content": "ping"}], + "tools": [_GET_WEATHER_TOOL], + "api_key": "test-key", + "cache": {"no-cache": True}, + "stream": stream, + **provider_kwargs, + } + + +async def _served_chunk_ids(caller: str, request: Mapping[str, object]) -> frozenset[str]: + if caller == "acompletion": + async_stream: Final = await litellm.acompletion(**request) + assert isinstance(async_stream, CustomStreamWrapper) + return frozenset([chunk.id async for chunk in async_stream]) + sync_stream: Final = litellm.completion(**request) + assert isinstance(sync_stream, CustomStreamWrapper) + return frozenset(chunk.id for chunk in sync_stream) + + +@pytest.mark.parametrize("caller", ["completion", "acompletion"]) +@_BRIDGED_PROVIDERS +@pytest.mark.asyncio +async def test_bridged_chat_call_is_logged_with_the_callers_call_type( + monkeypatch: pytest.MonkeyPatch, caller: str, model: str, url: str, provider_kwargs: Mapping[str, object] +) -> None: + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = _searched_responses_body(model) + request: Final = _bridged_request(model, provider_kwargs, stream=False) + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith=url).mock(return_value=httpx.Response(200, json=response_body)) + if caller == "acompletion": + await litellm.acompletion(**request) + else: + litellm.completion(**request) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10) + + assert (logged.call_type, logged.payload["call_type"]) == (caller, caller) + assert isinstance(logged.response_obj, ModelResponse) + assert logged.response_obj.id == response_body["id"] + assert logged.response_obj.choices[0].message.content == "pong" + assert logged.payload["response"]["choices"][0]["message"]["content"] == "pong" + assert logged.payload["response_cost"] == litellm.completion_cost( + completion_response=ResponsesAPIResponse.model_validate(response_body), model=model + ) + + +@pytest.mark.parametrize("caller", ["completion", "acompletion"]) +@_BRIDGED_PROVIDERS +@pytest.mark.asyncio +async def test_bridged_chat_stream_is_logged_as_the_chat_completion_it_streamed( + monkeypatch: pytest.MonkeyPatch, caller: str, model: str, url: str, provider_kwargs: Mapping[str, object] +) -> None: + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = _searched_responses_body(model) + request: Final = _bridged_request(model, provider_kwargs, stream=True) + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith=url).mock( + return_value=httpx.Response( + 200, text=_responses_sse(response_body), headers={"content-type": "text/event-stream"} + ) + ) + served_ids: Final = await _served_chunk_ids(caller, request) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10) + + assert isinstance(logged.complete_streaming_response, ModelResponse) + assert logged.response_obj is logged.complete_streaming_response + assert served_ids == {logged.complete_streaming_response.id} + assert logged.complete_streaming_response.choices[0].message.content == "pong" + assert logged.payload["response"]["choices"][0]["message"]["content"] == "pong" + assert logged.payload["response_cost"] == litellm.completion_cost( + completion_response=ResponsesAPIResponse.model_validate(response_body), model=model + ) + + +def _json_message(item_id: str, document: Mapping[str, object]) -> Mapping[str, object]: + return { + "type": "message", + "id": item_id, + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": json.dumps(document), "annotations": []}], + } + + +@pytest.mark.parametrize("caller", ["completion", "acompletion"]) +@pytest.mark.asyncio +async def test_bridged_structured_output_call_is_logged_with_the_json_document_the_client_got( + monkeypatch: pytest.MonkeyPatch, caller: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = { + **_searched_responses_body("openai/gpt-5.3-codex"), + "output": [_json_message("msg_first", {"answer": "pong"}), _json_message("msg_second", {"answer": "again"})], + } + request: Final = { + **_bridged_request("openai/gpt-5.3-codex", {}, stream=False), + "response_format": {"type": "json_object"}, + } + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith="https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=response_body) + ) + served: Final = await litellm.acompletion(**request) if caller == "acompletion" else litellm.completion(**request) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10) + + assert isinstance(served, ModelResponse) + assert isinstance(logged.response_obj, ModelResponse) + served_message: Final = served.choices[0].message + logged_message: Final = logged.response_obj.choices[0].message + assert json.loads(served_message.content) == {"answer": "pong"} + assert logged_message.content == served_message.content + assert logged_message.provider_specific_fields == served_message.provider_specific_fields + assert logged.payload["response"]["choices"][0]["message"]["content"] == served_message.content + + +async def _logged_text_completion( + monkeypatch: pytest.MonkeyPatch, caller: str +) -> tuple[TextCompletionResponse, _LoggedSuccess, Mapping[str, object]]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = _searched_responses_body("openai/gpt-5.3-codex") + request: Final = {"model": "openai/gpt-5.3-codex", "prompt": "ping", "api_key": "test-key"} + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith="https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=response_body) + ) + served: Final = ( + await litellm.atext_completion(**request) if caller == "acompletion" else litellm.text_completion(**request) + ) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10) + assert isinstance(served, TextCompletionResponse) + return served, logged, response_body + + +@pytest.mark.parametrize(("caller", "call_type"), [("completion", "text_completion"), ("acompletion", "atext_completion")]) +@pytest.mark.asyncio +async def test_bridged_text_completion_is_logged_and_priced_under_the_callers_call_type( + monkeypatch: pytest.MonkeyPatch, caller: str, call_type: str +) -> None: + served, logged, response_body = await _logged_text_completion(monkeypatch, caller) + + assert (logged.call_type, logged.payload["call_type"]) == (call_type, call_type) + assert getattr(logged.response_obj, "id") == served.id == response_body["id"] + assert logged.payload["response_cost"] == litellm.completion_cost( + completion_response=ResponsesAPIResponse.model_validate(response_body), model="openai/gpt-5.3-codex" + ) + + +@pytest.mark.parametrize("caller", ["completion", "acompletion"]) +@pytest.mark.asyncio +async def test_bridged_text_completion_is_logged_as_the_text_completion_it_served( + monkeypatch: pytest.MonkeyPatch, caller: str +) -> None: + served, logged, _ = await _logged_text_completion(monkeypatch, caller) + + assert isinstance(logged.response_obj, TextCompletionResponse) + assert logged.response_obj.choices[0].text == served.choices[0].text == "pong" + assert logged.payload["response"]["object"] == served.object == "text_completion" + assert logged.payload["response"]["choices"][0]["text"] == "pong" + + +@pytest.mark.asyncio +async def test_a_bridged_chat_response_waits_on_the_usage_ack_like_any_chat_response( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy import proxy_server + from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_proxy_response + from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings + from tests.unit.proxy.spend_tracking.test_usage_meter import CommitBarrier + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + database: Final = CommitBarrier() + runtime: Final = UsageCaptureRuntime() + runtime.start(database.write, UsageCaptureSettings(batch_size=2)) + monkeypatch.setattr(proxy_server, "usage_capture", runtime) + monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": True}) + logging_obj: Final = Logging( + model="openai/gpt-5.3-codex", + messages=[], + stream=False, + call_type="acompletion", + start_time=datetime.datetime.now(), + litellm_call_id="bridged-chat", + function_id="bridged-chat", + ) + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith="https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=_searched_responses_body("openai/gpt-5.3-codex")) + ) + served: Final = await litellm.acompletion( + model="openai/gpt-5.3-codex", + messages=[{"role": "user", "content": "ping"}], + api_key="test-key", + litellm_logging_obj=logging_obj, + ) + response: Final = asyncio.create_task(capture_proxy_response(logging_obj, served)) + try: + await database.entered.wait() + assert not response.done() + assert database.committed == () + database.release.set() + await response + assert "bridged-chat:inbound" in database.committed + finally: + database.release.set() + await runtime.stop(1) + await response + + +def _served_message_start_id(events: Sequence[bytes]) -> object: + lines: Final = b"".join(events).decode().splitlines() + starts: Final = tuple( + json.loads(line.removeprefix("data:")) for line in lines if line.startswith("data:") and '"message_start"' in line + ) + assert len(starts) == 1 + return starts[0]["message"]["id"] + + +@pytest.mark.parametrize("stream", [False, True], ids=["non-stream", "stream"]) +@pytest.mark.asyncio +async def test_bridged_messages_call_is_logged_as_anthropic_messages_under_the_id_the_client_got( + monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = {**_searched_responses_body("openai/gpt-5.3-codex"), "output": [_PONG_MESSAGE]} + upstream: Final = ( + httpx.Response(200, text=_responses_sse(response_body), headers={"content-type": "text/event-stream"}) + if stream + else httpx.Response(200, json=response_body) + ) + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith="https://api.openai.com/v1/responses").mock(return_value=upstream) + served: Final = await litellm.anthropic.messages.acreate( + model="openai/gpt-5.3-codex", + max_tokens=64, + messages=[{"role": "user", "content": "ping"}], + api_key="test-key", + stream=stream, + ) + served_id: Final = ( + _served_message_start_id([event async for event in cast(AsyncIterator[bytes], served)]) + if stream + else cast(Mapping[str, object], served)["id"] + ) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller["acompletion"].get, timeout=10) + + assert (logged.call_type, logged.payload["call_type"]) == ("anthropic_messages", "anthropic_messages") + assert logged.payload["id"] == served_id + assert logged.payload["response"]["choices"][0]["message"]["content"] == "pong" + + +@pytest.mark.parametrize("caller", ["completion", "acompletion"]) +@pytest.mark.asyncio +async def test_bridged_chat_stream_keeps_the_upstream_headers_when_its_output_cannot_be_translated( + monkeypatch: pytest.MonkeyPatch, caller: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = {**_searched_responses_body("openai/gpt-5.3-codex"), "output": []} + request: Final = _bridged_request("openai/gpt-5.3-codex", {}, stream=True) + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith="https://api.openai.com/v1/responses").mock( + return_value=httpx.Response( + 200, + text=_responses_sse(response_body), + headers={"content-type": "text/event-stream", "x-request-id": "req_upstream"}, + ) + ) + await _served_chunk_ids(caller, request) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10) + + assert isinstance(logged.complete_streaming_response, ModelResponse) + assert logged.complete_streaming_response.id == response_body["id"] + assert logged.payload["hidden_params"]["additional_headers"]["llm_provider-x-request-id"] == "req_upstream" + + def test_convert_chat_completion_messages_to_responses_api_image_input(): from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index 0aee61d8365..2fd3f16ab17 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -112,6 +112,57 @@ def test_redaction_reuses_message_rules_for_request_body_and_provider_fields(sha assert result == original_result and params == original_params +def _reasoning_items() -> list[dict[str, object]]: + return [{"type": "reasoning", "id": "rs_redaction", "summary": [{"type": "summary_text", "text": SECRET}]}] + + +def _message_response() -> ModelResponse: + return ModelResponse( + choices=[{"index": 0, "message": {"role": "assistant", "content": "answer", "reasoning_items": _reasoning_items()}}] + ) + + +def _delta_choices() -> dict[str, object]: + return {"choices": [{"index": 0, "delta": {"content": "answer", "reasoning_items": _reasoning_items()}}]} + + +@pytest.mark.parametrize( + ("details", "result"), + [ + pytest.param({}, _message_response(), id="message_object"), + pytest.param({}, _message_response().model_dump(), id="message_dict"), + pytest.param({}, _delta_choices(), id="delta_dict"), + pytest.param( + {"stream": True, "complete_streaming_response": ModelResponseStream(**_delta_choices())}, + None, + id="stream_chunk", + ), + pytest.param({"standard_logging_object": {"response": _message_response().model_dump()}}, None, id="logged"), + pytest.param({"standard_logging_object": {"response": _delta_choices()}}, None, id="logged_delta"), + ], +) +def test_redaction_hides_reasoning_item_summaries(details: dict[str, object], result: object) -> None: + assert SECRET in str(details) + str(result) + + redacted: Final = perform_redaction(details, result) + + assert SECRET not in str(redacted) and SECRET not in str(details), (redacted, details) + + +def test_an_opted_out_callback_gets_a_text_completion_with_its_answer_redacted() -> None: + logprobs: Final = {"tokens": [SECRET], "token_logprobs": [-0.1], "text_offset": [0], "top_logprobs": [{SECRET: -0.1}]} + response: Final = TextCompletionResponse( + choices=[{"index": 0, "text": SECRET, "logprobs": logprobs, "finish_reason": "stop"}] + ) + + redacted: Final = redacted_response_obj_for_custom_logger(response, CustomLogger(message_logging=False)) + + assert isinstance(redacted, TextCompletionResponse) + assert redacted.choices[0].text == "redacted-by-litellm" + assert SECRET not in redacted.model_dump_json() + assert response.choices[0].text == SECRET, "the caller's own response is not redacted" + + def test_redaction_redacts_reasoning_copied_into_hidden_params_in_place() -> None: hidden_params: Final = {"model_id": "deployment", "provider_specific_fields": _provider_fields()} original_hidden_params: Final = copy.deepcopy(hidden_params) From baddc67041e0888b8bcdc32d43e86c946c867c76 Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Tue, 3 Nov 2026 04:56:22 -0800 Subject: [PATCH 003/122] fix(anthropic)!: carry prompt_cache_breakpoint to cache_control on chat completions to Claude (#49783) * fix(anthropic): carry prompt_cache_breakpoint to cache_control on chat completions to Claude A content part carrying OpenAI's prompt_cache_breakpoint on a /v1/chat/completions request to Claude (Anthropic, Bedrock Invoke and Converse, Vertex AI, Azure AI) now reaches Claude as cache_control {"type": "ephemeral"} on the same block, so prompt caching works for OpenAI-shaped clients instead of the marker being stripped. Only parts OpenAI accepts the marker on are translated, a client cache_control on the same block wins, and Converse translates only for models that take cache points. A 30m ttl or a malformed marker answers UnsupportedParamsError without drop_params and becomes a default 5m mark or no mark with it. * fix(anthropic): hold translated prompt_cache_breakpoint marks to the 4-breakpoint cap Marks past Claude's 4 cache breakpoints now answer 400 without drop_params and keep the earliest under it, counting the client's own message, tool, and top-level cache_control marks. The ttl and malformed-marker errors name drop_params for proxy users too and no longer name a single provider. * fix(bedrock): count Converse tool cache points in the breakpoint cap and gate unmapped models A client tool cache_control or a tool_config injection point becomes a Converse tool cachePoint, so it now takes one of the 4 slots before a translated marker does. Converse only translates prompt_cache_breakpoint for a model the cost map says takes prompt caching. An unmapped model such as an application inference profile ARN opts in through model_info supports_prompt_cache_breakpoint, so a marker sent to every model of a mixed fleet no longer turns a non-caching profile's 200 into Bedrock's 403. Router prompt-caching affinity now treats a marked block like a cache_control one, and requests without a marker skip the translation after one pass over their messages. * fix(bedrock): count only the tool cache points Converse sends against the breakpoint cap * fix(anthropic,bedrock): cap only the cache breakpoints each provider sends, and drop marks Nova cannot take on tool results * perf(anthropic,bedrock): find the breakpoint cut-off by binary search so trimming many marks stays fast * fix(anthropic): drop the duplicate drop_params left by the main merge * fix(bedrock): let an inference profile's cache breakpoint flag win over its base_model * fix(anthropic): keep the first and last prompt_cache_breakpoint marks past the cap Under drop_params the translated marks that fit the cap are now split between the start and the end of the request, so the stable prefix and the newest turns both stay cached. The cap count also includes a native tool search tool that keeps its cache_control on the wire. Bedrock Converse now translates the marker only for the Claude and Nova families, which Bedrock documents as taking cache checkpoints, unless the deployment sets supports_prompt_cache_breakpoint. A guarded user turn keeps its cache mark when the request fits the cap. * refactor(bedrock): decide guarded cache marks with a typed bool helper * fix(bedrock): validate guarded blocks before reading their cache mark * fix(anthropic): let injection points replace client marks and keep non-Claude routes untranslated When a deployment sets cache_control_injection_points, the client's prompt_cache_breakpoint marks are dropped on targets that do not read them, so the operator's injection points decide every Claude breakpoint. OpenCode Go models served over the Anthropic messages endpoint translate the marker only for Claude, and guarded Converse text keeps a cache mark only on a model known to take cache points. * fix(anthropic): keep client marks unless a message injection point replaces them on a translating route A tool_config-only injection point no longer strips client prompt_cache_breakpoint marks, a flagged Bedrock profile no longer counts as reading the marker natively, and guarded Converse text skips the tool count when no message carries a cache mark. * fix(anthropic): keep Bedrock and Claude deployments flagged for the marker on cache_control injection A supports_prompt_cache_breakpoint flag on a Bedrock profile or a Claude deployment made the injection hook write OpenAI markers and prompt_cache_options, which Bedrock answers 400 for. Routes that translate the marker now never take the OpenAI dialect. * fix(bedrock): reserve only the tool cache points Converse sends when trimming Claude cache marks under drop_params * fix(anthropic): name the marker's accepted ttl values in the malformed prompt_cache_breakpoint 400 --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: ryan-crabbe-berri --- .../anthropic_cache_control_hook.py | 228 ++++++++- .../prompt_templates/common_utils.py | 50 ++ litellm/llms/anthropic/chat/transformation.py | 20 +- .../bedrock/chat/converse_transformation.py | 158 +++++- litellm/llms/bedrock/common_utils.py | 51 +- litellm/llms/opencode/chat/transformation.py | 3 + litellm/router_utils/prompt_caching_cache.py | 25 +- .../test_anthropic_cache_control_hook.py | 130 +++++ .../test_anthropic_chat_transformation.py | 179 +++++++ .../chat/test_converse_transformation.py | 455 ++++++++++++++++++ .../llms/opencode/chat/test_transformation.py | 20 + .../router_utils/test_prompt_caching_cache.py | 27 ++ 12 files changed, 1308 insertions(+), 38 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 4302b24ca33..4e473c2f3f0 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -9,6 +9,7 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and """ +import bisect import copy import itertools import os @@ -21,12 +22,16 @@ from urllib.parse import urlparse from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm._logging import verbose_logger +from litellm.exceptions import UnsupportedParamsError from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.integrations.prompt_management_base import PromptManagementClient from litellm.litellm_core_utils.prompt_templates.common_utils import ( + PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT, cache_control_as_prompt_cache_breakpoint, + prompt_cache_breakpoint_as_cache_control, with_prompt_cache_breakpoint, + without_prompt_cache_breakpoints, ) from litellm.litellm_core_utils.prompt_templates.factory import ( tool_call_ids_answered_by_client, @@ -327,6 +332,12 @@ def _part_with_cache_control_renamed(part: object) -> object: return cache_control_as_prompt_cache_breakpoint(_CONTENT_PART_ADAPTER.validate_python(part)) +def _part_with_prompt_cache_breakpoint_renamed(part: object, drop_params: bool) -> object: + if not _carries_prompt_cache_breakpoint(part): + return part + return prompt_cache_breakpoint_as_cache_control(_CONTENT_PART_ADAPTER.validate_python(part), drop_params) + + def configured_injection_points(value: object) -> Sequence[CacheControlInjectionPoint]: if not isinstance(value, (list, tuple)): return () @@ -551,6 +562,10 @@ def _carries_cache_breakpoint(block: object) -> bool: return any(_attribute_or_key(block, key) is not None for key in CACHE_BREAKPOINT_KEYS) +def _carries_cache_control(block: object) -> bool: + return _attribute_or_key(block, "cache_control") is not None + + def _attribute_or_key(value: object, key: str) -> object | None: if hasattr(value, key): return _ATTRIBUTE_VALUE.validate_python(getattr(value, key)) @@ -630,6 +645,12 @@ def _builds_anthropic_messages(model: str, custom_llm_provider: str | None) -> b return isinstance(chat_config, AnthropicConfig) and not isinstance(chat_config, OpenAIGPTConfig) +def _translates_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None) -> bool: + return (custom_llm_provider or _served_provider_for_model(model)) == "bedrock" or _builds_anthropic_messages( + model, custom_llm_provider + ) + + def _custom_llm_provider(non_default_params: Mapping[str, object], point: Mapping[str, object]) -> str | None: provider: Final = non_default_params.get("custom_llm_provider") or point.get(CUSTOM_LLM_PROVIDER_STAMP) return provider if isinstance(provider, str) else None @@ -673,10 +694,79 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool: return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES +def _carries_prompt_cache_breakpoint(block: object) -> bool: + return _accepts_prompt_cache_breakpoint(block) and _attribute_or_key(block, "prompt_cache_breakpoint") is not None + + def _may_carry_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool: return message.get("role") != "assistant" +def _tool_result_carries_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool: + return message.get("role") == "tool" and any( + _carries_prompt_cache_breakpoint(part) for part in _as_object_list(message.get("content")) or () + ) + + +def _part_without_prompt_cache_breakpoint(part: object) -> object: + if not _carries_prompt_cache_breakpoint(part): + return part + return { + key: value + for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items() + if key != "prompt_cache_breakpoint" + } + + +def _tool_result_without_prompt_cache_breakpoints(message: AllMessageValues) -> AllMessageValues: + if not _tool_result_carries_prompt_cache_breakpoint(message): + return message + parts: Final = _as_object_list(_attribute_or_key(message, "content")) or [] + return cast( # cast-ok: the same tool message with its part marks removed + AllMessageValues, {**message, "content": [_part_without_prompt_cache_breakpoint(part) for part in parts]} + ) + + +def _created_cache_control(original: object, translated: object) -> bool: + return ( + _attribute_or_key(original, "cache_control") is None + and _attribute_or_key(translated, "cache_control") is not None + ) + + +def _created_cache_controls(original: object, translated: object) -> tuple[bool, ...]: + if translated is original: + return () + original_parts: Final = _as_object_list(_attribute_or_key(original, "content")) or [] + translated_parts: Final = _as_object_list(_attribute_or_key(translated, "content")) or [] + return tuple(_created_cache_control(before, after) for before, after in zip(original_parts, translated_parts)) + + +def _part_without_cache_control(part: object) -> object: + return {key: value for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items() if key != "cache_control"} + + +def _message_dropping_created_cache_controls_between( + message: AllMessageValues, created: Sequence[bool], marks_before: int, first_dropped: int, first_kept_after: int +) -> AllMessageValues: + if not any(created): + return message + parts: Final = _as_object_list(_attribute_or_key(message, "content")) or [] + ordinals: Final = itertools.accumulate((int(flag) for flag in created), initial=marks_before) + return cast( # cast-ok: the same message with its middle translated marks removed + AllMessageValues, + { + **message, + "content": [ + _part_without_cache_control(part) + if is_created and first_dropped <= ordinal < first_kept_after + else part + for part, is_created, ordinal in zip(parts, created, ordinals) + ], + }, + ) + + # Set by a caller whose message list is not the one that goes upstream -- today the # Responses API layer, whose `instructions` only becomes a system message further down. # Tells this hook to hand role-targeted points to the pass holding the final messages @@ -762,9 +852,6 @@ class AnthropicCacheControlHook(CustomPromptManagement): if not injection_points: return model, messages, non_default_params - # Create a deep copy of messages to avoid modifying the original list - processed_messages = copy.deepcopy(messages) - message_points: Final = tuple( cast(CacheControlMessageInjectionPoint, point) for point in injection_points @@ -775,6 +862,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): request_params: Final = cast(Mapping[str, object], non_default_params) # cast-ok: base hook's bare dict custom_llm_provider: Final = _custom_llm_provider(request_params, injection_points[0]) stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect") + model_info: Final = routed_model_info(request_params.get("model_info")) openai_dialect: Final = ( stamped_dialect if isinstance(stamped_dialect, bool) @@ -783,9 +871,14 @@ class AnthropicCacheControlHook(CustomPromptManagement): custom_llm_provider, request_params.get("api_base") or request_params.get("base_url"), request_params.get("prompt_cache_options"), - routed_model_info(request_params.get("model_info")), + model_info, ) ) + processed_messages = copy.deepcopy( + [without_prompt_cache_breakpoints(message) for message in messages] + if not openai_dialect and message_points and _translates_prompt_cache_breakpoint(model, custom_llm_provider) + else messages + ) # A provisional message list defers every role-targeted point to the pass holding # the final one: a role with no message here may have one there, and settling all # of them in one pass is what lets config order decide the shared breakpoint @@ -876,7 +969,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) ) if hosted_flag is not None: - return hosted_flag + return hosted_flag and not _translates_prompt_cache_breakpoint(model, custom_llm_provider) if not supports_openai_prompt_cache_breakpoint(model, model_info=model_info): return False if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai": @@ -929,6 +1022,19 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) return int(wire_cache_control is not None) + tool_blocks + envelope_blocks + @staticmethod + def tools_keeping_cache_control_on_chat(tools: object) -> tuple[object, ...]: + return tuple( + tool for tool in _as_object_list(tools) or () if not _chat_transform_drops_tool_cache_control(tool) + ) + + @staticmethod + def count_chat_external_cache_breakpoints(optional_params: object) -> int: + params: Final = _validated_object_mapping(optional_params) or {} + return AnthropicCacheControlHook.count_external_cache_breakpoints( + _as_object_list(params.get("tools")), params.get("cache_control") + ) + @staticmethod def count_external_cache_breakpoints_on_messages_route( tools: Iterable[object] | None, cache_control: object, request_kwargs: object @@ -1084,6 +1190,16 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) return message_count + content_count + tool_call_count + @staticmethod + def count_converse_cache_points(message: AllMessageValues) -> int: + content: Final = _as_object_list(_attribute_or_key(message, "content")) + tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls")) or [] + marked_parts: Final = sum(1 for block in content or () if _carries_cache_control(block)) + marked_tool_calls: Final = sum(1 for tool_call in tool_calls if _carries_cache_control(tool_call)) + if message.get("role") == "tool": + return int(_carries_cache_control(message) or marked_parts > 0) + return (int(_carries_cache_control(message)) if content is None else marked_parts) + marked_tool_calls + @staticmethod def _message_has_cache_control(message: AllMessageValues, answered_tool_ids: Container[str]) -> bool: """Return True if the message already carries any cache_control.""" @@ -1142,6 +1258,92 @@ class AnthropicCacheControlHook(CustomPromptManagement): return renamed return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed) + @staticmethod + def prompt_cache_breakpoints_as_cache_control(message: AllMessageValues, drop_params: bool) -> AllMessageValues: + """Every OpenAI ``prompt_cache_breakpoint`` on a part OpenAI accepts one on moved to Anthropic's + ``cache_control``. Marks with no OpenAI slot are left for the caller to strip.""" + if not _may_carry_prompt_cache_breakpoint(message): + return message + parts: Final = _as_object_list(_attribute_or_key(message, "content")) + if parts is None or not any(_carries_prompt_cache_breakpoint(part) for part in parts): + return message + return cast( # cast-ok: the same message with its part marks renamed + AllMessageValues, + {**message, "content": [_part_with_prompt_cache_breakpoint_renamed(part, drop_params) for part in parts]}, + ) + + @staticmethod + def prompt_cache_breakpoints_dropped_from_tool_results( + messages: Sequence[AllMessageValues], drop_params: bool, model: str + ) -> tuple[AllMessageValues, ...]: + if not any(_tool_result_carries_prompt_cache_breakpoint(message) for message in messages): + return tuple(messages) + if not drop_params: + raise UnsupportedParamsError( + message=( + f"{model} takes no cache breakpoint on a tool result, so prompt_cache_breakpoint on a tool " + f"message has no equivalent. Remove it, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it." + ), + status_code=400, + ) + verbose_logger.debug("Dropping prompt_cache_breakpoint from tool results for %s under drop_params", model) + return tuple(_tool_result_without_prompt_cache_breakpoints(message) for message in messages) + + @staticmethod + def prompt_cache_breakpoints_as_capped_cache_control( + messages: Sequence[AllMessageValues], + drop_params: bool, + sent_breakpoints: Callable[[AllMessageValues], int], + external_breakpoints: Callable[[], int], + ) -> tuple[AllMessageValues, ...]: + """``prompt_cache_breakpoints_as_cache_control`` over the whole request, held to the provider's + breakpoint cap, counting each message by the breakpoints ``sent_breakpoints`` says its provider sends. + Under ``drop_params`` the first and last translated marks fill the slots the client's own + ``cache_control`` marks leave, the first half taking the odd slot, so the stable prefix and the newest + turns stay cached, and without it a request over the cap is rejected.""" + translated: Final = tuple( + AnthropicCacheControlHook.prompt_cache_breakpoints_as_cache_control(message, drop_params) + for message in messages + ) + if all(after is before for before, after in zip(messages, translated)): + return translated + created: Final = tuple(_created_cache_controls(before, after) for before, after in zip(messages, translated)) + created_marks: Final = sum(sum(flags) for flags in created) + if created_marks == 0: + return translated + external: Final = external_breakpoints() + breakpoints: Final = sum(map(sent_breakpoints, translated)) + external + excess: Final = breakpoints - MAX_CACHE_CONTROL_BLOCKS + if excess <= 0: + return translated + if not drop_params: + raise UnsupportedParamsError( + message=( + f"This model takes at most {MAX_CACHE_CONTROL_BLOCKS} cache breakpoints per request, and this " + f"request has {breakpoints} once its prompt_cache_breakpoint marks are translated. Remove {excess} " + f"of them, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to keep the first and last of them." + ), + status_code=400, + ) + verbose_logger.debug("Dropping the middle prompt_cache_breakpoint marks past the cap under drop_params") + marks_before: Final = tuple(itertools.accumulate((sum(flags) for flags in created), initial=0)) + + def capped_request(kept_marks: int) -> tuple[AllMessageValues, ...]: + first_dropped: Final = (kept_marks + 1) // 2 + first_kept_after: Final = created_marks - kept_marks // 2 + return tuple( + _message_dropping_created_cache_controls_between( + message, flags, before, first_dropped, first_kept_after + ) + for message, flags, before in zip(translated, created, marks_before) + ) + + def over_cap(kept_marks: int) -> bool: + return sum(map(sent_breakpoints, capped_request(kept_marks))) + external > MAX_CACHE_CONTROL_BLOCKS + + first_over_cap: Final = bisect.bisect_left(range(max(0, created_marks - excess) + 1), True, key=over_cap) + return capped_request(max(0, first_over_cap - 1)) + @staticmethod def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues: if not _may_carry_prompt_cache_breakpoint(message): @@ -1487,9 +1689,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): configured: Final = configured_injection_points(non_default_params.get("cache_control_injection_points")) if configured: - tools_keeping_marks: Final = tuple( - tool for tool in tools or () if not _chat_transform_drops_tool_cache_control(tool) - ) + tools_keeping_marks: Final = AnthropicCacheControlHook.tools_keeping_cache_control_on_chat(tools) stamped: Final = AnthropicCacheControlHook._stamped_for_prompt_hook( configured, AnthropicCacheControlHook.count_external_cache_breakpoints( @@ -1782,9 +1982,17 @@ def messages_within_cache_control_cap( Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first. """ - reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints( - _validated_object_list(tools), cache_control, request_kwargs + return messages_within_reserved_cache_control_cap( + messages, + AnthropicCacheControlHook.count_external_cache_breakpoints( + _validated_object_list(tools), cache_control, request_kwargs + ), ) + + +def messages_within_reserved_cache_control_cap( + messages: Sequence[_MessageT], reserved_marks: int +) -> Sequence[_MessageT]: kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks) dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept)) if dropped > 0: diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 6287dc17e8f..6ecab166dcd 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -21,12 +21,14 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm import verbose_logger +from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.batch_utils import InMemoryFile from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionAudioObject, + ChatCompletionCachedContent, ChatCompletionFileObject, ChatCompletionImageObject, ChatCompletionPromptCacheBreakpoint, @@ -2183,6 +2185,9 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool: _MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object]) +PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT: Final = ( + "set `drop_params: true` on the request or in litellm_settings (`litellm.drop_params = True` in the SDK)" +) def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: @@ -2262,6 +2267,51 @@ def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT: ) +def prompt_cache_breakpoint_as_cache_control(block: _MarkedT, drop_params: bool) -> _MarkedT: + """OpenAI's ``prompt_cache_breakpoint`` mark rewritten as Anthropic's ``cache_control`` on the same block. + A ``cache_control`` the client already wrote on the block wins, TTL included.""" + marker: Final = block.get("prompt_cache_breakpoint") + if marker is None: + return block + unmarked: Final = {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"} + control: Final = ( + None + if block.get("cache_control") is not None + else _cache_control_for_prompt_cache_breakpoint(marker, drop_params) + ) + marked: Final = unmarked if control is None else {**unmarked, "cache_control": control} + return cast(_MarkedT, marked) # cast-ok: same block shape as the input with the marker key renamed + + +def _cache_control_for_prompt_cache_breakpoint(marker: object, drop_params: bool) -> ChatCompletionCachedContent | None: + try: + breakpoint_mark: Final = _PROMPT_CACHE_BREAKPOINT.validate_python(marker) + except ValidationError: + if not drop_params: + raise UnsupportedParamsError( + message=( + f"prompt_cache_breakpoint must be {{'mode': 'explicit'}} with an optional ttl of '30m' or '12h', got " + f"{marker!r}. Fix the marker, or " + f"{PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it." + ), + status_code=400, + ) + verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker) + return None + ttl: Final = breakpoint_mark.get("ttl") + if ttl is not None and not drop_params: + raise UnsupportedParamsError( + message=( + f"This model's prompt cache lasts 5m, or 1h where it offers one, so prompt_cache_breakpoint " + f"ttl={ttl!r} has no equivalent. Remove the ttl, {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to send " + "a 5m breakpoint instead, or send cache_control {'type': 'ephemeral', 'ttl': '1h'} on the block " + "where the model offers a 1h cache." + ), + status_code=400, + ) + return ChatCompletionCachedContent(type="ephemeral") + + _BREAKPOINT_BLOCK_LIST_KEYS: Final = frozenset({"content", "tool_calls"}) _BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object]) _BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object]) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 1977e3e2a7d..6b96df8f879 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -2370,6 +2370,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params ) + def translates_prompt_cache_breakpoint(self, model: str) -> bool: + return True + def transform_request( self, model: str, @@ -2383,19 +2386,32 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): """ ## VALIDATE REQUEST """Anthropic requires ``tools`` when messages include tool blocks; LiteLLM injects a dummy tool if omitted (no ``modify_params`` needed).""" + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, cache_control_count from litellm.litellm_core_utils.prompt_templates.factory import ( anthropic_messages_pt, ) drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params")) + translated_messages: Final = ( + list( + AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control( + messages, + drop_params, + cache_control_count, + lambda: AnthropicCacheControlHook.count_chat_external_cache_breakpoints(optional_params), + ) + ) + if self.translates_prompt_cache_breakpoint(model) + else messages + ) capped_messages: Final = ( list( self._messages_within_cache_control_cap( - messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params) + translated_messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params) ) ) if drop_params - else messages + else translated_messages ) if "tools" not in optional_params and has_tool_call_blocks(capped_messages): diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index f2985237554..056f34f1bf4 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -23,6 +23,11 @@ from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, RESPONSE_FORMAT_TOOL_NAME, ) +from litellm.integrations.anthropic_cache_control_hook import ( + MAX_CACHE_CONTROL_BLOCKS, + AnthropicCacheControlHook, + configured_injection_points, +) from litellm.litellm_core_utils.core_helpers import ( as_str_mapping, drops_unsupported_params, @@ -131,6 +136,7 @@ from ..common_utils import ( bedrock_capability_model, bedrock_converse_supports_parallel_tool_use_config, bedrock_model_accepts_cache_points, + bedrock_model_known_to_accept_cache_points, bedrock_reasoning_effort_disabled, bedrock_rejects_stop_sequences, get_anthropic_beta_from_headers, @@ -204,6 +210,11 @@ def _has_native_text_format(output_config: object) -> bool: return False +def _cache_mark_kept(block: Mapping[str, object], keeps_cache_marks: bool) -> Mapping[str, object]: + cache_control: Final = block.get("cache_control") + return {"cache_control": cache_control} if keeps_cache_marks and cache_control is not None else {} + + class AmazonConverseConfig(BaseConfig): """ Reference - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html @@ -243,7 +254,7 @@ class AmazonConverseConfig(BaseConfig): @staticmethod def _convert_consecutive_user_messages_to_guarded_text( - messages: list[AllMessageValues], optional_params: dict + messages: list[AllMessageValues], optional_params: dict, keeps_cache_marks: bool = False ) -> list[AllMessageValues]: """ Convert consecutive user messages at the end to guarded_text type if guardrailConfig is present @@ -281,7 +292,11 @@ class AmazonConverseConfig(BaseConfig): new_content = [] for item in content: if isinstance(item, dict) and item.get("type") == "text": - new_item = {"type": "guarded_text", "text": item["text"]} + new_item = { + "type": "guarded_text", + "text": item["text"], + **_cache_mark_kept(_JSON_OBJECT.validate_python(item), keeps_cache_marks), + } new_content.append(new_item) else: new_content.append(item) @@ -289,7 +304,13 @@ class AmazonConverseConfig(BaseConfig): messages_copy[user_message_index]["content"] = new_content elif isinstance(content, str): # If content is a string, convert it to guarded_text - messages_copy[user_message_index]["content"] = [{"type": "guarded_text", "text": content}] + messages_copy[user_message_index]["content"] = [ + { + "type": "guarded_text", + "text": content, + **_cache_mark_kept(_JSON_OBJECT.validate_python(user_message), keeps_cache_marks), + } + ] return messages_copy @@ -1595,6 +1616,74 @@ class AmazonConverseConfig(BaseConfig): return ChatCompletionTextObject(type="text", text=text) return ChatCompletionTextObject(type="text", text=text, cache_control=cache_control) + def _messages_with_cache_marks( + self, + messages: Sequence[AllMessageValues], + model: str, + deployment_model: str, + drop_params: bool, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + model_info: Mapping[str, object] | None, + ) -> Sequence[AllMessageValues]: + if not bedrock_model_known_to_accept_cache_points(model, deployment_model, model_info): + return messages + markable: Final = ( + AnthropicCacheControlHook.prompt_cache_breakpoints_dropped_from_tool_results(messages, drop_params, model) + if "nova" in model.lower() + else messages + ) + return AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control( + markable, + drop_params, + AnthropicCacheControlHook.count_converse_cache_points, + lambda: self._tool_cache_points_on_the_wire( + model, deployment_model, optional_params, litellm_params, model_info + ), + ) + + def _tool_cache_points_on_the_wire( + self, + model: str, + deployment_model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + model_info: Mapping[str, object] | None, + ) -> int: + tools: Final = optional_params.get("tools") + bedrock_tools, _ = self._process_tools_and_beta( + tools if isinstance(tools, list) else [], model, None, {}, litellm_params, deployment_model, model_info + ) + injects_tool_config_point: Final = bool(bedrock_tools) and any( + point.get("location") == "tool_config" + for point in configured_injection_points(optional_params.get("cache_control_injection_points")) + ) + return sum(1 for tool in bedrock_tools if "cachePoint" in tool) + int(injects_tool_config_point) + + def _guarded_text_keeps_cache_marks( + self, + messages: Sequence[AllMessageValues], + system_content_blocks: Sequence[SystemContentBlock], + model: str, + deployment_model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + model_info: Mapping[str, object] | None, + ) -> bool: + if "guardrailConfig" not in optional_params or not bedrock_model_known_to_accept_cache_points( + model, deployment_model, model_info + ): + return False + message_cache_points: Final = sum(map(AnthropicCacheControlHook.count_converse_cache_points, messages)) + if message_cache_points == 0: + return False + cache_points: Final = ( + sum(1 for block in system_content_blocks if "cachePoint" in block) + + message_cache_points + + self._tool_cache_points_on_the_wire(model, deployment_model, optional_params, litellm_params, model_info) + ) + return cache_points <= MAX_CACHE_CONTROL_BLOCKS + def _transform_system_message( self, messages: list[AllMessageValues], @@ -2152,22 +2241,27 @@ class AmazonConverseConfig(BaseConfig): return data - @staticmethod def _claude_messages_within_cache_control_cap( + self, messages: Sequence[AllMessageValues], capability_model: str, + deployment_model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object], + model_info: Mapping[str, object] | None, ) -> Sequence[AllMessageValues]: if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower(): return messages from litellm.integrations.anthropic_cache_control_hook import ( - messages_within_cache_control_cap, + messages_within_reserved_cache_control_cap, tool_message_with_single_cache_mark, ) - return messages_within_cache_control_cap( - tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools") + return messages_within_reserved_cache_control_cap( + tuple(map(tool_message_with_single_cache_mark, messages)), + self._tool_cache_points_on_the_wire( + capability_model, deployment_model, optional_params, litellm_params, model_info + ), ) async def _async_transform_request( @@ -2183,10 +2277,20 @@ class AmazonConverseConfig(BaseConfig): base_model: Final = litellm_params.get("base_model") capability_model: Final = bedrock_capability_model(model, base_model) model_info: Final = routed_model_info(litellm_params.get("model_info")) + json_optional_params: Final = _JSON_OBJECT.validate_python(optional_params) + marked_messages: Final = self._messages_with_cache_marks( + messages, + capability_model, + model, + drops_unsupported_params(litellm_params.get("drop_params")), + json_optional_params, + litellm_params, + model_info, + ) messages, system_content_blocks = self._transform_system_message( list( self._claude_messages_within_cache_control_cap( - messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params + marked_messages, capability_model, model, json_optional_params, litellm_params, model_info ) ), model=capability_model, @@ -2195,8 +2299,19 @@ class AmazonConverseConfig(BaseConfig): ) # Convert last user message to guarded_text if guardrailConfig is present + guard_input: Final = self._handle_orphaned_tool_blocks(messages, optional_params) messages = self._convert_consecutive_user_messages_to_guarded_text( - self._handle_orphaned_tool_blocks(messages, optional_params), optional_params + guard_input, + optional_params, + keeps_cache_marks=self._guarded_text_keeps_cache_marks( + guard_input, + system_content_blocks, + capability_model, + model, + json_optional_params, + litellm_params, + model_info, + ), ) ## TRANSFORMATION ## @@ -2276,10 +2391,20 @@ class AmazonConverseConfig(BaseConfig): base_model: Final = litellm_params.get("base_model") capability_model: Final = bedrock_capability_model(model, base_model) model_info: Final = routed_model_info(litellm_params.get("model_info")) + json_optional_params: Final = _JSON_OBJECT.validate_python(optional_params) + marked_messages: Final = self._messages_with_cache_marks( + messages, + capability_model, + model, + drops_unsupported_params(litellm_params.get("drop_params")), + json_optional_params, + litellm_params, + model_info, + ) messages, system_content_blocks = self._transform_system_message( list( self._claude_messages_within_cache_control_cap( - messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params + marked_messages, capability_model, model, json_optional_params, litellm_params, model_info ) ), model=capability_model, @@ -2288,8 +2413,19 @@ class AmazonConverseConfig(BaseConfig): ) # Convert last user message to guarded_text if guardrailConfig is present + guard_input: Final = self._handle_orphaned_tool_blocks(messages, optional_params) messages = self._convert_consecutive_user_messages_to_guarded_text( - self._handle_orphaned_tool_blocks(messages, optional_params), optional_params + guard_input, + optional_params, + keeps_cache_marks=self._guarded_text_keeps_cache_marks( + guard_input, + system_content_blocks, + capability_model, + model, + json_optional_params, + litellm_params, + model_info, + ), ) _data: Final[CommonRequestObject] = self._transform_request_helper( diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 1f10b73505b..c1b3b5616b8 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1559,22 +1559,63 @@ def bedrock_model_accepts_cache_points( def bedrock_declared_cache_point_support(model: str, model_info: Mapping[str, object] | None = None) -> bool | None: if _OPENAI_FAMILY_MODEL_RE.search(model): return False + explicit_marker_support: Final = _explicit_cache_point_support(model, model_info) + if explicit_marker_support is not None: + return explicit_marker_support + return _mapped_prompt_caching_support(model, model_info) + + +def bedrock_model_known_to_accept_cache_points( + model: str, deployment_model: str | None = None, model_info: Mapping[str, object] | None = None +) -> bool: + """Like ``bedrock_model_accepts_cache_points``, but only for the Claude and Nova families Bedrock documents + as taking cache checkpoints, unless the entry sets ``supports_prompt_cache_breakpoint``: an OpenAI + ``prompt_cache_breakpoint`` reaches every model of a mixed fleet, not only the ones a client marked for + Bedrock caching, and other families price implicit cache reads yet may reject the cachePoint field.""" + deployment_declared: Final = ( + None if deployment_model is None else _declared_marker_support(deployment_model, model_info) + ) + declared: Final = ( + deployment_declared if deployment_declared is not None else _declared_marker_support(model, model_info) + ) + return declared is True + + +def _declared_marker_support(model: str, model_info: Mapping[str, object] | None) -> bool | None: + if _OPENAI_FAMILY_MODEL_RE.search(model): + return False + explicit_marker_support: Final = _explicit_cache_point_support(model, model_info) + if explicit_marker_support is not None: + return explicit_marker_support + mapped: Final = _mapped_prompt_caching_support(model, model_info) + if not _documents_cache_checkpoints(model): + return None if mapped is None else False + return mapped is not False + + +def _documents_cache_checkpoints(model: str) -> bool: + return _is_claude(model) or get_bedrock_base_model(model).startswith("amazon.nova") + + +def _explicit_cache_point_support(model: str, model_info: Mapping[str, object] | None) -> bool | None: deployment_flag: Final = None if model_info is None else model_info.get("supports_prompt_cache_breakpoint") if deployment_flag is not None: return deployment_flag is True catalog_entry: Final = _catalog_entry_reader(model_info) - map_keys: Final = (model, get_bedrock_base_model(model)) registered_keys: Final = tuple(f"bedrock/{route}{model}" for route in ("", "converse/", "converse_like/")) - explicit_marker_support: Final = next( + return next( ( entry.get("supports_prompt_cache_breakpoint") is True - for key in (*registered_keys, *map_keys) + for key in (*registered_keys, model, get_bedrock_base_model(model)) if (entry := catalog_entry(key)) is not None and entry.get("supports_prompt_cache_breakpoint") is not None ), None, ) - if explicit_marker_support is not None: - return explicit_marker_support + + +def _mapped_prompt_caching_support(model: str, model_info: Mapping[str, object] | None) -> bool | None: + catalog_entry: Final = _catalog_entry_reader(model_info) + map_keys: Final = (model, get_bedrock_base_model(model)) if not any(catalog_entry(key) is not None for key in map_keys): return None deployment_caching: Final = None if model_info is None else lax_bool(model_info.get("supports_prompt_caching")) diff --git a/litellm/llms/opencode/chat/transformation.py b/litellm/llms/opencode/chat/transformation.py index 7e16f93d7cb..c6d288b6d84 100644 --- a/litellm/llms/opencode/chat/transformation.py +++ b/litellm/llms/opencode/chat/transformation.py @@ -112,6 +112,9 @@ class OpenCodeGoMessagesChatConfig(_GoReportedCostMixin, AnthropicConfig): def custom_llm_provider(self) -> str: return "opencode_go" + def translates_prompt_cache_breakpoint(self, model: str) -> bool: + return "claude" in model.lower() + def get_complete_url( self, api_base: str | None, diff --git a/litellm/router_utils/prompt_caching_cache.py b/litellm/router_utils/prompt_caching_cache.py index f78b5982186..1d91255e28c 100644 --- a/litellm/router_utils/prompt_caching_cache.py +++ b/litellm/router_utils/prompt_caching_cache.py @@ -55,6 +55,17 @@ def _sorted_pairs(pairs: Iterable[tuple[str, JsonValue]]) -> tuple[tuple[str, Js return tuple(sorted(pairs, key=lambda pair: pair[0])) +_CACHE_MARKER_KEYS: Final = frozenset(("cache_control", "prompt_cache_breakpoint")) + + +def _is_cache_marked(block: Mapping[str, object]) -> bool: + match block: + case {"cache_control": {"type": "ephemeral"}}: + return True + case _: + return block.get("prompt_cache_breakpoint") is not None + + def _canonical_bytes(value: object) -> bytes: return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() @@ -66,7 +77,7 @@ def _block_unit( return _canonical_bytes((envelope, block)), message_run_type block_type: Final = block.get("type") block_run_type: Final = block_type if isinstance(block_type, str) and block_type in _TOOL_RUN_BLOCK_TYPES else None - stripped: Final = _sorted_pairs(item for item in block.items() if item[0] != "cache_control") + stripped: Final = _sorted_pairs(item for item in block.items() if item[0] not in _CACHE_MARKER_KEYS) return _canonical_bytes((envelope, stripped)), message_run_type or block_run_type @@ -264,15 +275,9 @@ class PromptCachingCache: continue for content_idx, content_block in enumerate(content): - if isinstance(content_block, dict): - cache_control = content_block.get("cache_control") - if ( - cache_control is not None - and isinstance(cache_control, dict) - and cache_control.get("type") == "ephemeral" - ): - last_cacheable_message_idx = msg_idx - last_cacheable_content_idx = content_idx + if isinstance(content_block, dict) and _is_cache_marked(content_block): + last_cacheable_message_idx = msg_idx + last_cacheable_content_idx = content_idx # If no cacheable block found, return empty list (no cacheable prefix) if last_cacheable_message_idx is None: diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index c040d8e5bac..766038d5c4f 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -1401,6 +1401,109 @@ def _marks_the_route_sends(model: str, custom_llm_provider: str | None, messages _SECOND_SEARCH_CALL: Final = {**_SEARCH_CALL, "id": "srvtoolu_02Search"} +_CLIENT_MARKED_CONVERSATION: Final = ( + {"role": "system", "content": [{"type": "text", "text": "Handbook", "prompt_cache_breakpoint": {"mode": "explicit"}}]}, + {"role": "user", "content": [{"type": "text", "text": "First", "prompt_cache_breakpoint": {"mode": "explicit"}}]}, + {"role": "assistant", "content": "First answer"}, + {"role": "user", "content": "Second"}, +) + + +def _hook_result( + model: str, points: tuple[Mapping[str, object], ...] = ({"location": "message", "index": -1},), **params: object +) -> tuple[list[AllMessageValues], Mapping[str, object]]: + _, processed, returned_params = AnthropicCacheControlHook().get_chat_completion_prompt( + model=model, + messages=cast(list[AllMessageValues], copy.deepcopy(list(_CLIENT_MARKED_CONVERSATION))), + non_default_params={"cache_control_injection_points": list(points), **params}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return processed, returned_params + + +def _hook_output( + model: str, points: tuple[Mapping[str, object], ...] = ({"location": "message", "index": -1},), **params: object +) -> list[AllMessageValues]: + return _hook_result(model, points, **params)[0] + + +def test_cache_control_injection_points_replace_client_prompt_cache_breakpoints_on_claude(): + request: Final = AnthropicConfig().transform_request( + model="claude-sonnet-5-5", + messages=_hook_output("anthropic/claude-sonnet-5-5"), + optional_params={"max_tokens": 64}, + litellm_params={}, + headers={}, + ) + + assert _marks_in(request["system"]) == 0 + assert [_marks_in(message) for message in request["messages"]] == [0, 0, 1] + assert "prompt_cache_breakpoint" not in json.dumps(request) + + +@pytest.mark.parametrize( + "sibling_model_infos", + [ + pytest.param((), id="one-deployment"), + pytest.param( + ({"supports_prompt_cache_breakpoint": True, "base_model": "anthropic.claude-unmapped-v1:0"},), + id="beside-a-deployment-of-the-same-profile-with-a-base-model", + ), + ], +) +@pytest.mark.parametrize("route", ["bedrock", "bedrock/converse"]) +def test_cache_control_injection_points_replace_client_prompt_cache_breakpoints_on_a_flagged_bedrock_profile( + local_model_cost_map, sibling_model_infos: tuple[Mapping[str, object], ...], route: str +): + from litellm import Router + + model: Final = f"{route}/arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-flagged" + Router( + model_list=[ + { + "model_name": f"claude-profile-{position}", + "litellm_params": {"model": model, "aws_region_name": "us-east-1"}, + "model_info": model_info, + } + for position, model_info in enumerate(({"supports_prompt_cache_breakpoint": True}, *sibling_model_infos)) + ] + ) + processed, params = _hook_result(model) + + assert "prompt_cache_breakpoint" not in json.dumps(processed) + assert "prompt_cache_options" not in params + assert [_marks_in(message) for message in processed] == [0, 0, 0, 1] + + +def test_tool_config_injection_point_leaves_client_prompt_cache_breakpoints_on_claude(): + processed: Final = _hook_output("anthropic/claude-sonnet-5-5", points=({"location": "tool_config"},)) + + assert [message["content"] for message in processed[:2]] == [ + message["content"] for message in _CLIENT_MARKED_CONVERSATION[:2] + ] + + +@pytest.mark.parametrize( + ("model", "params"), + [ + pytest.param("openai/gpt-5.6", {"api_base": "https://gateway.example.com/v1"}, id="reads-them"), + pytest.param("openai/gpt-4o", {}, id="openai-model-that-predates-them"), + pytest.param("hosted_vllm/qwen3", {"api_base": "http://vllm.example.com/v1"}, id="self-hosted"), + pytest.param("azure_ai/qa-unflagged-gpt", {"api_base": "https://qa.services.ai.azure.com/models"}, id="foundry"), + ], +) +def test_cache_control_injection_points_keep_client_prompt_cache_breakpoints_on_a_route_that_does_not_translate_them( + model: str, params: Mapping[str, object] +): + processed: Final = _hook_output(model, **params) + + assert [message["content"] for message in processed[:2]] == [ + message["content"] for message in _CLIENT_MARKED_CONVERSATION[:2] + ] + + @pytest.mark.parametrize( ("model", "custom_llm_provider"), [ @@ -4965,6 +5068,33 @@ class TestMalformedInjectionPointsAreIgnored: assert kwargs == {"cache_control_injection_points": value} +def test_capping_thousands_of_marks_on_one_tool_result_counts_the_request_a_logarithmic_number_of_times(): + def marked_parts(prefix: str, count: int) -> list[dict]: + return [ + {"type": "text", "text": f"{prefix} {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(count) + ] + + messages: Final = [ + {"role": "user", "content": marked_parts("part", 4)}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": marked_parts("result", 2000)}, + ] + sent_breakpoints: Final = MagicMock(side_effect=AnthropicCacheControlHook.count_converse_cache_points) + + capped: Final = AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control( + cast(List[AllMessageValues], messages), True, sent_breakpoints, lambda: 0 + ) + + assert [part.get("cache_control") for part in capped[0]["content"]] == [{"type": "ephemeral"}] * 3 + [None] + assert ["cache_control" in part for part in capped[2]["content"]] == [False] * 1997 + [True] * 3 + assert sent_breakpoints.call_count < 3 * 20 + + def test_cache_control_hook_leaves_the_duplicate_tool_result_warning_to_the_request_builder( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ): diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 13ff11e8a3f..fc873832956 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,4 +1,5 @@ import copy +import itertools import json import logging from collections.abc import Iterator @@ -4422,6 +4423,184 @@ def test_transform_request_includes_cache_control(): assert result["cache_control"] == {"type": "ephemeral"} +def _anthropic_body(messages: list, litellm_params: dict | None = None, tools: list | None = None) -> dict: + return AnthropicConfig().transform_request( + model="claude-sonnet-5-5", + messages=messages, + optional_params={"max_tokens": 100, **({"tools": tools} if tools else {})}, + litellm_params=litellm_params or {}, + headers={}, + ) + + +def test_transform_request_carries_prompt_cache_breakpoint_to_cache_control(): + body: Final = _anthropic_body( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are terse.", "prompt_cache_breakpoint": {"mode": "explicit"}} + ], + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit"}}, + { + "type": "text", + "text": "Bye", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + "prompt_cache_breakpoint": {"mode": "explicit"}, + }, + ], + }, + ] + ) + + assert body["system"] == [{"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}] + assert body["messages"] == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "Bye", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + ], + } + ] + + +@pytest.mark.parametrize( + "marker, expected_block", + [ + ({"mode": "explicit", "ttl": "30m"}, {"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}), + ({"mode": "implicit"}, {"type": "text", "text": "Hello"}), + ], +) +def test_transform_request_prompt_cache_breakpoint_without_claude_equivalent(marker, expected_block): + messages: Final = [{"role": "user", "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": marker}]}] + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"): + _anthropic_body(copy.deepcopy(messages)) + + body: Final = _anthropic_body(messages, litellm_params={"drop_params": True}) + assert body["messages"] == [{"role": "user", "content": [expected_block]}] + + +def test_transform_request_names_the_marker_ttl_rule_for_a_ttl_only_claude_takes(): + messages: Final = [ + {"role": "user", "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "1h"}}]} + ] + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="optional ttl of '30m' or '12h'"): + _anthropic_body(messages) + + +def test_transform_request_keeps_the_first_and_last_prompt_cache_breakpoints_within_the_cap(): + client_marked_part: Final = {"type": "text", "text": "part 0", "cache_control": {"type": "ephemeral"}} + marked_parts: Final = [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(1, 5) + ] + messages: Final = [{"role": "user", "content": [client_marked_part, *marked_parts]}] + tools: Final = [{"name": "lookup", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}}] + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="at most 4 cache breakpoints"): + _anthropic_body(copy.deepcopy(messages), tools=copy.deepcopy(tools)) + + body: Final = _anthropic_body(messages, litellm_params={"drop_params": True}, tools=tools) + assert [part.get("cache_control") for part in body["messages"][0]["content"]] == [ + {"type": "ephemeral"}, + {"type": "ephemeral"}, + None, + None, + {"type": "ephemeral"}, + ] + assert body["tools"][0]["cache_control"] == {"type": "ephemeral"} + + +def test_transform_request_counts_a_cache_marked_tool_search_tool_toward_the_cap(): + config: Final = AnthropicConfig() + optional_params: Final = config.map_openai_params( + non_default_params={ + "max_tokens": 100, + "tools": [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + optional_params={}, + model="claude-sonnet-5-5", + drop_params=False, + ) + marked_turns: Final = [ + { + "role": "user", + "content": [{"type": "text", "text": f"turn {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}], + } + for index in range(4) + ] + + def body(litellm_params: dict) -> dict: + return config.transform_request( + model="claude-sonnet-5-5", + messages=copy.deepcopy(marked_turns), + optional_params=copy.deepcopy(optional_params), + litellm_params=litellm_params, + headers={}, + ) + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="this request has 5"): + body({}) + + capped: Final = body({"drop_params": True}) + assert capped["tools"][0]["cache_control"] == {"type": "ephemeral"} + sent_parts: Final = itertools.chain.from_iterable(message["content"] for message in capped["messages"]) + assert [part.get("cache_control") for part in sent_parts] == [ + {"type": "ephemeral"}, + {"type": "ephemeral"}, + None, + {"type": "ephemeral"}, + ] + + +@pytest.mark.parametrize( + "unsent_marks", + [ + pytest.param( + [ + { + "role": "assistant", + "content": [{"type": "text", "text": "Noted.", "prompt_cache_breakpoint": {"mode": "explicit"}}], + } + ], + id="assistant_part", + ), + pytest.param( + [{"role": "user", "content": "Also this.", "prompt_cache_breakpoint": {"mode": "explicit"}}], + id="message_level", + ), + ], +) +def test_transform_request_caps_only_the_marks_claude_receives(unsent_marks: list[dict]): + marked_parts: Final = [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} for index in range(4) + ] + + body: Final = _anthropic_body( + [{"role": "user", "content": marked_parts}, *unsent_marks, {"role": "user", "content": "Go on."}] + ) + + sent_marks: Final = [ + part.get("cache_control") + for part in itertools.chain.from_iterable(message["content"] for message in body["messages"]) + if isinstance(part, dict) + ] + assert sent_marks.count({"type": "ephemeral"}) == 4 + + def test_compaction_block_empty_list_not_added(): """ Test that empty compaction_blocks list is not added to provider_specific_fields. diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 8cab1018d25..7c19ef5b751 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -10073,6 +10073,461 @@ def test_bedrock_prompt_caching_message(messages, expected_cache_control): assert "cachePoint" not in json.dumps(transformed_messages) +def test_converse_carries_prompt_cache_breakpoint_to_cache_point(): + request: Final = litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=[ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are terse.", "prompt_cache_breakpoint": {"mode": "explicit"}} + ], + }, + { + "role": "user", + "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit"}}], + }, + ], + optional_params={}, + litellm_params={}, + ) + + assert request["system"] == [{"text": "You are terse."}, {"cachePoint": {"type": "default"}}] + assert request["messages"] == [ + {"role": "user", "content": [{"text": "Hello"}, {"cachePoint": {"type": "default"}}]} + ] + + +def test_converse_keeps_the_first_and_last_prompt_cache_breakpoints_within_the_cap(): + messages: Final = [ + { + "role": "user", + "content": [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(5) + ], + } + ] + + def transform(litellm_params: dict) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=copy.deepcopy(messages), + optional_params={}, + litellm_params=litellm_params, + ) + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="at most 4 cache breakpoints"): + transform({}) + + cache_point: Final = {"cachePoint": {"type": "default"}} + assert transform({"drop_params": True})["messages"] == [ + { + "role": "user", + "content": [ + {"text": "part 0"}, + cache_point, + {"text": "part 1"}, + cache_point, + {"text": "part 2"}, + {"text": "part 3"}, + cache_point, + {"text": "part 4"}, + cache_point, + ], + } + ] + + +_CONVERSE_CAP_TOOL: Final[dict[str, object]] = { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object", "properties": {}}}, +} + + +@pytest.mark.parametrize( + "tool_cache_params", + [ + {"tools": [{**_CONVERSE_CAP_TOOL, "cache_control": {"type": "ephemeral"}}]}, + { + "tools": [_CONVERSE_CAP_TOOL], + "cache_control_injection_points": [{"location": "tool_config", "control": {"type": "ephemeral"}}], + }, + ], + ids=["client_tool_cache_control", "tool_config_injection_point"], +) +def test_converse_reserves_the_tool_cache_point_within_the_cap(tool_cache_params: dict[str, object]): + messages: Final = [ + { + "role": "user", + "content": [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(4) + ], + } + ] + + def transform(litellm_params: dict) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=copy.deepcopy(messages), + optional_params=copy.deepcopy(tool_cache_params), + litellm_params=litellm_params, + ) + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="this request has 5"): + transform({}) + + request: Final = transform({"drop_params": True}) + cache_point: Final = {"cachePoint": {"type": "default"}} + assert request["messages"][0]["content"] == [ + {"text": "part 0"}, + cache_point, + {"text": "part 1"}, + cache_point, + {"text": "part 2"}, + {"text": "part 3"}, + cache_point, + ] + assert request["toolConfig"]["tools"][-1] == cache_point + + + +@pytest.mark.parametrize( + "unsent_tool_params", + [ + { + "tools": [ + _CONVERSE_CAP_TOOL, + {"type": "tool_search_tool_regex_20251119", "name": "find", "cache_control": {"type": "ephemeral"}}, + ] + }, + {"tools": [_CONVERSE_CAP_TOOL, {"type": "web_search", "cache_control": {"type": "ephemeral"}}]}, + { + "tools": [{"type": "web_search"}], + "cache_control_injection_points": [{"location": "tool_config", "control": {"type": "ephemeral"}}], + }, + ], + ids=["tool_search_tool", "built_in_tool", "tool_config_point_with_no_tool_sent"], +) +@pytest.mark.parametrize("litellm_params", [{}, {"drop_params": True}], ids=["strict", "drop_params"]) +def test_converse_leaves_the_cap_to_markers_when_a_marked_tool_is_not_sent( + unsent_tool_params: dict[str, object], litellm_params: dict[str, object] +): + request: Final = litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(4) + ], + } + ], + optional_params=copy.deepcopy(unsent_tool_params), + litellm_params=dict(litellm_params), + ) + + cache_point: Final = {"cachePoint": {"type": "default"}} + assert request["messages"][0]["content"] == [ + {"text": "part 0"}, + cache_point, + {"text": "part 1"}, + cache_point, + {"text": "part 2"}, + cache_point, + {"text": "part 3"}, + cache_point, + ] + assert cache_point not in request.get("toolConfig", {}).get("tools", []) + +def _converse_tool_turn(*result_parts: dict[str, object]) -> list[dict[str, object]]: + return [ + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": list(result_parts)}, + ] + + +def _marked_text_parts(count: int) -> list[dict[str, object]]: + return [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(count) + ] + + +def _marked_tool_result_parts() -> list[dict[str, object]]: + return [ + {"type": "text", "text": f"result {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(2) + ] + + +def test_converse_counts_one_cache_point_for_a_tool_result_with_several_marks(): + request: Final = litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=[ + {"role": "user", "content": _marked_text_parts(3)}, + *_converse_tool_turn(*_marked_tool_result_parts()), + ], + optional_params={"tools": [_CONVERSE_CAP_TOOL]}, + litellm_params={}, + ) + + assert len(_collect_cache_points(request["messages"])) == 4 + + +def test_converse_drops_marks_until_a_merged_tool_result_fits_the_cap(): + messages: Final = [ + {"role": "user", "content": _marked_text_parts(4)}, + *_converse_tool_turn(*_marked_tool_result_parts()), + ] + + def transform(litellm_params: dict) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=copy.deepcopy(messages), + optional_params={"tools": [_CONVERSE_CAP_TOOL]}, + litellm_params=litellm_params, + ) + + with pytest.raises(litellm.UnsupportedParamsError, match="this request has 5"): + transform({}) + + request: Final = transform({"drop_params": True}) + cache_point: Final = {"cachePoint": {"type": "default"}} + assert request["messages"][0]["content"] == [ + {"text": "part 0"}, + cache_point, + {"text": "part 1"}, + cache_point, + {"text": "part 2"}, + cache_point, + {"text": "part 3"}, + ] + assert len(_collect_cache_points(request["messages"][1:])) == 1 + + +def test_converse_drops_prompt_cache_breakpoint_from_a_nova_tool_result(): + messages: Final = [ + { + "role": "system", + "content": [{"type": "text", "text": "You are terse.", "prompt_cache_breakpoint": {"mode": "explicit"}}], + }, + {"role": "user", "content": "Look it up."}, + *_converse_tool_turn(*_marked_tool_result_parts()), + ] + + def transform(litellm_params: dict) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model="us.amazon.nova-lite-v1:0", + messages=copy.deepcopy(messages), + optional_params={"tools": [_CONVERSE_CAP_TOOL]}, + litellm_params=litellm_params, + ) + + with pytest.raises(litellm.UnsupportedParamsError, match="tool result"): + transform({}) + + request: Final = transform({"drop_params": True}) + assert request["system"] == [{"text": "You are terse."}, {"cachePoint": {"type": "default"}}] + assert _collect_cache_points(request["messages"]) == [] + + +def test_converse_translates_prompt_cache_breakpoint_on_an_unmapped_arn_only_when_flagged(local_model_cost_map): + from litellm import Router + + flagged_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-flagged" + unflagged_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/mistral" + Router( + model_list=[ + { + "model_name": "claude-profile", + "litellm_params": {"model": f"bedrock/{flagged_arn}", "aws_region_name": "us-east-1"}, + "model_info": {"supports_prompt_cache_breakpoint": True}, + }, + { + "model_name": "mistral-profile", + "litellm_params": {"model": f"bedrock/{unflagged_arn}", "aws_region_name": "us-east-1"}, + }, + ] + ) + + def user_content(model: str, ttl: dict) -> list: + return litellm.AmazonConverseConfig()._transform_request( + model=model, + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}}], + } + ], + optional_params={}, + litellm_params={}, + )["messages"][0]["content"] + + assert user_content(flagged_arn, {}) == [{"text": "Hello"}, {"cachePoint": {"type": "default"}}] + assert user_content(unflagged_arn, {}) == [{"text": "Hello"}] + assert user_content(unflagged_arn, {"ttl": "30m"}) == [{"text": "Hello"}] + + + +def test_converse_prompt_cache_breakpoint_follows_the_arn_flag_over_its_base_model(local_model_cost_map): + from litellm import Router + + opted_in_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-next" + opted_out_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-opted-out" + Router( + model_list=[ + { + "model_name": "opted-in", + "litellm_params": {"model": f"bedrock/{opted_in_arn}", "aws_region_name": "us-east-1"}, + "model_info": {"supports_prompt_cache_breakpoint": True}, + }, + { + "model_name": "opted-out", + "litellm_params": {"model": f"bedrock/{opted_out_arn}", "aws_region_name": "us-east-1"}, + "model_info": {"supports_prompt_cache_breakpoint": False}, + }, + ] + ) + + def user_content(model: str, base_model: str, ttl: dict) -> list: + return litellm.AmazonConverseConfig()._transform_request( + model=model, + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}}], + } + ], + optional_params={}, + litellm_params={"base_model": base_model}, + )["messages"][0]["content"] + + assert user_content(opted_in_arn, "anthropic.claude-unmapped-v1:0", {}) == [ + {"text": "Hello"}, + {"cachePoint": {"type": "default"}}, + ] + assert user_content(opted_out_arn, "anthropic.claude-sonnet-5-5", {"ttl": "30m"}) == [{"text": "Hello"}] + + +def test_converse_translates_prompt_cache_breakpoint_only_for_families_that_take_cache_points(local_model_cost_map): + litellm.register_model( + {"zai.glm-next-v1:0": {"litellm_provider": "bedrock_converse", "mode": "chat", "supports_prompt_caching": True}} + ) + + def user_content(model: str, mark: dict) -> list: + return litellm.AmazonConverseConfig()._transform_request( + model=model, + messages=[{"role": "user", "content": [{"type": "text", "text": "Hello", **mark}]}], + optional_params={}, + litellm_params={}, + )["messages"][0]["content"] + + marker: Final = {"prompt_cache_breakpoint": {"mode": "explicit"}} + client_mark: Final = {"cache_control": {"type": "ephemeral"}} + cache_point: Final = {"cachePoint": {"type": "default"}} + assert user_content("zai.glm-next-v1:0", marker) == [{"text": "Hello"}] + assert user_content("zai.glm-next-v1:0", client_mark) == [{"text": "Hello"}, cache_point] + assert user_content("us.anthropic.claude-next-v1:0", marker) == [{"text": "Hello"}, cache_point] + assert user_content("us.amazon.nova-next-v1:0", marker) == [{"text": "Hello"}, cache_point] + + +def test_converse_prompt_cache_breakpoint_on_guarded_user_text_keeps_its_cache_point(local_model_cost_map): + guardrail: Final = {"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "DRAFT"}} + + def request(system_marks: int) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=[ + *( + { + "role": "system", + "content": [{"type": "text", "text": f"Rule {i}", "cache_control": {"type": "ephemeral"}}], + } + for i in range(system_marks) + ), + { + "role": "user", + "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit"}}], + }, + ], + optional_params=dict(guardrail), + litellm_params={}, + ) + + assert request(0)["messages"][0]["content"] == [ + {"guardContent": {"text": {"text": "Hello"}}}, + {"cachePoint": {"type": "default"}}, + ] + with pytest.raises(litellm.UnsupportedParamsError): + request(4) + + client_marked: Final = litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=[ + *( + { + "role": "system", + "content": [{"type": "text", "text": f"Rule {i}", "cache_control": {"type": "ephemeral"}}], + } + for i in range(4) + ), + {"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}]}, + ], + optional_params=dict(guardrail), + litellm_params={}, + ) + assert sum("cachePoint" in block for block in client_marked["system"]) == 4 + assert client_marked["messages"][0]["content"] == [{"guardContent": {"text": {"text": "Hello"}}}] + + +def test_converse_guarded_user_text_keeps_no_cache_point_on_a_model_not_known_to_take_one(local_model_cost_map): + request: Final = litellm.AmazonConverseConfig()._transform_request( + model="arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/unmapped1", + messages=[{"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}]}], + optional_params={"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "DRAFT"}}, + litellm_params={}, + ) + + assert request["messages"][0]["content"] == [{"guardContent": {"text": {"text": "Hello"}}}] + + +def _converse_request_with_30m_breakpoint(model: str, litellm_params: dict) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model=model, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}} + ], + } + ], + optional_params={}, + litellm_params=litellm_params, + ) + + +def test_converse_rejects_30m_breakpoint_ttl_only_where_a_cache_point_is_written(): + claude: Final = "anthropic.claude-sonnet-5-5" + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"): + _converse_request_with_30m_breakpoint(claude, {}) + + assert _converse_request_with_30m_breakpoint(claude, {"drop_params": True})["messages"] == [ + {"role": "user", "content": [{"text": "Hello"}, {"cachePoint": {"type": "default"}}]} + ] + assert _converse_request_with_30m_breakpoint("meta.llama3-1-8b-instruct-v1:0", {})["messages"] == [ + {"role": "user", "content": [{"text": "Hello"}]} + ] + + @pytest.mark.parametrize( "model, expected_supports_tool_call", [ diff --git a/tests/unit/llms/opencode/chat/test_transformation.py b/tests/unit/llms/opencode/chat/test_transformation.py index 91361a3d81d..08c80db294c 100644 --- a/tests/unit/llms/opencode/chat/test_transformation.py +++ b/tests/unit/llms/opencode/chat/test_transformation.py @@ -14,6 +14,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.opencode.chat.transformation import OpenCodeGoMessagesChatConfig from litellm.llms.opencode.common_utils import go_headers @@ -591,3 +592,22 @@ def test_stream_chunk_builder_reads_list_cost_from_dict_chunks(): assert assembled is not None assert assembled._hidden_params["response_cost"] == GO_REPORTED_USAGE["opencode"]["list_cost"] + + +@pytest.mark.parametrize(("model", "cache_marks"), [("qwen3.8-flash", 0), ("minimax-m3", 0), ("claude-haiku-5-5", 4)]) +def test_go_messages_route_carries_prompt_cache_breakpoint_to_cache_control_only_for_claude(model: str, cache_marks: int): + marked_notes: Final = [ + {"type": "text", "text": f"Note {n}", "prompt_cache_breakpoint": {"mode": "explicit"}} for n in range(4) + ] + + request: Final = OpenCodeGoMessagesChatConfig().transform_request( + model=model, + messages=[{"role": "user", "content": [*marked_notes, {"type": "text", "text": "Answer in one line."}]}], + optional_params={"max_tokens": 64}, + litellm_params={}, + headers={}, + ) + + wire: Final = json.dumps(request) + assert wire.count('"cache_control"') == cache_marks + assert "prompt_cache_breakpoint" not in wire diff --git a/tests/unit/router_utils/test_prompt_caching_cache.py b/tests/unit/router_utils/test_prompt_caching_cache.py index ce8f04b8c54..1500d7433d2 100644 --- a/tests/unit/router_utils/test_prompt_caching_cache.py +++ b/tests/unit/router_utils/test_prompt_caching_cache.py @@ -1,4 +1,5 @@ import asyncio +from collections.abc import Mapping from typing import Final, cast import pytest @@ -377,6 +378,32 @@ def test_extract_cacheable_prefix_mixed_string_and_list_content(): assert isinstance(result[2]["content"], list) +def test_prompt_cache_breakpoint_marks_the_same_cacheable_prefix_as_cache_control(): + def _messages(marker: Mapping[str, Mapping[str, str]]) -> list[AllMessageValues]: + return cast( # cast-ok: the text part type does not declare the prompt_cache_breakpoint key + list[AllMessageValues], + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Here is the full text of a complex legal agreement" * 400, **marker} + ], + }, + {"role": "user", "content": "what are the key terms?"}, + ], + ) + + marked_key: Final = PromptCachingCache.get_prompt_caching_cache_key( + _messages({"prompt_cache_breakpoint": {"mode": "explicit"}}), None + ) + + assert marked_key is not None + assert marked_key == PromptCachingCache.get_prompt_caching_cache_key( + _messages({"cache_control": {"type": "ephemeral"}}), None + ) + assert PromptCachingCache.get_prompt_caching_cache_key(_messages({}), None) is None + + @pytest.mark.asyncio async def test_same_cacheable_prefix_routes_to_same_deployment() -> None: from litellm import Router From 1cc7da6274b861e5841c4ddbabf68c2ac8fcf002 Mon Sep 17 00:00:00 2001 From: tessellark <349031000+tessellark@users.noreply.github.com> Date: Tue, 3 Nov 2026 08:14:04 -0800 Subject: [PATCH 004/122] fix(proxy): list slash-less wildcard groups like claude-* on /v1/models and /model/info (#49832) * fix(proxy): list slash-less wildcard groups like claude-* on /v1/models and /model/info * fix(proxy): expand team-owned wildcard rows on /model/info through their public name * fix(proxy): keep listing wildcard names wider than their deployment and skip the Azure placeholder A group like *gemini-3* over gemini/gemini-3* lists the catalog ids the name matches and routes back, as main did on /model/info, now on /v1/models too. Wildcard listings no longer list the Azure-LLM placeholder. Team-owned wildcard rows on /model/info go back to main's behavior. * fix(proxy): keep Azure listings on slashed wildcards and list bare deployments through custom_llm_provider get_provider_models returns main's list again, and only slash-less names skip the Azure-LLM catalog placeholder. A slash-less name over a bare wildcard deployment now lists the catalog of its custom_llm_provider. * test(proxy): pin custom_llm_provider guards and slashed wider-name listing * fix(proxy): list bare ids for wider names over a custom_llm_provider deployment --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/auth/model_checks.py | 113 +++++++++++----- litellm/utils.py | 5 +- tests/unit/proxy/auth/test_model_checks.py | 145 ++++++++++++++++++++- 3 files changed, 227 insertions(+), 36 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index e41c9354a8a..73edcd66bd7 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -23,7 +23,12 @@ from litellm.router import Router from litellm.router_api import fallback_chain from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params from litellm.types.utils import LlmProviders -from litellm.utils import ProviderConfigManager, get_valid_models, potential_model_names_in_lookup_order +from litellm.utils import ( + AZURE_MODEL_PLACEHOLDER, + ProviderConfigManager, + get_valid_models, + potential_model_names_in_lookup_order, +) LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders) _PROVIDERS_SENDING_THE_DEPLOYMENT_PATH_AS_THE_MODEL_ID: Final = frozenset({LlmProviders.TOGETHER_AI.value}) @@ -375,6 +380,10 @@ def _ids_routing_to_catalog_models( catalog_ids: Sequence[str], routes_to_model: Callable[[str, str], bool], ) -> tuple[str, ...]: + if name_pattern.count("*") > deployment_pattern.count("*"): + return _catalog_ids_a_wider_name_matches( + name_pattern, deployment_pattern, provider, catalog_ids, routes_to_model + ) if name_pattern.count("*") != deployment_pattern.count("*") or _lists_name_fragments( name_pattern, deployment_pattern ): @@ -419,6 +428,23 @@ def _ids_routing_to_catalog_models( ) +def _catalog_ids_a_wider_name_matches( + name_pattern: str, + deployment_pattern: str, + provider: str, + catalog_ids: Sequence[str], + routes_to_model: Callable[[str, str], bool], +) -> tuple[str, ...]: + name_regex: Final = _wildcard_name_regex(name_pattern) + return tuple( + catalog_id + for catalog_id in _ids_routing_to_catalog_models( + deployment_pattern, deployment_pattern, provider, catalog_ids, routes_to_model + ) + if name_regex.fullmatch(catalog_id) + ) + + def _model_info_field(deployment: Mapping[str, object], field: str) -> object: model_info: Final = deployment.get("model_info") return model_info.get(field) if is_str_keyed_mapping(model_info) else None @@ -456,11 +482,12 @@ def _routes_to_model( return lambda listed_id, model: routed_models is None or model in routed_models(listed_id, team_id) -def _deployment_pattern_naming_listed_ids(wildcard_suffix: str, litellm_params: LiteLLM_Params | None) -> str | None: +def _deployment_pattern_naming_listed_ids(wildcard_model: str, litellm_params: LiteLLM_Params | None) -> str | None: if litellm_params is None or "/" not in litellm_params.model or not _check_wildcard_routing(litellm_params.model): return None + name_suffix: Final = wildcard_model.split("/", 1)[-1] deployment_suffix: Final = litellm_params.model.partition("/")[2] - return litellm_params.model if wildcard_suffix != "*" or deployment_suffix != "*" else None + return litellm_params.model if name_suffix != "*" or deployment_suffix != "*" else None def get_known_models_from_wildcard( @@ -468,6 +495,43 @@ def get_known_models_from_wildcard( litellm_params: LiteLLM_Params | None = None, routes_to_model: Callable[[str, str], bool] = lambda _listed_id, _model: True, ) -> list[str]: + provider_id_prefix: Final = _custom_llm_provider_prefix(wildcard_model, litellm_params) + if litellm_params is None or provider_id_prefix is None: + return list(_known_models_from_wildcard(wildcard_model, litellm_params, routes_to_model)) + return [ + listed_id.removeprefix(provider_id_prefix) + for listed_id in _known_models_from_wildcard( + wildcard_model, + litellm_params.model_copy(update={"model": f"{provider_id_prefix}{litellm_params.model}"}), + lambda listed_id, model: routes_to_model( + listed_id.removeprefix(provider_id_prefix), model.removeprefix(provider_id_prefix) + ), + ) + ] + + +def _custom_llm_provider_prefix(wildcard_model: str, litellm_params: LiteLLM_Params | None) -> str | None: + if ( + litellm_params is None + or not litellm_params.custom_llm_provider + or "/" in wildcard_model + or wildcard_model == "*" + or "/" in litellm_params.model + or not _check_wildcard_routing(litellm_params.model) + ): + return None + return f"{litellm_params.custom_llm_provider}/" + + +def _without_azure_placeholder(catalog_ids: Sequence[str]) -> tuple[str, ...]: + return tuple(catalog_id for catalog_id in catalog_ids if catalog_id != AZURE_MODEL_PLACEHOLDER) + + +def _known_models_from_wildcard( + wildcard_model: str, + litellm_params: LiteLLM_Params | None, + routes_to_model: Callable[[str, str], bool], +) -> tuple[str, ...]: wildcard_model_to_expand: Final = ( litellm_params.model if wildcard_model == "*" @@ -476,15 +540,14 @@ def get_known_models_from_wildcard( and "/" in litellm_params.model else wildcard_model ) - try: - wildcard_provider_prefix, wildcard_suffix = wildcard_model_to_expand.split("/", 1) - except ValueError: # safely fail - return [] deployment_pattern_naming_listed_ids: Final = ( wildcard_model_to_expand if wildcard_model_to_expand != wildcard_model - else _deployment_pattern_naming_listed_ids(wildcard_suffix, litellm_params) + else _deployment_pattern_naming_listed_ids(wildcard_model, litellm_params) ) + wildcard_provider_prefix, has_provider_prefix, wildcard_suffix = wildcard_model_to_expand.partition("/") + if not has_provider_prefix and deployment_pattern_naming_listed_ids is None: + return () # Use provider from litellm_params when available, otherwise from wildcard prefix # (e.g., "openai" from "openai/*" - needed for BYOK where wildcard isn't in router) @@ -501,16 +564,14 @@ def get_known_models_from_wildcard( wildcard_models = get_provider_models(provider=provider, litellm_params=litellm_params) if wildcard_models is None: - return [] + return () if deployment_pattern_naming_listed_ids is not None: - return list( - _ids_routing_to_catalog_models( - name_pattern=wildcard_model, - deployment_pattern=deployment_pattern_naming_listed_ids, - provider=provider, - catalog_ids=wildcard_models, - routes_to_model=routes_to_model, - ) + return _ids_routing_to_catalog_models( + name_pattern=wildcard_model, + deployment_pattern=deployment_pattern_naming_listed_ids, + provider=provider, + catalog_ids=wildcard_models if has_provider_prefix else _without_azure_placeholder(wildcard_models), + routes_to_model=routes_to_model, ) if wildcard_suffix != "*": model_prefix: Final = wildcard_suffix.replace("*", "") @@ -526,7 +587,7 @@ def get_known_models_from_wildcard( # add model prefix to wildcard models wildcard_models = [f"{model_prefix}{model.removeprefix(provider_id_prefix)}" for model in wildcard_models] - return list( + return tuple( dict.fromkeys(_under_wildcard_prefix(model, wildcard_provider_prefix, provider) for model in wildcard_models) ) @@ -564,16 +625,6 @@ def _names_no_deployment_serves_by_name( return tuple(name for name in names if not deployed_by_name(name, team_id)) -def _wildcard_pattern_to_expand(model_name: str, litellm_model: str) -> str | None: - if not _check_wildcard_routing(model_name): - return None - if "/" in model_name or model_name == "*": - return model_name - if _check_wildcard_routing(litellm_model): - return litellm_model - return model_name - - def _can_expand_to_an_allowed_name(wildcard_pattern: str, allowed_model_names: Collection[str] | None) -> bool: if allowed_model_names is None: return True @@ -601,10 +652,10 @@ def expand_wildcard_deployments_for_model_info( raw_params = deployment.get("litellm_params") litellm_params_dict: dict[str, object] = raw_params if isinstance(raw_params, dict) else {} litellm_model = str(litellm_params_dict.get("model") or "") - if (wildcard_pattern := _wildcard_pattern_to_expand(model_name, litellm_model)) is None: + if not _check_wildcard_routing(model_name): expanded.append(deployment) continue - if not _can_expand_to_an_allowed_name(wildcard_pattern, allowed_model_names): + if not _can_expand_to_an_allowed_name(model_name, allowed_model_names): continue try: @@ -613,7 +664,7 @@ def expand_wildcard_deployments_for_model_info( expanded.append(deployment) continue expanded_names = get_known_models_from_wildcard( - wildcard_model=wildcard_pattern, + wildcard_model=model_name, litellm_params=litellm_params, routes_to_model=_routes_to_model(routed_models, _deployment_team_id(deployment)), ) diff --git a/litellm/utils.py b/litellm/utils.py index d881289deed..e003f0a3987 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9069,6 +9069,9 @@ def bundled_provider_models(provider: str) -> tuple[str, ...]: return tuple(model for model in bundled if not is_hidden_as_retired(cost_map.get(model, {}))) +AZURE_MODEL_PLACEHOLDER: Final = "Azure-LLM" + + def get_valid_models( check_provider_endpoint: bool | None = None, custom_llm_provider: str | None = None, @@ -9124,7 +9127,7 @@ def get_valid_models( continue if provider == "azure": - valid_models.append("Azure-LLM") + valid_models.append(AZURE_MODEL_PLACEHOLDER) elif provider_config is not None and check_provider_endpoint and provider is not None: valid_models.extend( _get_valid_models_from_provider_api( diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index eef7e51982a..bc9366aa607 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -19,6 +19,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.x509.oid import NameOID from litellm import Router +from litellm.utils import AZURE_MODEL_PLACEHOLDER def test_get_team_models_for_all_models_and_team_only_models(): @@ -1105,13 +1106,13 @@ def test_expand_wildcard_invalid_litellm_params_passthrough(): @pytest.mark.parametrize( ("allowed_model_names", "expected_names", "expected_listings"), [ - pytest.param(None, ["hosted_vllm/qwen2.5"], 1, id="unscoped"), - pytest.param(["hosted_vllm/qwen2.5"], ["hosted_vllm/qwen2.5"], 1, id="granted-a-listed-id"), - pytest.param(["qwen-open"], [], 0, id="granted-a-name-the-alias-matches"), + pytest.param(None, ["qwen-qwen2.5"], 1, id="unscoped"), + pytest.param(["qwen-qwen2.5"], ["qwen-qwen2.5"], 1, id="granted-a-listed-name"), + pytest.param(["hosted_vllm/qwen2.5"], [], 0, id="granted-only-the-provider-id"), ], ) @respx.mock -def test_expand_wildcard_alias_without_a_slash_lists_and_filters_by_its_provider_pattern( +def test_expand_wildcard_alias_without_a_slash_lists_and_filters_by_its_own_pattern( monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest, allowed_model_names: list[str] | None, @@ -2099,6 +2100,34 @@ _ROUTE_CF_IDS: Final = {_ROUTE_CF_ID, _ROUTE_CF_SUFFIXED_ID} pytest.param( "*", "openai/*", "openai/gpt-partial-route-9*", set(), id="provider-prefixed-key-filter-over-catch-all" ), + pytest.param( + "gpt-partial-route-*", + "openai/gpt-partial-route-*", + None, + {f"openai/{_ROUTE_OPENAI_ID}"}, + id="slashless-name-repeating-the-deployment-partial-word", + ), + pytest.param("team-*", "cloudflare/@cf/*", None, _ROUTE_CF_IDS, id="slashless-alias-over-partial-deployment"), + pytest.param( + "team-*", "openai/*", None, {f"openai/{_ROUTE_OPENAI_ID}"}, id="slashless-alias-over-provider-catch-all" + ), + pytest.param( + "team-*", "cloudflare/@cf/*/*", None, set(), id="slashless-alias-with-fewer-wildcards-than-its-deployment" + ), + pytest.param( + "*partial-route-9*", + "openai/gpt-partial-route-9*", + None, + {f"openai/{_ROUTE_OPENAI_ID}"}, + id="slashless-name-with-more-wildcards-than-its-deployment", + ), + pytest.param( + "gemini/*partial-route-9*", + "gemini/gemini-*", + None, + {_ROUTE_GEMINI_ID}, + id="slashed-name-with-more-wildcards-than-its-deployment", + ), pytest.param("*", "cloudflare/@cf/*", None, _ROUTE_CF_IDS, id="catch-all-over-partial-deployment"), pytest.param("*", "gemini/gemini-partial-route-9*", None, set(), id="catch-all-over-mid-word-deployment"), pytest.param("*", "openai/responses/gpt-partial-route-9*", None, set(), id="catch-all-over-mid-word-route"), @@ -2176,6 +2205,114 @@ def test_every_listed_partial_wildcard_id_routes_to_the_catalog_model_it_names( assert listed_model_info_names == sorted(reachable) +@pytest.mark.parametrize( + ("model_name", "expected_v1_models"), + [ + pytest.param("azure/*", [f"azure/{AZURE_MODEL_PLACEHOLDER}"], id="provider-wildcard-lists-as-before"), + pytest.param("*", [AZURE_MODEL_PLACEHOLDER], id="catch-all-lists-as-before"), + pytest.param("my-*", [], id="slashless-alias-skips-the-placeholder"), + ], +) +def test_only_slashless_azure_wildcards_skip_the_catalog_placeholder( + model_name: str, expected_v1_models: list[str] +) -> None: + from litellm import Router + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list + + router: Final = Router(model_list=[{"model_name": model_name, "litellm_params": {"model": "azure/*"}}]) + + v1_models: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=[model_name], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + model_info_names: Final = [ + str(row["model_name"]) + for row in expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.get_model_list() or []], router + ) + ] + + assert v1_models == expected_v1_models + assert model_info_names == (expected_v1_models or [model_name]) + + +@pytest.mark.parametrize( + ("model_name", "deployment_model"), + [ + pytest.param("gpt-*", "gpt-*", id="same-wildcards-as-its-deployment"), + pytest.param("*gpt-5*", "gpt-5*", id="more-wildcards-than-its-deployment"), + ], +) +def test_slashless_wildcard_over_a_bare_deployment_lists_its_custom_llm_provider_catalog( + model_name: str, deployment_model: str +) -> None: + from fnmatch import fnmatchcase + + from litellm import Router + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list + from litellm.utils import get_valid_models + + router: Final = Router( + model_list=[ + { + "model_name": model_name, + "litellm_params": {"model": deployment_model, "custom_llm_provider": "openai", "api_key": "sk-test"}, + } + ] + ) + + v1_models: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=[model_name], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + model_info_rows: Final = expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.get_model_list() or []], router + ) + openai_ids: Final = frozenset(model.removeprefix("openai/") for model in get_valid_models(custom_llm_provider="openai")) + routed_models: Final = { + listed_id: [deployment["litellm_params"]["model"] for deployment in router.wildcard_route_deployments(listed_id, None)] + for listed_id in v1_models + } + + assert v1_models + assert [ + listed_id for listed_id in v1_models if listed_id not in openai_ids or not fnmatchcase(listed_id, model_name) + ] == [] + assert {listed_id: models for listed_id, models in routed_models.items() if models != [listed_id]} == {} + assert sorted(str(row["model_name"]) for row in model_info_rows) == sorted(v1_models) + assert [row for row in model_info_rows if row["litellm_params"]["model"] != row["model_name"]] == [] + + +@pytest.mark.parametrize( + ("model_name", "deployment_model"), + [ + pytest.param("*", "*", id="catch-all-over-a-bare-catch-all"), + pytest.param("foo/gpt-*", "gpt-*", id="slashed-name-over-a-bare-deployment"), + pytest.param("gpt-*", "openai/gpt-*", id="slashless-name-over-a-provider-prefixed-deployment"), + ], +) +def test_custom_llm_provider_changes_no_listing_outside_slashless_names_over_bare_deployments( + model_name: str, deployment_model: str +) -> None: + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + with_provider: Final = get_known_models_from_wildcard( + model_name, LiteLLM_Params(model=deployment_model, custom_llm_provider="openai") + ) + without_provider: Final = get_known_models_from_wildcard(model_name, LiteLLM_Params(model=deployment_model)) + + assert with_provider == without_provider + + def test_inference_provider_route_lists_every_id_the_plain_provider_wildcard_lists() -> None: from litellm.proxy.auth.model_checks import get_known_models_from_wildcard from litellm.types.router import LiteLLM_Params From 36cd03f10d804c4689edd7b848724df387ab417c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 09:10:58 -0800 Subject: [PATCH 005/122] fix(gemini): lower the gemini-3.5-flash-lite batch cache price and date the gemini-2.5-pro and 2.5-flash shutdown (#49871) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 4 +++- model_prices_and_context_window.json | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index be93d6fcd9b..0f895c63ca8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -31421,6 +31421,7 @@ "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "deprecation_date": "2026-12-01", "input_cost_per_audio_token": 1e-06, "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 3e-07, @@ -32004,6 +32005,7 @@ "output_cost_per_token_flex": 5e-06, "supports_url_context": true, "cache_read_input_token_cost_above_200k_tokens_batches": 2.5e-07, + "deprecation_date": "2026-12-01", "input_cost_per_token_above_200k_tokens_batches": 1.25e-06, "output_cost_per_token_above_200k_tokens_batches": 7.5e-06 }, @@ -32107,7 +32109,7 @@ }, "gemini/gemini-3.5-flash-lite": { "cache_read_input_token_cost": 3e-08, - "cache_read_input_token_cost_batches": 2e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, "input_cost_per_token": 3e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index be93d6fcd9b..0f895c63ca8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -31421,6 +31421,7 @@ "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "deprecation_date": "2026-12-01", "input_cost_per_audio_token": 1e-06, "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 3e-07, @@ -32004,6 +32005,7 @@ "output_cost_per_token_flex": 5e-06, "supports_url_context": true, "cache_read_input_token_cost_above_200k_tokens_batches": 2.5e-07, + "deprecation_date": "2026-12-01", "input_cost_per_token_above_200k_tokens_batches": 1.25e-06, "output_cost_per_token_above_200k_tokens_batches": 7.5e-06 }, @@ -32107,7 +32109,7 @@ }, "gemini/gemini-3.5-flash-lite": { "cache_read_input_token_cost": 3e-08, - "cache_read_input_token_cost_batches": 2e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, "input_cost_per_token": 3e-07, From add7dea9aa87eb5e845b396ac892c5e7f649e29c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:53:49 +0000 Subject: [PATCH 006/122] fix(vertex-ai): drop batch prices from gemini-robotics-er-2-preview (#49895) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 12 ++---------- model_prices_and_context_window.json | 12 ++---------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0f895c63ca8..974f1bb0c8b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76956,23 +76956,19 @@ }, "vertex_ai/gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, - "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, - "output_cost_per_token_batches": 2.5e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", - "/v1/completions", - "/v1/batch" + "/v1/completions" ], "supported_modalities": [ "text", @@ -77002,23 +76998,19 @@ }, "gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, - "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, - "output_cost_per_token_batches": 2.5e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", - "/v1/completions", - "/v1/batch" + "/v1/completions" ], "supported_modalities": [ "text", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0f895c63ca8..974f1bb0c8b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76956,23 +76956,19 @@ }, "vertex_ai/gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, - "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, - "output_cost_per_token_batches": 2.5e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", - "/v1/completions", - "/v1/batch" + "/v1/completions" ], "supported_modalities": [ "text", @@ -77002,23 +76998,19 @@ }, "gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, - "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, - "output_cost_per_token_batches": 2.5e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", - "/v1/completions", - "/v1/batch" + "/v1/completions" ], "supported_modalities": [ "text", From 0aa7079f042f9611272b2a08f50b405eec7975af Mon Sep 17 00:00:00 2001 From: shrey Date: Tue, 3 Nov 2026 10:51:09 -0800 Subject: [PATCH 007/122] fix(spend): read only the newest rows when paging sessions (LIT-8513) (#48886) --- litellm/proxy/spend_tracking/session_page.py | 221 +++++++++ .../spend_management_endpoints.py | 210 +++----- .../spend/test_session_grouped_logs_pages.py | 455 ++++++++++++++++++ .../proxy/spend_tracking/test_session_page.py | 327 +++++++++++++ .../test_spend_management_endpoints.py | 103 ++-- .../test_spend_query_optimization.py | 40 +- 6 files changed, 1117 insertions(+), 239 deletions(-) create mode 100644 litellm/proxy/spend_tracking/session_page.py create mode 100644 tests/integration/spend/test_session_grouped_logs_pages.py create mode 100644 tests/unit/proxy/spend_tracking/test_session_page.py diff --git a/litellm/proxy/spend_tracking/session_page.py b/litellm/proxy/spend_tracking/session_page.py new file mode 100644 index 00000000000..810afbfa7b7 --- /dev/null +++ b/litellm/proxy/spend_tracking/session_page.py @@ -0,0 +1,221 @@ +"""Which sessions a logs page shows and how many match, read from the newest rows instead of every row. Reads grow +until one settles both, so the result is what grouping every row would give.""" + +import math +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass +from typing import Final, Protocol, TypeAlias + +from typing_extensions import ReadOnly, TypedDict + +MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')" +AGENT_CALL_TYPE_SQL: Final = "'asend_message'" +SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)" +SESSION_REPRESENTATIVE_ORDER_SQL: Final = ( + f"(call_type = {AGENT_CALL_TYPE_SQL}) DESC, " + f'CASE WHEN call_type = {AGENT_CALL_TYPE_SQL} THEN "endTime" END DESC NULLS LAST, ' + f'call_type IN {MCP_CALL_TYPES_SQL}, "startTime" DESC, request_id' +) + + +class _ReadRow(TypedDict): + session_key: ReadOnly[str | None] + api_key: ReadOnly[str | None] + last_activity: ReadOnly[str | None] + settled: ReadOnly[bool | None] + rows_read: ReadOnly[int] + session_count: ReadOnly[int] + + +class _SupportsQueryRaw(Protocol): + async def query_raw(self, query: str, *args: object) -> Sequence[_ReadRow]: ... + + +@dataclass(frozen=True, slots=True) +class Session: + key: str + api_key: str + last_activity: str + + +@dataclass(frozen=True, slots=True) +class SessionPage: + sessions: tuple[Session, ...] + has_more: bool + next_cursor: str | None + total: int + total_is_capped: bool + + +@dataclass(frozen=True, slots=True) +class _Request: + cursor: tuple[str, str, str] | None + offset: int + page_limit: int + sort_desc: bool + status: str | None + cap: int + + def first_read_size(self) -> int | None: + """Every row when oldest first, or when filtered by outcome, which any of a session's rows can decide. Otherwise + the fewest that could settle a page: a session past the cap and an older row.""" + return self.cap + 2 if self.sort_desc and self.status is None else None + + +@dataclass(frozen=True, slots=True) +class _Read: + candidates: tuple[Session, ...] + last_settled: bool + rows_read: int + session_count: int + saw_every_row: bool + + def settles(self, request: _Request) -> bool: + page_settled: Final = ( + request.page_limit <= 0 + or self.saw_every_row + or (len(self.candidates) > request.page_limit and self.last_settled) + ) + return page_settled and (self.saw_every_row or self.session_count > request.cap) + + def next_size(self, request: _Request) -> int: + rows_per_session: Final = self.rows_read / self.session_count + return max(2 * self.rows_read, math.ceil(1.25 * (request.cap + 1) * rows_per_session)) + + +_Reader: TypeAlias = Callable[[int | None], Awaitable[_Read]] + + +def _parse_session_cursor(session_cursor: str | None) -> tuple[str, str, str] | None: + if session_cursor is None or session_cursor.count("|") < 2: + return None + last_activity, _, rest = session_cursor.partition("|") + api_key, _, session_key = rest.partition("|") + if not last_activity or not session_key: + return None + return (last_activity, session_key, api_key) + + +async def read_session_page( + db: _SupportsQueryRaw, + *, + where_clause: str, + params: Sequence[object], + status: str | None, + page: int, + page_size: int, + sort_desc: bool, + session_cursor: str | None, + cap: int, +) -> SessionPage: + """One page of sessions after ``session_cursor``, or after ``(page - 1) * page_size`` sessions, keeping only those + whose outcome is ``status`` when given. ``where_clause`` reads ``params`` as ``$1`` onward.""" + cursor: Final = _parse_session_cursor(session_cursor) + offset: Final = (page - 1) * page_size if cursor is None else 0 + request: Final = _Request(cursor, offset, min(page_size, cap - offset), sort_desc, status, cap) + return await _session_page(_sql_reader(db, where_clause, params, request), request) + + +async def _session_page(read: _Reader, request: _Request) -> SessionPage: + last = await read(request.first_read_size()) # rebind-ok: each read replaces the last until one settles the page + while not last.settles(request): + last = await read(last.next_size(request)) + has_more: Final = len(last.candidates) > request.page_limit + sessions: Final = last.candidates[: request.page_limit] + final: Final = sessions[-1] if has_more and sessions else None + return SessionPage( + sessions=sessions, + has_more=has_more, + next_cursor=f"{final.last_activity}|{final.api_key}|{final.key}" if final else None, + total=min(last.session_count, request.cap), + total_is_capped=last.session_count > request.cap, + ) + + +def _sessions_sql(request: _Request, status_index: int) -> str: + """Each session's last activity, from its rows in ``scanned``, keeping only sessions whose outcome is the status.""" + if request.status is None: + return ( + 'SELECT session_key, api_key, MAX("startTime") AS last_activity FROM scanned GROUP BY session_key, api_key' + ) + return f""" + SELECT session_key, api_key, last_activity + FROM ( + SELECT DISTINCT ON (session_key, api_key) + session_key, + api_key, + status, + MAX("startTime") OVER (PARTITION BY session_key, api_key) AS last_activity + FROM scanned + ORDER BY session_key, api_key, {SESSION_REPRESENTATIVE_ORDER_SQL} + ) AS outcomes + WHERE COALESCE(status, 'success') = ${status_index} + """ + + +def _sql_reader(db: _SupportsQueryRaw, where_clause: str, params: Sequence[object], request: _Request) -> _Reader: + status_index: Final = len(params) + 1 + status_args: Final = () if request.status is None else (request.status,) + cursor_index: Final = status_index + len(status_args) + after_cursor: Final = ( + f"WHERE (last_activity, session_key, api_key) {'<' if request.sort_desc else '>'} " + f"(${cursor_index}::timestamp, ${cursor_index + 1}, ${cursor_index + 2})" + if request.cursor + else "" + ) + outcome_columns: Final = ', status, call_type, "endTime", request_id' if request.status is not None else "" + direction: Final = "DESC" if request.sort_desc else "ASC" + page_rows: Final = request.page_limit + 1 if request.page_limit > 0 else 0 + skip: Final = f"OFFSET {request.offset}" if page_rows and request.offset else "" + + async def read(size: int | None) -> _Read: + # LEFT JOIN keeps the summary row when no session is on the page. + rows: Final = await db.query_raw( + f""" + WITH scanned AS ( + SELECT {SESSION_KEY_EXPR} AS session_key, api_key, "startTime"{outcome_columns} + FROM "LiteLLM_SpendLogs" + WHERE {where_clause} + {f'ORDER BY "startTime" DESC LIMIT {size}' if size is not None else ""} + ), + sessions AS ({_sessions_sql(request, status_index)}), + summary AS ( + SELECT COUNT(*) AS rows_read, + MIN("startTime") AS oldest, + (SELECT COUNT(*) FROM sessions) AS session_count + FROM scanned + ) + SELECT page.session_key, + page.api_key, + page.last_activity::text AS last_activity, + page.last_activity > summary.oldest AS settled, + summary.rows_read, + summary.session_count + FROM summary + LEFT JOIN LATERAL ( + SELECT session_key, api_key, last_activity + FROM sessions + {after_cursor} + ORDER BY last_activity {direction}, session_key {direction}, api_key {direction} + LIMIT {page_rows} {skip} + ) AS page ON TRUE + ORDER BY page.last_activity {direction}, page.session_key {direction}, page.api_key {direction} + """, + *params, + *status_args, + *(request.cursor or ()), + ) + rows_read: Final = int(rows[0]["rows_read"]) + return _Read( + candidates=tuple( + Session(str(row["session_key"]), str(row["api_key"]), str(row["last_activity"])) + for row in rows + if row["session_key"] is not None + ), + last_settled=bool(rows[-1]["settled"]), + rows_read=rows_read, + session_count=int(rows[0]["session_count"]), + saw_every_row=size is None or rows_read < size, + ) + + return read diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 89200792c0b..fcd719b24b9 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -52,6 +52,13 @@ from litellm.proxy.auth.authorization_dependencies import ( from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.openai_error_payload import error_status_code, openai_error_param, openai_error_type from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup +from litellm.proxy.spend_tracking.session_page import ( + AGENT_CALL_TYPE_SQL, + MCP_CALL_TYPES_SQL, + SESSION_KEY_EXPR, + SESSION_REPRESENTATIVE_ORDER_SQL, + read_session_page, +) from litellm.proxy.spend_tracking.spend_capture_rate import ( ProviderBillingCredentialMissing, ProviderBillingRequestFailed, @@ -87,23 +94,15 @@ router: Final = APIRouter() SPEND_LOGS_PAGINATION_COUNT_CAP: Final = 10000 _DECODED_JSON: Final = TypeAdapter(object) -_SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)" -_SESSION_GROUP_KEY_SQL: Final = f"{_SESSION_KEY_EXPR}, api_key" -_MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')" -_AGENT_CALL_TYPE_SQL: Final = "'asend_message'" -_SESSION_REPRESENTATIVE_ORDER_SQL: Final = ( - f"(call_type = {_AGENT_CALL_TYPE_SQL}) DESC, " - f'CASE WHEN call_type = {_AGENT_CALL_TYPE_SQL} THEN "endTime" END DESC NULLS LAST, ' - f'call_type IN {_MCP_CALL_TYPES_SQL}, "startTime" DESC, request_id' -) +_SESSION_GROUP_KEY_SQL: Final = f"{SESSION_KEY_EXPR}, api_key" _BATCH_CALL_TYPES_SQL: Final = "('acreate_batch', 'create_batch', 'aretrieve_batch', 'retrieve_batch')" _SPAN_TYPE_SQL_CONDITIONS: Final[Mapping[str, str]] = MappingProxyType( { - "mcp": f"call_type IN {_MCP_CALL_TYPES_SQL}", - "agent": f"call_type = {_AGENT_CALL_TYPE_SQL}", + "mcp": f"call_type IN {MCP_CALL_TYPES_SQL}", + "agent": f"call_type = {AGENT_CALL_TYPE_SQL}", "batch": f"call_type IN {_BATCH_CALL_TYPES_SQL}", "llm": ( - f"(call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL} " + f"(call_type NOT IN {MCP_CALL_TYPES_SQL} AND call_type != {AGENT_CALL_TYPE_SQL} " f"AND call_type NOT IN {_BATCH_CALL_TYPES_SQL})" ), } @@ -2948,23 +2947,6 @@ async def ui_view_spend_logs( sql_params.append(json.dumps(used_client_oauth_token)) p += 1 - if status_filter is not None and group_by_session is True and not is_search_lookup: - session_filter_conditions: Final = " AND ".join(sql_conditions) or "TRUE" - sql_conditions.append( - f"""({_SESSION_GROUP_KEY_SQL}) IN ( - SELECT session_key, api_key FROM ( - SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) - {_SESSION_KEY_EXPR} AS session_key, api_key, status - FROM "LiteLLM_SpendLogs" - WHERE {session_filter_conditions} - ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} - ) AS session_outcomes - WHERE COALESCE(status, 'success') = ${p} - )""" - ) - sql_params.append(status_filter) - p += 1 - if ( group_by_session is True and not is_v2 @@ -2977,12 +2959,30 @@ async def ui_view_spend_logs( sql_conditions=sql_conditions, sql_params=sql_params, next_param_index=p, + status_filter=status_filter, page=page, page_size=page_size, sort_desc=order_direction != "asc", session_cursor=session_cursor, ) + if status_filter is not None and group_by_session is True and not is_search_lookup: + session_filter_conditions: Final = " AND ".join(sql_conditions) or "TRUE" + sql_conditions.append( + f"""({_SESSION_GROUP_KEY_SQL}) IN ( + SELECT session_key, api_key FROM ( + SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) + {SESSION_KEY_EXPR} AS session_key, api_key, status + FROM "LiteLLM_SpendLogs" + WHERE {session_filter_conditions} + ORDER BY {_SESSION_GROUP_KEY_SQL}, {SESSION_REPRESENTATIVE_ORDER_SQL} + ) AS session_outcomes + WHERE COALESCE(status, 'success') = ${p} + )""" + ) + sql_params.append(status_filter) + p += 1 + # Build the ORDER BY expression. ttft_ms is computed from # completionStartTime - startTime; non-streaming rows (where # completionStartTime is null or equals endTime) yield NULL, so we @@ -3031,7 +3031,7 @@ async def ui_view_spend_logs( {_SPEND_LOG_LIST_COLUMNS} FROM "LiteLLM_SpendLogs" WHERE {joined_conditions} - ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} + ORDER BY {_SESSION_GROUP_KEY_SQL}, {SESSION_REPRESENTATIVE_ORDER_SQL} ) AS session_representatives ORDER BY {exact_request_id_first}{_order_expr} {_sql_dir}{_nulls_clause}, request_id LIMIT ${p} OFFSET ${p + 1} @@ -3080,22 +3080,6 @@ async def ui_view_spend_logs( raise handle_exception_on_proxy(e) -class _SessionPageRow(TypedDict): - session_key: ReadOnly[str] - api_key: ReadOnly[str] - last_activity: ReadOnly[str] - - -def _parse_session_cursor(session_cursor: str | None) -> tuple[str, str, str] | None: - if session_cursor is None or session_cursor.count("|") < 2: - return None - last_activity, _, rest = session_cursor.partition("|") - api_key, _, session_key = rest.partition("|") - if not last_activity or not session_key: - return None - return (last_activity, session_key, api_key) - - async def _fetch_session_representatives( prisma_client: "PrismaClient", where_clause: str, @@ -3104,16 +3088,18 @@ async def _fetch_session_representatives( session_keys: Sequence[tuple[str, str]], ) -> list[dict[str, object]]: # mutable-ok: _build_ui_spend_logs_response writes session counts onto each row """Fetch the final agent outcome, or newest non-MCP row, of each ``(session_key, api_key)`` session, in ``session_keys`` order.""" + # Lets Postgres use the session_id and request_id indexes. rep_query: Final = f""" SELECT * FROM ( SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) {_SPEND_LOG_LIST_COLUMNS} FROM "LiteLLM_SpendLogs" WHERE {where_clause} + AND (session_id = ANY(${next_param_index}::text[]) OR request_id = ANY(${next_param_index}::text[])) AND ({_SESSION_GROUP_KEY_SQL}) IN ( SELECT * FROM unnest(${next_param_index}::text[], ${next_param_index + 1}::text[]) ) - ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} + ORDER BY {_SESSION_GROUP_KEY_SQL}, {SESSION_REPRESENTATIVE_ORDER_SQL} ) AS session_representatives """ rep_rows: Final[Sequence[dict[str, object]]] = await _query_raw( # mutable-ok: rows are enriched in place @@ -3129,117 +3115,32 @@ async def _fetch_session_representatives( return [rep_by_key[key] for key in session_keys if key in rep_by_key] -async def _count_grouped_sessions( - prisma_client: "PrismaClient", - where_clause: str, - sql_params: Sequence[object], - next_param_index: int, -) -> tuple[int, bool]: - """Count the sessions matching the filter, returning ``(total, total_is_capped)`` bounded by the count cap.""" - count_query: Final = f""" - SELECT COUNT(*) AS total_count - FROM ( - SELECT 1 - FROM "LiteLLM_SpendLogs" - WHERE {where_clause} - GROUP BY {_SESSION_GROUP_KEY_SQL} - LIMIT ${next_param_index} - ) AS bounded_sessions - """ - count_rows: Final[Sequence[_SpendLogsCountRow]] = await _query_raw( - prisma_client, count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1 - ) - raw_total: Final = int(count_rows[0]["total_count"]) if count_rows else 0 - return ( - (SPEND_LOGS_PAGINATION_COUNT_CAP, True) if raw_total > SPEND_LOGS_PAGINATION_COUNT_CAP else (raw_total, False) - ) - - async def _ui_session_grouped_spend_logs( prisma_client: "PrismaClient", sql_conditions: Sequence[str], sql_params: Sequence[object], next_param_index: int, + status_filter: str | None, page: int, page_size: int, sort_desc: bool, session_cursor: str | None, ) -> Mapping[str, object]: - """ - One row per session, keyset-paginated by session last activity. - - Sessions are derived on the fly from ``LiteLLM_SpendLogs`` (no extra - table): rows sharing a ``session_id`` and ``api_key`` form a session, rows - without a session id are singletons keyed by ``request_id``. A page is the - next ``page_size`` sessions ordered by ``(MAX(startTime), session_key, - api_key)``, resumed from the ``session_cursor`` keyset - ``'||'`` instead of an OFFSET, so - page depth does not degrade the query plan. A request for ``page > 1`` - without a cursor (the UI jumping straight to the last page, or back to a - page it never walked through) falls back to ``OFFSET (page - 1) * - page_size``, trimmed to the end of the ``SPEND_LOGS_PAGINATION_COUNT_CAP`` - window the capped ``total`` promises, so a page never runs past that total - and one starting at or past it returns no rows without a query. Each session is represented - by its final agent outcome (or newest non-MCP row), enriched by ``_build_ui_spend_logs_response`` - exactly like the flat listing, and the response carries - ``next_session_cursor`` / ``has_more`` while ``total`` counts sessions - (capped like the flat total). A page that runs out of sessions while still - holding some is itself the end of the list, so its ``total`` is - ``offset + len(page)`` and the grouped count query is skipped; a page that - starts past the end says nothing about the total, so that one is counted. - """ + """One row per session, keyset-paginated by last activity. ``read_session_page`` picks and counts the sessions, + and each is shown by its final agent outcome or newest non-MCP row.""" where_clause: Final = " AND ".join(sql_conditions) if sql_conditions else "TRUE" - cmp_op: Final = "<" if sort_desc else ">" - direction: Final = "DESC" if sort_desc else "ASC" - - cursor: Final = _parse_session_cursor(session_cursor) - having_clause: Final = ( - f'HAVING (MAX("startTime"), {_SESSION_GROUP_KEY_SQL}) {cmp_op} ' - f"(${next_param_index}::timestamp, ${next_param_index + 1}, ${next_param_index + 2})" - if cursor - else "" + session_page: Final = await read_session_page( + prisma_client.db, # pyright: ignore[reportArgumentType] # PrismaWrapper forwards raw queries via __getattr__ + where_clause=where_clause, + params=sql_params, + status=status_filter, + page=page, + page_size=page_size, + sort_desc=sort_desc, + session_cursor=session_cursor, + cap=SPEND_LOGS_PAGINATION_COUNT_CAP, ) - cursor_params: Final[tuple[object, ...]] = cursor if cursor else () - limit_index: Final = next_param_index + len(cursor_params) - offset: Final = (page - 1) * page_size if cursor is None else 0 - page_limit: Final = min(page_size, SPEND_LOGS_PAGINATION_COUNT_CAP - offset) - offset_params: Final[tuple[int, ...]] = (offset,) if offset and page_limit > 0 else () - offset_clause: Final = f"OFFSET ${limit_index + 1}" if offset_params else "" - - page_query: Final = f""" - SELECT {_SESSION_KEY_EXPR} AS session_key, - api_key, - MAX("startTime")::text AS last_activity - FROM "LiteLLM_SpendLogs" - WHERE {where_clause} - GROUP BY {_SESSION_GROUP_KEY_SQL} - {having_clause} - ORDER BY MAX("startTime") {direction}, {_SESSION_KEY_EXPR} {direction}, api_key {direction} - LIMIT ${limit_index} {offset_clause} - """ - page_rows: Final[Sequence[_SessionPageRow]] = ( - () - if page_limit <= 0 - else await _query_raw(prisma_client, page_query, *sql_params, *cursor_params, page_limit + 1, *offset_params) - ) - - has_more: Final = len(page_rows) > page_limit - visible_rows: Final = page_rows[:page_limit] - next_cursor: Final = ( - f"{visible_rows[-1]['last_activity']}|{visible_rows[-1]['api_key']}|{visible_rows[-1]['session_key']}" - if has_more and visible_rows - else None - ) - - page_starts_inside_the_list: Final = offset == 0 or len(page_rows) > 0 - page_ends_the_list: Final = cursor is None and page_limit > 0 and not has_more and page_starts_inside_the_list - total_records, total_is_capped = ( - (offset + len(page_rows), False) - if page_ends_the_list - else await _count_grouped_sessions(prisma_client, where_clause, sql_params, next_param_index) - ) - - session_keys: Final = tuple((row["session_key"], row["api_key"]) for row in visible_rows) + session_keys: Final = tuple((session.key, session.api_key) for session in session_page.sessions) data: Final[list[dict[str, object]]] = ( # mutable-ok: _build_ui_spend_logs_response writes onto each row await _fetch_session_representatives( prisma_client=prisma_client, @@ -3253,18 +3154,17 @@ async def _ui_session_grouped_spend_logs( ) _hydrate_spend_log_metadata(data) - total_pages: Final = (total_records + page_size - 1) // page_size response: Final[Mapping[str, object]] = await _build_ui_spend_logs_response( prisma_client, data, - total_records, + session_page.total, page, page_size, - total_pages, + (session_page.total + page_size - 1) // page_size, enrich_session_counts=True, - total_is_capped=total_is_capped, + total_is_capped=session_page.total_is_capped, ) - return {**response, "next_session_cursor": next_cursor, "has_more": has_more} + return {**response, "next_session_cursor": session_page.next_cursor, "has_more": session_page.has_more} class RequestResponsePayload(NamedTuple): @@ -4720,16 +4620,16 @@ async def _build_ui_spend_logs_response( ) ), 0)::bigint AS session_total_duration_ms, COUNT(*) FILTER ( - WHERE call_type IN {_MCP_CALL_TYPES_SQL} + WHERE call_type IN {MCP_CALL_TYPES_SQL} )::int AS mcp_tool_call_count, COALESCE(SUM(spend) FILTER ( - WHERE call_type IN {_MCP_CALL_TYPES_SQL} + WHERE call_type IN {MCP_CALL_TYPES_SQL} ), 0)::double precision AS mcp_tool_call_spend, COUNT(*) FILTER (WHERE LOWER(cache_hit) = 'true')::int AS session_cache_hit_count, COUNT(*) FILTER ( - WHERE call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL} + WHERE call_type NOT IN {MCP_CALL_TYPES_SQL} AND call_type != {AGENT_CALL_TYPE_SQL} )::int AS session_llm_count, - COUNT(*) FILTER (WHERE call_type = {_AGENT_CALL_TYPE_SQL})::int AS session_agent_count, + COUNT(*) FILTER (WHERE call_type = {AGENT_CALL_TYPE_SQL})::int AS session_agent_count, COALESCE(SUM(prompt_tokens), 0)::bigint AS session_total_prompt_tokens, COALESCE(SUM(completion_tokens), 0)::bigint AS session_total_completion_tokens, COALESCE(SUM(total_tokens), 0)::bigint AS session_total_tokens diff --git a/tests/integration/spend/test_session_grouped_logs_pages.py b/tests/integration/spend/test_session_grouped_logs_pages.py new file mode 100644 index 00000000000..19854d60bd3 --- /dev/null +++ b/tests/integration/spend/test_session_grouped_logs_pages.py @@ -0,0 +1,455 @@ +import itertools +import json +import os +import uuid +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence +from contextlib import asynccontextmanager +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Final +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +import psycopg +import pytest +from prisma import Prisma +from psycopg import sql +from pydantic import BaseModel, ConfigDict, Field + +from litellm.proxy.spend_tracking.session_page import read_session_page +from litellm.proxy.spend_tracking.spend_management_endpoints import _fetch_session_representatives +from tests.integration._support.client import Gateway +from tests.integration._support.database import write_rows + +_URL: Final = "/spend/logs/ui" +_WINDOW_SECONDS: Final = 3600 + + +@dataclass(frozen=True, slots=True) +class _Row: + request: str + session: str | None + api_key: str + second: int + status: str = "success" + model: str = "m-a" + + +_ROWS: Final = ( + _Row("z1", "z", "key-1", -1), + _Row("a1", "a", "key-1", 10), + _Row("a2", "a", "key-1", 50), + _Row("b1", "b", "key-1", 20, model="m-b"), + _Row("b2", "b", "key-1", 60), + _Row("b3", "b", "key-1", 30), + _Row("c1", "c", "key-2", 60), + _Row("d1", "b", "key-2", 40, model="m-b"), + _Row("e1", None, "key-1", 55, model="m-b"), + _Row("f1", "", "key-1", 45), + _Row("g1", "g", "key-3", 5, status="failure"), + _Row("g2", "g", "key-3", 15, status="failure"), + *(_Row(f"h{second - 69}", "h", "key-1", second) for second in range(70, 76)), + _Row("i1", "i", "key-3", 35, status="failure"), + _Row("i2", "i", "key-3", _WINDOW_SECONDS + 1), +) +_NEWEST_FIRST: Final = ( + ("h", "key-1"), + ("c", "key-2"), + ("b", "key-1"), + ("e1", "key-1"), + ("a", "key-1"), + ("f1", "key-1"), + ("b", "key-2"), + ("i", "key-3"), + ("g", "key-3"), +) + + +class _LogRow(BaseModel): + model_config = ConfigDict(frozen=True) + + request_id: str + session_id: str | None + api_key: str + session_total_count: int + + +class _Page(BaseModel): + model_config = ConfigDict(frozen=True) + + data: tuple[_LogRow, ...] + total: int + total_is_capped: bool + has_more: bool + next_session_cursor: str | None + + +@dataclass(frozen=True, slots=True) +class _Seeded: + prefix: str + window: Mapping[str, str] + + def session(self, row: _LogRow) -> tuple[str, str]: + key: Final = row.session_id or row.request_id + assert key.startswith(self.prefix), key + return key.removeprefix(self.prefix), row.api_key + + +@pytest.fixture +def seeded(gateway: Gateway) -> Iterator[_Seeded]: + prefix: Final = f"sess{uuid.uuid4().hex[:12]}-" + start: Final = datetime(1900, 1, 1) + timedelta(days=int(prefix[4:16], 16) % 200000) + rows: Final = [ + { + "request_id": f"{prefix}{row.request}", + "session_id": None if row.session is None else f"{prefix}{row.session}" if row.session else "", + "api_key": row.api_key, + "start_time": (start + timedelta(seconds=row.second)).isoformat(), + "status": row.status, + "model": row.model, + } + for row in _ROWS + ] + with gateway.scenario() as scenario: + scenario.cleanups.callback( + write_rows, 'DELETE FROM "LiteLLM_SpendLogs" WHERE request_id LIKE %s', (f"{prefix}%",) + ) + write_rows( + 'INSERT INTO "LiteLLM_SpendLogs" (request_id, call_type, api_key, "startTime", "endTime", model, ' + "session_id, status) SELECT request_id, 'acompletion', api_key, start_time, start_time, model, session_id, " + "status FROM jsonb_to_recordset(%s::jsonb) AS seeded(request_id text, session_id text, api_key text, " + "start_time timestamp, status text, model text)", + (json.dumps(rows),), + ) + yield _Seeded( + prefix, + { + "start_date": f"{start:%Y-%m-%d %H:%M:%S}", + "end_date": f"{start + timedelta(seconds=_WINDOW_SECONDS):%Y-%m-%d %H:%M:%S}", + "group_by_session": "true", + }, + ) + + +def _page(gateway: Gateway, seeded: _Seeded, params: Mapping[str, str]) -> _Page: + return _Page.model_validate(gateway.get(_URL, {**seeded.window, **params})) + + +def _cursor_walk(gateway: Gateway, seeded: _Seeded, params: Mapping[str, str], number: int = 1) -> tuple[_Page, ...]: + page: Final = _page(gateway, seeded, {**params, "page_size": "2", "page": str(number)}) + if page.next_session_cursor is None or number > len(_NEWEST_FIRST): + return (page,) + following: Final = {**params, "session_cursor": page.next_session_cursor} + return (page, *_cursor_walk(gateway, seeded, following, number + 1)) + + +def _rows(pages: tuple[_Page, ...]) -> Iterator[_LogRow]: + return itertools.chain.from_iterable(page.data for page in pages) + + +def _sessions(seeded: _Seeded, pages: tuple[_Page, ...]) -> tuple[tuple[str, str], ...]: + return tuple(seeded.session(row) for row in _rows(pages)) + + +def test_cursor_pages_show_every_session_once_newest_first_with_its_newest_row( + gateway: Gateway, seeded: _Seeded +) -> None: + pages: Final = _cursor_walk(gateway, seeded, {}) + assert _sessions(seeded, pages) == _NEWEST_FIRST + assert [(page.total, page.total_is_capped) for page in pages] == [(9, False)] * 5 + assert [page.has_more for page in pages] == [True, True, True, True, False] + representatives: Final = tuple( + (row.request_id.removeprefix(seeded.prefix), row.session_total_count) for row in _rows(pages) + ) + assert representatives == ( + ("h6", 6), + ("c1", 1), + ("b2", 3), + ("e1", 1), + ("a2", 2), + ("f1", 1), + ("d1", 1), + ("i1", 2), + ("g2", 2), + ) + + +@pytest.mark.parametrize( + "sort_order, expected", + [("desc", _NEWEST_FIRST), ("asc", tuple(reversed(_NEWEST_FIRST)))], + ids=["newest-first", "oldest-first"], +) +def test_numbered_pages_show_the_same_sessions_as_the_cursor_walk( + gateway: Gateway, seeded: _Seeded, sort_order: str, expected: tuple[tuple[str, str], ...] +) -> None: + pages: Final = tuple( + _page(gateway, seeded, {"page_size": "2", "page": str(number), "sort_order": sort_order}) + for number in range(1, 7) + ) + assert _sessions(seeded, pages) == expected + assert [len(page.data) for page in pages] == [2, 2, 2, 2, 1, 0] + assert [page.has_more for page in pages] == [True, True, True, True, False, False] + assert {page.total for page in pages} == {len(expected)} + + +def test_failure_filter_keeps_only_sessions_whose_outcome_failed(gateway: Gateway, seeded: _Seeded) -> None: + pages: Final = _cursor_walk(gateway, seeded, {"status_filter": "failure"}) + assert _sessions(seeded, pages) == (("i", "key-3"), ("g", "key-3")) + assert pages[0].total == 2 + + +def test_model_filter_ranks_each_session_by_its_matching_rows_only(gateway: Gateway, seeded: _Seeded) -> None: + pages: Final = _cursor_walk(gateway, seeded, {"model": "m-b"}) + assert _sessions(seeded, pages) == (("e1", "key-1"), ("b", "key-2"), ("b", "key-1")) + assert tuple(row.request_id.removeprefix(seeded.prefix) for row in _rows(pages)) == ("e1", "d1", "b1") + assert pages[0].total == 3 + + +_COPIED_ROWS: Final = 20_000 +_COPY_START: Final = datetime(2000, 1, 1, tzinfo=UTC) +_COPY_WINDOW: Final = ( + "\"startTime\" >= ($1::timestamptz AT TIME ZONE 'UTC') AND \"startTime\" <= ($2::timestamptz AT TIME ZONE 'UTC')" +) +_COPY_WINDOW_PARAMS: Final = (_COPY_START, _COPY_START + timedelta(days=1)) + + +def _generated_sessions(rows_per_session: int) -> tuple[sql.SQL, tuple[object, ...]]: + """`_COPIED_ROWS` rows a second apart in sessions of `rows_per_session` rows, a one-row session keyed by request_id.""" + return sql.SQL( + 'INSERT INTO {table} (request_id, call_type, api_key, "startTime", "endTime", session_id) ' + "SELECT 'r' || n, 'acompletion', 'key-1', moment, moment, " + "CASE WHEN %s > 1 THEN 's' || (n - 1) / %s END FROM generate_series(1, %s) n, " + "LATERAL (SELECT %s::timestamp + n * INTERVAL '1 second' AS moment) seconds" + ), (rows_per_session, rows_per_session, _COPIED_ROWS, _COPY_START.replace(tzinfo=None)) + + +def _sessions_tied_at_the_newest_moment() -> tuple[sql.SQL, tuple[object, ...]]: + """`_generated_sessions(1)` plus thirty one-row sessions sharing a newer moment, inserted and keyed against session + order so a read cut inside the tie misses the sessions that rank first.""" + start: Final = _COPY_START.replace(tzinfo=None) + return sql.SQL( + 'INSERT INTO {table} (request_id, call_type, api_key, "startTime", "endTime", session_id) ' + "SELECT request_id, 'acompletion', 'key-1', moment, moment, session_id FROM (" + "SELECT n AS ordinal, 'r' || n AS request_id, %s::timestamp + n * INTERVAL '1 second' AS moment, " + "NULL AS session_id FROM generate_series(1, %s) n UNION ALL " + "SELECT %s + 30 - n, 't' || lpad((29 - n)::text, 2, '0'), %s::timestamp + %s * INTERVAL '1 second', " + "'s' || lpad(n::text, 2, '0') FROM generate_series(0, 29) n) seeded ORDER BY ordinal" + ), (start, _COPIED_ROWS, _COPIED_ROWS, start, _COPIED_ROWS + 1) + + +@dataclass(frozen=True, slots=True) +class _Call: + session: str + call_type: str + status: str | None + second: int + + +def _calls(calls: tuple[_Call, ...]) -> tuple[sql.SQL, tuple[object, ...]]: + """One row per call, `second` seconds into the window, ending a second later.""" + seeded: Final = tuple( + { + "request_id": f"{call.session}-{call.second}", + "call_type": call.call_type, + "session_id": call.session, + "status": call.status, + "start_time": (_COPY_START + timedelta(seconds=call.second)).replace(tzinfo=None).isoformat(), + } + for call in calls + ) + return sql.SQL( + 'INSERT INTO {table} (request_id, call_type, api_key, "startTime", "endTime", session_id, status) ' + "SELECT request_id, call_type, 'key-1', start_time, start_time + INTERVAL '1 second', session_id, status " + "FROM jsonb_to_recordset(%s::jsonb) AS calls(request_id text, call_type text, session_id text, status text, " + "start_time timestamp)" + ), (json.dumps(seeded),) + + +@asynccontextmanager +async def _spend_logs_copy(insert: sql.SQL, args: tuple[object, ...]) -> AsyncIterator[Prisma]: + """A private schema holding a copy of the migrated `LiteLLM_SpendLogs` with its indexes, seeded by `insert`.""" + schema: Final = f"integration_{uuid.uuid4().hex}" + url: Final = os.environ["DATABASE_URL"] + table: Final = sql.Identifier(schema, "LiteLLM_SpendLogs") + with psycopg.connect(url, autocommit=True) as setup: + setup.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema))) + try: + setup.execute(sql.SQL('CREATE TABLE {} (LIKE "LiteLLM_SpendLogs" INCLUDING ALL)').format(table)) + setup.execute(insert.format(table=table), args) + setup.execute(sql.SQL("ANALYZE {}").format(table)) + parsed: Final = urlsplit(url) + query: Final = urlencode({**dict(parse_qsl(parsed.query)), "schema": schema}) + database: Final = Prisma(datasource={"url": urlunsplit(parsed._replace(query=query))}) + await database.connect() + try: + yield database + finally: + await database.disconnect() + finally: + setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema))) + + +class _PlanNode(BaseModel): + model_config = ConfigDict(frozen=True) + + relation_name: str | None = Field(default=None, alias="Relation Name") + actual_rows: float = Field(alias="Actual Rows") + rows_removed_by_filter: float = Field(default=0, alias="Rows Removed by Filter") + actual_loops: int = Field(alias="Actual Loops") + plans: tuple["_PlanNode", ...] = Field(default=(), alias="Plans") + + def spend_log_rows_scanned(self) -> int: + own: Final = ( + (self.actual_rows + self.rows_removed_by_filter) * self.actual_loops + if self.relation_name == "LiteLLM_SpendLogs" + else 0 + ) + return round(own) + sum(child.spend_log_rows_scanned() for child in self.plans) + + +class _Explained(BaseModel): + model_config = ConfigDict(frozen=True) + + plan: _PlanNode = Field(alias="Plan") + + +class _ExplainedReads: + """Runs each read through Prisma, first under EXPLAIN ANALYZE to count the `LiteLLM_SpendLogs` rows it scans.""" + + def __init__(self, database: Prisma) -> None: + self.database: Final = database + self.rows_scanned: tuple[int, ...] = () + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + explained: Final = await self.database.query_raw(f"EXPLAIN (ANALYZE, FORMAT JSON) {query}", *args) + plan: Final = _Explained.model_validate(explained[0]["QUERY PLAN"][0]).plan + self.rows_scanned = (*self.rows_scanned, plan.spend_log_rows_scanned()) + return await self.database.query_raw(query, *args) + + +@pytest.mark.parametrize( + "rows_per_session, page, newest_session, rows_scanned_per_read, total", + [ + pytest.param(1, 1, ("r", 20_000), (102,), (100, True), id="first-page"), + pytest.param(1, 3, ("r", 19_980), (102,), (100, True), id="third-page"), + pytest.param(10, 1, ("s", 1_999), (102, 1_171), (100, True), id="second-read-sized-by-the-first-reads-density"), + pytest.param(100, 1, ("s", 199), (102, 6_439, 12_878), (100, True), id="third-read-at-least-doubled"), + pytest.param(1_000, 1, ("s", 19), (102, 12_878, 20_000), (20, False), id="last-read-runs-out-of-rows"), + ], +) +async def test_a_newest_first_page_scans_only_the_rows_its_reads_ask_for( + rows_per_session: int, + page: int, + newest_session: tuple[str, int], + rows_scanned_per_read: tuple[int, ...], + total: tuple[int, bool], +) -> None: + async with _spend_logs_copy(*_generated_sessions(rows_per_session)) as database: + reads: Final = _ExplainedReads(database) + session_page: Final = await read_session_page( + reads, + where_clause=_COPY_WINDOW, + params=_COPY_WINDOW_PARAMS, + page=page, + page_size=10, + status=None, + sort_desc=True, + session_cursor=None, + cap=100, + ) + prefix, newest = newest_session + assert tuple(session.key for session in session_page.sessions) == tuple( + f"{prefix}{newest - n}" for n in range(10) + ) + assert (session_page.total, session_page.total_is_capped) == total + assert reads.rows_scanned == rows_scanned_per_read + + +async def test_a_page_cut_inside_a_tie_reads_on_until_the_tie_is_whole() -> None: + async with _spend_logs_copy(*_sessions_tied_at_the_newest_moment()) as database: + reads: Final = _ExplainedReads(database) + session_page: Final = await read_session_page( + reads, + where_clause=_COPY_WINDOW, + params=_COPY_WINDOW_PARAMS, + page=1, + page_size=5, + status=None, + sort_desc=True, + session_cursor=None, + cap=10, + ) + assert tuple(session.key for session in session_page.sessions) == ("s29", "s28", "s27", "s26", "s25") + assert reads.rows_scanned == (12, 24, 48) + + +@dataclass(frozen=True, slots=True) +class _ExplainedPrisma: + db: _ExplainedReads + + +async def test_a_pages_representatives_are_found_by_session_not_by_scanning_the_window() -> None: + async with _spend_logs_copy(*_generated_sessions(rows_per_session=10)) as database: + reads: Final = _ExplainedReads(database) + representatives: Final = await _fetch_session_representatives( + prisma_client=_ExplainedPrisma(reads), # pyright: ignore[reportArgumentType] # only .db is used + where_clause=_COPY_WINDOW, + sql_params=_COPY_WINDOW_PARAMS, + next_param_index=len(_COPY_WINDOW_PARAMS) + 1, + session_keys=(("s1999", "key-1"), ("s1998", "key-1")), + ) + assert tuple(row["request_id"] for row in representatives) == ("r20000", "r19990") + assert reads.rows_scanned == (20,), "only the two sessions' rows, not the window's 20,000" + + +async def test_a_page_number_far_past_the_cap_shows_no_session() -> None: + async with _spend_logs_copy(*_generated_sessions(rows_per_session=1)) as database: + session_page: Final = await read_session_page( + _ExplainedReads(database), + where_clause=_COPY_WINDOW, + params=_COPY_WINDOW_PARAMS, + page=9_300_000_000_000_000, + page_size=1000, + status=None, + sort_desc=True, + session_cursor=None, + cap=100, + ) + assert (session_page.sessions, session_page.total, session_page.total_is_capped) == ((), 100, True) + + +_OUTCOME_CALLS: Final = ( + _Call("agent", "asend_message", "failure", 10), + _Call("agent", "acompletion", "success", 20), + _Call("mcp", "acompletion", "failure", 5), + _Call("mcp", "call_mcp_tool", "success", 30), + _Call("unknown", "acompletion", None, 40), + _Call("failed", "acompletion", "success", 1), + _Call("failed", "acompletion", "failure", 50), + _Call("recovered", "acompletion", "failure", 2), + _Call("recovered", "acompletion", "success", 45), +) + + +@pytest.mark.parametrize( + "status, expected", + [("failure", ("failed", "mcp", "agent")), ("success", ("recovered", "unknown"))], +) +async def test_a_status_filter_keeps_sessions_whose_representative_call_has_that_status( + status: str, expected: tuple[str, ...] +) -> None: + """An agent message stands for its session, then the newest call that is not an MCP tool call, and a missing status + counts as success. The page stays ordered by each session's newest call.""" + async with _spend_logs_copy(*_calls(_OUTCOME_CALLS)) as database: + session_page: Final = await read_session_page( + _ExplainedReads(database), + where_clause=_COPY_WINDOW, + params=_COPY_WINDOW_PARAMS, + status=status, + page=1, + page_size=10, + sort_desc=True, + session_cursor=None, + cap=100, + ) + assert tuple(session.key for session in session_page.sessions) == expected + assert (session_page.total, session_page.total_is_capped) == (len(expected), False) diff --git a/tests/unit/proxy/spend_tracking/test_session_page.py b/tests/unit/proxy/spend_tracking/test_session_page.py new file mode 100644 index 00000000000..a86990bddbb --- /dev/null +++ b/tests/unit/proxy/spend_tracking/test_session_page.py @@ -0,0 +1,327 @@ +import itertools +import random +import re +from collections.abc import Sequence +from typing import Final, TypeAlias + +import pytest + +from litellm.proxy.spend_tracking.session_page import ( + Session, + _parse_session_cursor, + _Read, + _ReadRow, + _Request, + _session_page, + read_session_page, +) + +_Row: TypeAlias = tuple[int, str, str] # (startTime, session key, api_key) +_Ranked: TypeAlias = tuple[int, str, str] # (last activity, session key, api_key) + + +def _session_of(row: _Row) -> tuple[str, str]: + return row[1], row[2] + + +def _ranked(rows: Sequence[_Row], request: _Request) -> tuple[_Ranked, ...]: + """The sessions in ``rows`` in page order, after the request's cursor or offset.""" + by_session: Final = itertools.groupby(sorted(rows, key=_session_of), key=_session_of) + ranked: Final = tuple( + sorted( + ((max(start for start, _, _ in group), key, api_key) for (key, api_key), group in by_session), + reverse=request.sort_desc, + ) + ) + if request.cursor is None: + return ranked[request.offset :] + cursor: Final = (int(request.cursor[0]), request.cursor[1], request.cursor[2]) + return tuple(session for session in ranked if (session < cursor if request.sort_desc else session > cursor)) + + +class _Reads: + """Reads like the page query, whose LIMIT cuts tied rows lowest key first, the opposite of page order. It fails a + read no larger than the one before, since only growing reads end the loop.""" + + def __init__(self, rows: Sequence[_Row], request: _Request, every_row: bool = False) -> None: + self.newest_first: Final = tuple(sorted(rows, key=lambda row: (-row[0], row[1], row[2]))) + self.request: Final = request + self.every_row: Final = every_row + self.budgets: tuple[int | None, ...] = () + + async def __call__(self, budget: int | None) -> _Read: + assert not self.budgets or (self.budgets[-1] is not None and (budget is None or budget > self.budgets[-1])), ( + self.budgets, + budget, + ) + self.budgets = (*self.budgets, budget) + taken: Final = self.newest_first if budget is None or self.every_row else self.newest_first[:budget] + page: Final = _ranked(taken, self.request)[: self.request.page_limit + 1] if self.request.page_limit > 0 else () + return _Read( + candidates=tuple(Session(key, api_key, f"{at:06d}") for at, key, api_key in page), + last_settled=bool(page) and page[-1][0] > taken[-1][0], + rows_read=len(taken), + session_count=len(frozenset(_session_of(row) for row in taken)), + saw_every_row=budget is None or self.every_row or len(taken) < budget, + ) + + +def _session_rows(rng: random.Random, index: int, clock: int) -> tuple[_Row, ...]: + start, length = rng.randrange(0, clock), rng.choice((1, 1, 2, 3, 8, 40)) + return tuple((start + rng.randrange(0, 30), f"s{index:03d}", rng.choice(("k1", "k2"))) for _ in range(length)) + + +def _sessions(rng: random.Random) -> tuple[_Row, ...]: + """Sessions of one to forty rows over a short clock, so many rows share a time.""" + clock: Final = rng.choice((3, 40, 400)) + return tuple( + itertools.chain.from_iterable(_session_rows(rng, index, clock) for index in range(rng.randrange(1, 120))) + ) + + +def _random_cursor(rng: random.Random, ranked: Sequence[_Ranked]) -> tuple[str, str, str]: + at, key, api_key = ranked[rng.randrange(len(ranked))] + return f"{at:06d}", key, api_key + + +def _request(rng: random.Random, rows: Sequence[_Row]) -> _Request: + cap, page_size, sort_desc = rng.randrange(2, 60), rng.randrange(1, 12), rng.random() < 0.8 + mode: Final = rng.choice(("first", "offset", "cursor")) + full: Final = _ranked(rows, _Request(None, 0, 0, sort_desc, None, cap)) + cursor: Final = _random_cursor(rng, full) if mode == "cursor" and full else None + offset: Final = (rng.randrange(1, 2 * cap // page_size + 3) - 1) * page_size if mode == "offset" else 0 + return _Request(cursor, offset, min(page_size, cap - offset), sort_desc, None, cap) + + +@pytest.mark.parametrize("seed", range(60)) +@pytest.mark.asyncio +async def test_a_page_read_from_the_newest_rows_matches_grouping_every_row(seed: int) -> None: + rng: Final = random.Random(seed) + rows: Final = _sessions(rng) + for request in tuple(_request(rng, rows) for _ in range(12)): + grouped = await _session_page(_Reads(rows, request, every_row=True), request) + assert await _session_page(_Reads(rows, request), request) == grouped, request + + +@pytest.mark.parametrize( + "rows_per_session, expected_budgets", + [ + pytest.param(1, (102,), id="one-read-holds-the-cap"), + pytest.param(2, (102, 253), id="sized-by-the-first-reads-density"), + pytest.param(50, (102, 4293, 8586), id="doubles-when-the-density-misleads"), + pytest.param(500, (102, 12878, 62533), id="re-sized-by-each-reads-density"), + ], +) +@pytest.mark.asyncio +async def test_reads_grow_by_the_last_reads_rows_per_session_until_one_settles_the_page( + rows_per_session: int, expected_budgets: tuple[int, ...] +) -> None: + """150 sessions a row per second apart. A cap of 100 starts at 100 + 2 rows.""" + rows: Final = tuple((index, f"s{index // rows_per_session:05d}", "k") for index in range(150 * rows_per_session)) + request: Final = _Request(cursor=None, offset=0, page_limit=10, sort_desc=True, status=None, cap=100) + reads: Final = _Reads(rows, request) + await _session_page(reads, request) + assert reads.budgets == expected_budgets + + +@pytest.mark.asyncio +async def test_a_session_tied_with_the_reads_oldest_row_is_not_trusted() -> None: + """Thirty one-row sessions share one time and the first read takes 12 of them. The page's sixth session sits at the + read's oldest time, where the LIMIT may have cut a session that ranks before it, so the page reads on.""" + rows: Final = tuple((0, f"s{index:02d}", "k") for index in range(30)) + request: Final = _Request(cursor=None, offset=0, page_limit=5, sort_desc=True, status=None, cap=10) + reads: Final = _Reads(rows, request) + page: Final = await _session_page(reads, request) + assert tuple(session.key for session in page.sessions) == ("s29", "s28", "s27", "s26", "s25") + assert reads.budgets == (12, 24, 48) + + +@pytest.mark.parametrize("session_count, expected_total", [(100, (100, False)), (101, (100, True))]) +@pytest.mark.asyncio +async def test_the_total_is_capped_only_past_the_cap(session_count: int, expected_total: tuple[int, bool]) -> None: + rows: Final = tuple((index, f"s{index:05d}", "k") for index in range(session_count)) + request: Final = _Request(cursor=None, offset=0, page_limit=10, sort_desc=True, status=None, cap=100) + page: Final = await _session_page(_Reads(rows, request), request) + assert (page.total, page.total_is_capped) == expected_total + + +@pytest.mark.asyncio +async def test_a_page_number_near_the_bigint_limit_reads_once() -> None: + rows: Final = tuple((index, f"s{index:05d}", "k") for index in range(500)) + offset: Final = (9_300_000_000_000_000 - 1) * 1000 + request: Final = _Request( + cursor=None, offset=offset, page_limit=min(1000, 100 - offset), sort_desc=True, status=None, cap=100 + ) + reads: Final = _Reads(rows, request) + page: Final = await _session_page(reads, request) + assert reads.budgets == (102,) + assert (page.sessions, page.total, page.total_is_capped) == ((), 100, True) + + +@pytest.mark.parametrize( + "sort_desc, status", + [pytest.param(False, None, id="oldest-first"), pytest.param(True, "failure", id="filtered-by-outcome")], +) +@pytest.mark.asyncio +async def test_a_page_that_needs_every_row_reads_them_once(sort_desc: bool, status: str | None) -> None: + rows: Final = tuple((index, f"s{index:05d}", "k") for index in range(500)) + request: Final = _Request(cursor=None, offset=0, page_limit=10, sort_desc=sort_desc, status=status, cap=100) + reads: Final = _Reads(rows, request) + await _session_page(reads, request) + assert reads.budgets == (None,) + + +def test_parse_session_cursor() -> None: + assert _parse_session_cursor(None) is None + assert _parse_session_cursor("no-delimiter") is None + assert _parse_session_cursor("2026-08-29 10:00:00|sess-1") is None + assert _parse_session_cursor("|hashed-key|sess-1") is None + assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|") is None + assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|sess-1") == ( + "2026-08-29 10:00:00", + "sess-1", + "hashed-key", + ) + assert _parse_session_cursor("2026-08-29 10:00:00.123|hashed-key|sess|with|pipes") == ( + "2026-08-29 10:00:00.123", + "sess|with|pipes", + "hashed-key", + ) + + +def _bound(sql: str, args: tuple[object, ...], pattern: str) -> tuple[object, ...]: + """The values bound to the ``$N`` placeholders that ``pattern``'s groups capture in ``sql``.""" + found: Final = re.search(pattern, sql) + assert found, f"{pattern!r} not found in:\n{sql}" + assert all(1 <= int(index) <= len(args) for index in found.groups()), f"{found.groups()} outside $1..${len(args)}" + return tuple(args[int(index) - 1] for index in found.groups()) + + +class _TableDb: + """A table of ``rows`` matching rows in ``sessions`` sessions, whose reads return no page row.""" + + def __init__(self, rows: int, sessions: int) -> None: + self.rows: Final = rows + self.sessions: Final = sessions + self.calls: tuple[tuple[str, tuple[object, ...]], ...] = () + + async def query_raw(self, query: str, *args: object) -> tuple[_ReadRow, ...]: + self.calls = (*self.calls, (query, args)) + budget: Final = re.search(r"DESC LIMIT (\d+)", query) + rows_read: Final = min(int(budget.group(1)), self.rows) if budget else self.rows + return ( + _ReadRow( + session_key=None, + api_key=None, + last_activity=None, + settled=None, + rows_read=rows_read, + session_count=self.sessions, + ), + ) + + +def _page_bounds(sql: str) -> tuple[int, int]: + """The LIMIT and OFFSET of the read's sessions.""" + found: Final = re.search(r"LIMIT (\d+)(?: OFFSET (\d+))?\s*\) AS page", sql) + assert found, sql + return int(found.group(1)), int(found.group(2) or 0) + + +def _read_size(sql: str) -> int | None: + found: Final = re.search(r"DESC LIMIT (\d+)", sql) + return int(found.group(1)) if found else None + + +@pytest.mark.asyncio +async def test_a_read_that_fills_its_size_is_not_taken_for_every_row() -> None: + """A read that took every row it asked for may have stopped short, so its 50 sessions are no total: the reads + grow until one takes fewer rows than it asked for.""" + db: Final = _TableDb(rows=9_000, sessions=50) + page: Final = await read_session_page( + db, + where_clause="TRUE", + params=(), + page=1, + page_size=10, + status=None, + sort_desc=True, + session_cursor=None, + cap=100, + ) + assert tuple(_read_size(sql) for sql, _ in db.calls) == (102, 258, 652, 1647, 4159, 10502) + assert (page.total, page.total_is_capped) == (50, False) + + +@pytest.mark.parametrize("page_number", [pytest.param(5, id="at-the-cap"), pytest.param(6, id="past-the-cap")]) +@pytest.mark.asyncio +async def test_a_page_starting_at_or_past_the_cap_reads_once_and_shows_no_session(page_number: int) -> None: + db: Final = _TableDb(rows=1_000, sessions=150) + page: Final = await read_session_page( + db, + where_clause="TRUE", + params=(), + page=page_number, + page_size=25, + status=None, + sort_desc=True, + session_cursor=None, + cap=100, + ) + assert len(db.calls) == 1, "the first read already holds more sessions than the cap" + assert _page_bounds(db.calls[0][0])[0] == 0 + assert (page.total, page.total_is_capped) == (100, True) + + +@pytest.mark.asyncio +async def test_the_read_binds_the_filters_and_the_cursor_to_their_placeholders() -> None: + db: Final = _TableDb(rows=50, sessions=40) + where, params = "model = $1 AND api_key = $2", ("gpt-4o", "key-a") + for page, page_size, session_cursor in ((1, 3, "2026-10-28 09:00:00|key-a|sess|9"), (3, 1, None)): + await read_session_page( + db, + where_clause=where, + params=params, + page=page, + page_size=page_size, + status=None, + sort_desc=True, + session_cursor=session_cursor, + cap=100, + ) + (cursor_sql, cursor_args), (offset_sql, _) = db.calls + for sql, args in db.calls: + assert {int(index) for index in re.findall(r"\$(\d+)", sql)} == set(range(1, len(args) + 1)), ( + "every argument needs a placeholder and every placeholder an argument" + ) + assert _bound(sql, args, r"model = \$(\d+) AND api_key = \$(\d+)") == params + assert _bound( + cursor_sql, cursor_args, r"\(last_activity, session_key, api_key\) < \(\$(\d+)::timestamp, \$(\d+), \$(\d+)\)" + ) == ("2026-10-28 09:00:00", "sess|9", "key-a") + assert _page_bounds(cursor_sql) == (4, 0) + assert _page_bounds(offset_sql) == (2, 2) + + +@pytest.mark.asyncio +async def test_a_status_filtered_read_binds_the_status_between_the_filters_and_the_cursor() -> None: + db: Final = _TableDb(rows=50, sessions=40) + await read_session_page( + db, + where_clause="model = $1", + params=("gpt-4o",), + status="failure", + page=1, + page_size=3, + sort_desc=True, + session_cursor="2026-10-28 09:00:00|key-a|sess-9", + cap=100, + ) + ((sql, args),) = db.calls + assert {int(index) for index in re.findall(r"\$(\d+)", sql)} == set(range(1, len(args) + 1)) + assert _bound(sql, args, r"model = \$(\d+)") == ("gpt-4o",) + assert _bound(sql, args, r"COALESCE\(status, 'success'\) = \$(\d+)") == ("failure",) + assert _bound(sql, args, r"\(last_activity, session_key, api_key\) < \(\$(\d+)::timestamp, \$(\d+), \$(\d+)\)") == ( + "2026-10-28 09:00:00", + "sess-9", + "key-a", + ) diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 6598b94e60f..f8d0ebb0f77 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -5,7 +5,7 @@ import hashlib import json import re import sqlite3 -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import timezone from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -7151,37 +7151,19 @@ def test_scoped_spend_report_range_at_max_allowed(client, monkeypatch): app.dependency_overrides.pop(ps.user_api_key_auth, None) -def test_parse_session_cursor(): - from litellm.proxy.spend_tracking.spend_management_endpoints import ( - _parse_session_cursor, - ) - - assert _parse_session_cursor(None) is None - assert _parse_session_cursor("no-delimiter") is None - assert _parse_session_cursor("2026-08-29 10:00:00|sess-1") is None - assert _parse_session_cursor("|hashed-key|sess-1") is None - assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|") is None - assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|sess-1") == ( - "2026-08-29 10:00:00", - "sess-1", - "hashed-key", - ) - assert _parse_session_cursor("2026-08-29 10:00:00.123|hashed-key|sess|with|pipes") == ( - "2026-08-29 10:00:00.123", - "sess|with|pipes", - "hashed-key", - ) - - SESSION_GROUP_KEY_SQL = "COALESCE(NULLIF(session_id, ''), request_id), api_key" -def _session_grouped_mock_prisma(session_page_rows, session_total, representative_rows): +def _session_grouped_mock_prisma( + session_page_rows: Sequence[Mapping[str, object]], + session_total: int, + representative_rows: Sequence[Mapping[str, object]], +) -> MagicMock: """Mock prisma dispatching the raw queries the keyset grouped path emits.""" - async def mock_query_raw(sql_query, *params): - if "COUNT(*) AS total_count" in sql_query: - return [{"total_count": session_total}] + async def mock_query_raw(sql_query: str, *params: object) -> Sequence[Mapping[str, object]]: + if "rows_read" in sql_query: + return _session_read_rows(session_page_rows, session_total) if "DISTINCT ON" in sql_query: return representative_rows if "COALESCE(SUM(spend)" in sql_query: @@ -7199,7 +7181,7 @@ def _session_grouped_mock_prisma(session_page_rows, session_total, representativ "session_models": ["gpt-4o"], } ] - return session_page_rows + raise AssertionError(f"unexpected query:\n{sql_query}") mock_prisma = MagicMock() mock_prisma.db = MagicMock() @@ -7228,23 +7210,33 @@ def _session_page_row(session_key, last_activity): return {"session_key": session_key, "api_key": "hashed-key", "last_activity": last_activity} -def _session_grouped_paginating_prisma(sessions, counted_total=None): - """Mock prisma serving the grouped page query out of ``sessions``, honoring the LIMIT and OFFSET it asks for.""" +def _session_read_rows( + page_rows: Sequence[Mapping[str, object]], session_count: int +) -> tuple[Mapping[str, object], ...]: + """The read query's rows for ``page_rows``, as if the read held every matching row.""" + summary: Final = {"rows_read": 0, "session_count": session_count} + if not page_rows: + return ({"session_key": None, "api_key": None, "last_activity": None, "settled": None, **summary},) + return tuple({**row, "settled": True, **summary} for row in page_rows) - async def mock_query_raw(sql_query, *params): - if "COUNT(*) AS total_count" in sql_query: - return [{"total_count": min(len(sessions) if counted_total is None else counted_total, params[-1])}] + +def _session_grouped_paginating_prisma(sessions: Sequence[tuple[str, str]]) -> MagicMock: + """Mock prisma serving the grouped page read out of ``sessions``, honoring the LIMIT and OFFSET it asks for.""" + + async def mock_query_raw(sql_query: str, *params: object) -> Sequence[Mapping[str, object]]: if "DISTINCT ON" in sql_query: return [_session_representative_row(f"req-{session_key}", session_key) for session_key in params[-2]] if "COALESCE(SUM(spend)" in sql_query: return [] - bounds = re.search(r"LIMIT \$(\d+)(?: OFFSET \$(\d+))?", sql_query) - limit = params[int(bounds.group(1)) - 1] - offset = params[int(bounds.group(2)) - 1] if bounds.group(2) else 0 - return [ - _session_page_row(session_key, last_activity) - for session_key, last_activity in sessions[offset : offset + limit] - ] + bounds: Final = re.search(r"LIMIT (\d+)(?: OFFSET (\d+))?\s+\) AS page", sql_query) + limit, offset = int(bounds.group(1)), int(bounds.group(2) or 0) + return _session_read_rows( + [ + _session_page_row(session_key, last_activity) + for session_key, last_activity in sessions[offset : offset + limit] + ], + len(sessions), + ) mock_prisma = MagicMock() mock_prisma.db = MagicMock() @@ -7296,20 +7288,15 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc assert data["data"][1]["session_total_count"] == 1 emitted = [call.args for call in mock_prisma.db.query_raw.await_args_list] + assert len(emitted) == 3, "one read picks the page and counts it, then the representatives and their totals" page_query_sql = emitted[0][0] - assert f"GROUP BY {SESSION_GROUP_KEY_SQL}" in page_query_sql assert "OFFSET" not in page_query_sql - assert "HAVING" not in page_query_sql - assert emitted[0][-1] == 3, "page query fetches page_size + 1 sessions to detect has_more" + assert "LIMIT 3 " in page_query_sql, "the read fetches page_size + 1 sessions to detect has_more" - count_sql = emitted[1][0] - assert f"GROUP BY {SESSION_GROUP_KEY_SQL}" in count_sql - assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded" - - rep_call = emitted[2] + rep_call: Final = emitted[1] assert f"DISTINCT ON ({SESSION_GROUP_KEY_SQL})" in rep_call[0] assert ( - f"ORDER BY {SESSION_GROUP_KEY_SQL}, " + spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL + f"ORDER BY {SESSION_GROUP_KEY_SQL}, " + spend_management_endpoints.SESSION_REPRESENTATIVE_ORDER_SQL ) in rep_call[0] assert rep_call[-2] == ["sess-1", "req-solo"] assert rep_call[-1] == ["hashed-key", "hashed-key"] @@ -7319,7 +7306,7 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc @pytest.mark.asyncio async def test_ui_view_spend_logs_group_by_session_cursor_page(client, monkeypatch): - """A session_cursor becomes a HAVING keyset predicate instead of an OFFSET.""" + """A session_cursor becomes a keyset predicate on the read's sessions instead of an OFFSET.""" page_rows = [_session_page_row("sess-2", "2026-08-29 07:00:00")] reps = [_session_representative_row("req-2", "sess-2")] mock_prisma = _session_grouped_mock_prisma(page_rows, 3, reps) @@ -7352,7 +7339,7 @@ async def test_ui_view_spend_logs_group_by_session_cursor_page(client, monkeypat page_query_call = mock_prisma.db.query_raw.await_args_list[0] page_query_sql = page_query_call.args[0] - assert f'HAVING (MAX("startTime"), {SESSION_GROUP_KEY_SQL}) <' in page_query_sql + assert "WHERE (last_activity, session_key, api_key) <" in page_query_sql assert "OFFSET" not in page_query_sql cursor_index = page_query_call.args.index("2026-08-29 09:00:00") assert page_query_call.args[cursor_index : cursor_index + 3] == ( @@ -7403,9 +7390,9 @@ async def test_ui_view_spend_logs_group_by_session_jumps_to_page_without_cursor( @pytest.mark.asyncio async def test_ui_view_spend_logs_group_by_session_short_page_totals_itself(client, monkeypatch): - """A page that runs out of sessions is the end of the list, so the total comes from it and nothing is counted.""" + """A page that runs out of sessions has read every matching row, so that one read gives the total.""" sessions = tuple((f"sess-{index:02d}", f"2026-08-29 10:{59 - index:02d}:00") for index in range(10)) - mock_prisma = _session_grouped_paginating_prisma(sessions, counted_total=999) + mock_prisma: Final = _session_grouped_paginating_prisma(sessions) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) monkeypatch.setattr( "litellm.proxy.spend_tracking.spend_management_endpoints._is_admin_view_safe", @@ -7429,10 +7416,12 @@ async def test_ui_view_spend_logs_group_by_session_short_page_totals_itself(clie ) assert response.status_code == 200, response.text data = response.json() - assert data["total"] == 10, "the count query's 999 would have won if it had been asked" + assert data["total"] == 10 assert data["total_is_capped"] is False assert data["total_pages"] == 1 assert len(data["data"]) == 10 + reads: Final = tuple(call for call in mock_prisma.db.query_raw.await_args_list if "rows_read" in call.args[0]) + assert len(reads) == 1, "a page that ends the list is not read again for its total" finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) @@ -7517,7 +7506,7 @@ async def test_ui_view_spend_logs_group_by_session_page_past_count_cap_is_empty( assert past_cap_data["data"] == [] assert past_cap_data["has_more"] is False assert past_cap_data["total"] == cap - assert mock_prisma.db.query_raw.await_count == 1, "only the bounded count query runs past the capped window" + assert mock_prisma.db.query_raw.await_count == 1, "only the bounded read runs past the capped window" finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) @@ -7820,14 +7809,14 @@ def test_session_representative_uses_completed_agent_outcome(parent_status, chil ) result = connection.execute( "SELECT request_id, status FROM logs ORDER BY " - + spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL + + spend_management_endpoints.SESSION_REPRESENTATIVE_ORDER_SQL + " LIMIT 1" ).fetchone() assert result == ("parent", expected) connection.execute("DELETE FROM logs WHERE call_type = 'asend_message'") fallback = connection.execute( "SELECT request_id, status FROM logs ORDER BY " - + spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL + + spend_management_endpoints.SESSION_REPRESENTATIVE_ORDER_SQL + " LIMIT 1" ).fetchone() assert fallback == ("llm", "success") diff --git a/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py b/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py index 93fae093340..7a38c1d8457 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py +++ b/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py @@ -6,7 +6,9 @@ GitHub Issue: #17487 """ import datetime +from collections.abc import Mapping, Sequence from datetime import timezone +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -508,20 +510,10 @@ async def test_global_spend_report_team_group_forwards_team_id(monkeypatch): @pytest.mark.asyncio async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): - """ - With group_by_session=true, /spend/logs/ui must page and count SESSIONS, - not raw calls: the page is one representative row per session (DISTINCT - ON the session group key, preferring non-MCP calls, newest first), the - bounded count counts groups, and on the default startTime sort the page is - selected by keyset (no OFFSET) so deep pages do not degrade. Otherwise the - UI collapses a server page of N calls into fewer visible rows while the - footer still claims N (issue #38060). - """ + """With group_by_session=true, /spend/logs/ui pages and counts sessions, not calls, one representative row per + session, so the footer's total matches the rows shown (issue #38060).""" from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth - from litellm.proxy.spend_tracking.spend_management_endpoints import ( - SPEND_LOGS_PAGINATION_COUNT_CAP, - ui_view_spend_logs, - ) + from litellm.proxy.spend_tracking.spend_management_endpoints import ui_view_spend_logs group_key = "COALESCE(NULLIF(session_id, ''), request_id), api_key" session_rows = [ @@ -533,12 +525,12 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): {"request_id": "req-2", "api_key": "k", "metadata": "{}", "session_id": None}, ] - async def mock_query_raw(sql_query, *params): - if "COUNT(*) AS total_count" in sql_query: - return [{"total_count": 60}] + async def mock_query_raw(sql_query: str, *params: object) -> Sequence[Mapping[str, object]]: + if "rows_read" in sql_query: + return [{**row, "settled": True, "rows_read": 0, "session_count": 60} for row in session_rows] if "AS session_representatives" in sql_query: return representative_rows - return session_rows + return [] mock_prisma = MagicMock() mock_prisma.db = MagicMock() @@ -565,22 +557,16 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): user_api_key_dict=auth, group_by_session=True, session_cursor=None, + status_filter=None, ) emitted = [call[0] for call in mock_prisma.db.query_raw.call_args_list] page_sql = emitted[0][0] - assert f"GROUP BY {group_key}" in page_sql, f"page must select sessions, not calls. SQL was:\n{page_sql}" + assert "GROUP BY session_key, api_key" in page_sql, f"page must select sessions, not calls. SQL was:\n{page_sql}" assert "OFFSET" not in page_sql, "the startTime page must be keyset-selected, not offset-selected" - assert emitted[0][-1] == 51, "the page query fetches page_size + 1 sessions to detect has_more" + assert "LIMIT 51 " in page_sql, "the page read fetches page_size + 1 sessions to detect has_more" - count_call = emitted[1] - count_sql = count_call[0] - assert f"GROUP BY {group_key}" in count_sql, f"grouped total must count sessions. SQL was:\n{count_sql}" - assert "COUNT(*) OVER ()" not in count_sql - assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded" - assert count_call[-1] == SPEND_LOGS_PAGINATION_COUNT_CAP + 1 - - rep_sql = emitted[2][0] + rep_sql: Final = emitted[1][0] assert f"DISTINCT ON ({group_key})" in rep_sql, f"page must return one row per session. SQL was:\n{rep_sql}" assert ( f"ORDER BY {group_key}, (call_type = 'asend_message') DESC, " From 083961731c1d2bd59567432556ead3771620f0af Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:32:12 -0800 Subject: [PATCH 008/122] fix(model_prices): consolidate verified provider metadata and pricing (#49361) * fix(databricks): mark audio and video input support on the Gemini 3.5 to 3.8 Flash rows * fix(model_prices): add Vertex AI and Together AI retirement dates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): correct perplexity agent api pricing and add gemini-4-argon, nemotron-3.5-super Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): consolidate verified pricing and Aplomb metadata Absorb verified pricing from #49547 and #49548 and Aplomb 1 metadata from #49539. Preserve provider tiers and remove an identical duplicate retirement key. Co-authored-by: Liutasil501 Co-authored-by: Adam Dalloul <47503782+Adam-Dalloul@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(model_prices): sync generated EmpirioLabs endpoint metadata Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): verify Nebius and EmpirioLabs model metadata Verify canonical EmpirioLabs rows from #49695 against published model pages, preserving pricing tiers and excluding unverified variants and specialty billing. Add Nebius models and cached-input prices from the official catalog and correct the Voyage legacy input limit. Co-authored-by: Adam Dalloul <47503782+Adam-Dalloul@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): absorb Bedrock Opus 5 batch prices, Vertex Omni Flash limits and Gemini 2.5 shutdown dates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): drop the EmpirioLabs rows and endpoint flags from this sync * fix(model_prices): merge main and dedupe the gemini-2.5 deprecation_date keys * fix(model_prices): set supports_video_input on the two new Nebius video models * fix(model_prices): add Azure AI Claude Mythos 5.1 and match Foundry max effort support Co-authored-by: Emerson Gomes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Adam Dalloul <47503782+Adam-Dalloul@users.noreply.github.com> Co-authored-by: Emerson Gomes --- ...odel_prices_and_context_window_backup.json | 340 +++++++++++++----- model_prices_and_context_window.json | 340 +++++++++++++----- ...azure_ai_foundry_catalog_model_metadata.py | 1 + 3 files changed, 507 insertions(+), 174 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 974f1bb0c8b..5a46a0891fc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2135,12 +2135,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.5e-06, + "output_cost_per_token_batches": 1.25e-05 }, "anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2221,12 +2223,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "global.anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.5e-06, + "output_cost_per_token_batches": 1.25e-05 }, "global.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2307,12 +2311,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "us.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2393,12 +2399,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/eu-west-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "eu-west-1" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "eu.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2474,12 +2482,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-southeast-2/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "ap-southeast-4" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "au.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2560,7 +2570,9 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-northeast-1/index.json", + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "jp.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -3949,7 +3961,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true, + "supports_max_reasoning_effort": false, "prompt_cache_min_tokens": 512 }, "azure_ai/claude-fable-5-1": { @@ -3983,10 +3995,40 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true, + "supports_max_reasoning_effort": false, "prompt_cache_min_tokens": 512, "deprecation_date": "2027-12-05" }, + "azure_ai/claude-mythos-5-1": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 2.5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": false, + "prompt_cache_min_tokens": 512, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models" + }, "azure_ai/claude-opus-5": { "deprecation_date": "2027-07-08", "supports_mid_conversation_system": true, @@ -22269,7 +22311,7 @@ }, "mode": "chat", "output_cost_per_token": 3.74997e-06, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22287,7 +22329,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-7-flash": { "cache_creation_input_token_cost": 7.4998e-07, @@ -22301,7 +22345,7 @@ }, "mode": "chat", "output_cost_per_token": 3.74997e-06, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22319,7 +22363,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-6-flash": { "cache_creation_input_token_cost": 7.4998e-07, @@ -22335,7 +22381,7 @@ "mode": "chat", "output_cost_per_token": 3.74997e-06, "output_dbu_cost_per_token": 5.3571e-05, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22354,7 +22400,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-5-flash": { "cache_creation_input_token_cost": 1.87502e-06, @@ -22370,7 +22418,7 @@ "mode": "chat", "output_cost_per_token": 1.124998e-05, "output_dbu_cost_per_token": 0.000160714, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22388,7 +22436,9 @@ "input_cost_per_token_priority": 3.37498e-06, "output_cost_per_token_priority": 2.025002e-05, "cache_read_input_token_cost_priority": 3.3747e-07, - "cache_creation_input_token_cost_priority": 3.37498e-06 + "cache_creation_input_token_cost_priority": 3.37498e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-5-flash-lite": { "cache_creation_input_token_cost": 3.7499e-07, @@ -22404,7 +22454,7 @@ "mode": "chat", "output_cost_per_token": 3.12501e-06, "output_dbu_cost_per_token": 4.4643e-05, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22424,7 +22474,9 @@ "input_cost_per_token_priority": 6.7501e-07, "output_cost_per_token_priority": 5.62499e-06, "cache_read_input_token_cost_priority": 6.748e-08, - "cache_creation_input_token_cost_priority": 6.7501e-07 + "cache_creation_input_token_cost_priority": 6.7501e-07, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemma-3-12b": { "cache_creation_input_token_cost": 1.5001e-07, @@ -24167,7 +24219,7 @@ "max_output_tokens": 163840, "input_cost_per_token": 5e-07, "output_cost_per_token": 2.15e-06, - "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost": 3.5e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, @@ -24241,7 +24293,7 @@ "max_output_tokens": 163840, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 9.5e-07, - "cache_read_input_token_cost": 2.16e-07, + "cache_read_input_token_cost": 1.3e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, @@ -29639,6 +29691,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-lite-image": { + "deprecation_date": "2027-06-28", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, @@ -30638,6 +30691,7 @@ "google_maps_grounding_cost_per_query": 0.014 }, "vertex_ai/gemini-3.8-flash": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, @@ -30744,6 +30798,7 @@ "supports_native_streaming": true }, "vertex_ai/gemini-3.8-live": { + "deprecation_date": "2027-09-25", "gemini_audio_only_live": true, "input_cost_per_audio_token": 3e-06, "input_cost_per_image_token": 1e-06, @@ -32706,13 +32761,13 @@ "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_input_tokens": 131072, + "max_output_tokens": 57920, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-flash-preview", "supported_endpoints": [ "/v1/chat/completions" ], @@ -32730,7 +32785,8 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-11-06" }, "gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, @@ -32918,6 +32974,7 @@ "google_maps_grounding_cost_per_query": 0.014 }, "gemini-3.8-flash": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, @@ -44265,7 +44322,8 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/google%2Fgemma-3-27b-it" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 4e-08 }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { "max_input_tokens": 131072, @@ -44491,7 +44549,7 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash" }, "nebius/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, @@ -44499,7 +44557,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash-0731" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-08 }, "nebius/deepseek-ai/DeepSeek-V4-Pro": { "max_input_tokens": 1048576, @@ -44510,29 +44569,32 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.4e-07 }, "nebius/deepseek-ai/DeepSeek-V4-Pro-0813": { "input_cost_per_token": 1.32e-06, "litellm_provider": "nebius", - "max_input_tokens": 979000, + "max_input_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.96e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro-0813", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 4e-08 }, "nebius/deepseek-ai/DeepSeek-V4.1-Flash": { "input_cost_per_token": 3e-07, "litellm_provider": "nebius", - "max_input_tokens": 1048000, + "max_input_tokens": 1048576, "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://tokenfactory.nebius.com/endpoints?modals=endpoint-details&model-id=deepseek-ai/DeepSeek-V4.1-Flash", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1e-08 }, "nebius/MiniMaxAI/MiniMax-M2.5": { "max_input_tokens": 196608, @@ -44555,7 +44617,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M3" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 6e-08 }, "nebius/moonshotai/Kimi-K2.6": { "max_input_tokens": 262144, @@ -44567,7 +44630,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K2.6" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.6e-07 }, "nebius/moonshotai/Kimi-K2.7-Code": { "max_input_tokens": 262144, @@ -44579,10 +44643,11 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/moonshotai%2FKimi-K2.7-Code" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.5e-07 }, "nebius/moonshotai/Kimi-K3": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, @@ -44591,7 +44656,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K3" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 3e-07 }, "nebius/NousResearch/Hermes-4-405B": { "max_input_tokens": 131072, @@ -44602,7 +44668,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-405B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 2e-07 }, "nebius/NousResearch/Hermes-4-70B": { "max_input_tokens": 131072, @@ -44646,7 +44713,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNVIDIA-Nemotron-3-Nano-30B-A3B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-08 }, "nebius/nvidia/Nemotron-3-Nano-Omni": { "max_input_tokens": 262144, @@ -44669,7 +44737,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2Fnemotron-3-super-120b-a12b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 6e-08 }, "nebius/nvidia/Nemotron-3-Ultra-550b-a55b": { "max_input_tokens": 1048576, @@ -44680,7 +44749,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Ultra-550b-a55b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-07 }, "nebius/nvidia/Nemotron-3_5-Lightning": { "max_input_tokens": 1048576, @@ -44691,7 +44761,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3_5-Lightning" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 3e-08 }, "nebius/openai/gpt-oss-120b": { "max_input_tokens": 131072, @@ -44702,7 +44773,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/openai%2Fgpt-oss-120b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-07 }, "nebius/openbmb/MiniCPM-V-4_5": { "max_input_tokens": 32000, @@ -44712,7 +44784,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/openbmb%2FMiniCPM-V-4_5" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.3e-07 }, "nebius/Qwen/Qwen3-235B-A22B-Instruct-2507": { "max_input_tokens": 262144, @@ -44722,7 +44795,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-235B-A22B-Instruct-2507" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 5e-08 }, "nebius/Qwen/Qwen3-30B-A3B-Instruct-2507": { "max_input_tokens": 262144, @@ -44732,7 +44806,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-30B-A3B-Instruct-2507" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-07 }, "nebius/Qwen/Qwen3-Next-80B-A3B-Thinking": { "max_input_tokens": 128000, @@ -44754,7 +44829,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.5-397B-A17B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-07 }, "nebius/Qwen/Qwen3.8-27B": { "input_cost_per_token": 4.5e-07, @@ -44762,9 +44838,10 @@ "max_input_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.8-27B", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 1e-07 }, "nebius/zai-org/GLM-5.1": { "max_input_tokens": 202752, @@ -44775,7 +44852,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.1" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 2.6e-07 }, "nebius/zai-org/GLM-5.2": { "max_input_tokens": 1048576, @@ -44786,21 +44864,23 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.2" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.4e-07 }, "nebius/zai-org/GLM-5.3": { "input_cost_per_token": 1.4e-06, "litellm_provider": "nebius", - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 2.6e-07 }, "nebius/zai-org/GLM-5.3-Flash": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 5e-07, @@ -44808,8 +44888,9 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3-Flash", - "supports_vision": true + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "supports_vision": true, + "cache_read_input_token_cost": 3e-08 }, "nebius/BAAI/bge-en-icl": { "max_input_tokens": 32768, @@ -49813,7 +49894,7 @@ "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -49835,7 +49916,9 @@ "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-gov-west-1" - } + }, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05 }, "us-gov.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -52922,6 +53005,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-image": { + "deprecation_date": "2027-06-28", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, @@ -54150,7 +54234,7 @@ "voyage/voyage-lite-01": { "input_cost_per_token": 1e-07, "litellm_provider": "voyage", - "max_input_tokens": 4096, + "max_input_tokens": 4000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -64809,7 +64893,7 @@ "supports_vision": false }, "novita/deepseek/deepseek-v4-pro-0813": { - "cache_read_input_token_cost": 1.3200000000000002e-07, + "cache_read_input_token_cost": 4.4e-08, "input_cost_per_token": 1.32e-06, "litellm_provider": "novita", "max_input_tokens": 1048576, @@ -64921,13 +65005,13 @@ "supports_vision": false }, "novita/mindai/macaron-v1-venti": { - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 1.5e-06, + "cache_read_input_token_cost": 1.65e-07, + "input_cost_per_token": 8.25e-07, "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, "mode": "chat", - "output_cost_per_token": 4.5e-06, + "output_cost_per_token": 2.475e-06, "source": "https://novita.ai/pricing", "supports_function_calling": true, "supports_prompt_caching": true, @@ -65030,13 +65114,13 @@ "supports_vision": false }, "novita/mindai/macaron-v1-tall": { - "cache_read_input_token_cost": 8e-08, - "input_cost_per_token": 4.5000000000000003e-07, + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 2.475e-07, "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, "mode": "chat", - "output_cost_per_token": 2.6e-06, + "output_cost_per_token": 1.43e-06, "source": "https://novita.ai/pricing", "supports_function_calling": true, "supports_prompt_caching": true, @@ -66461,9 +66545,9 @@ "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { "max_output_tokens": 1048576, "max_input_tokens": 1048576, - "input_cost_per_token": 8e-08, + "input_cost_per_token": 6e-08, "output_cost_per_token": 1.8e-07, - "cache_read_input_token_cost": 1.6e-08, + "cache_read_input_token_cost": 1.5e-08, "supports_prompt_caching": true, "litellm_provider": "deepinfra", "mode": "chat", @@ -66842,8 +66926,8 @@ "deepinfra/google/gemma-4-31B-it": { "max_output_tokens": 262144, "max_input_tokens": 262144, - "input_cost_per_token": 1.3e-07, - "output_cost_per_token": 3.8e-07, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_function_calling": true, @@ -68554,7 +68638,10 @@ "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-gov-west-1" - } + }, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json" }, "bedrock/us-gov-west-1/anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -68829,7 +68916,10 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json" }, "bedrock/us-gov-east-1/anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -69852,6 +69942,14 @@ "cache_read_input_token_cost": 3e-08, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, + "perplexity/google/gemini-4-argon": { + "litellm_provider": "perplexity", + "mode": "responses", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, "perplexity/google/gemini-3.6-flash": { "litellm_provider": "perplexity", "mode": "responses", @@ -69979,6 +70077,14 @@ "cache_read_input_token_cost": 1.15e-09, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, + "perplexity/perplexity/nemotron-3.5-super-120b-a12b": { + "litellm_provider": "perplexity", + "mode": "responses", + "input_cost_per_token": 2e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 4e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, "perplexity/perplexity/nemotron-3-ultra-550b-a55b": { "litellm_provider": "perplexity", "mode": "responses", @@ -70104,12 +70210,12 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai", "max_output_tokens": 57920, - "max_input_tokens": 57920, + "max_input_tokens": 131072, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-1-1-flash", "supported_endpoints": [ "/v1beta/interactions" ], @@ -74505,12 +74611,14 @@ "input_cost_per_token": 5.5e-06, "cache_read_input_token_cost": 5.5e-07, "output_cost_per_token": 2.75e-05, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-northeast-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "ap-southeast-4" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "apac.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -75107,7 +75215,7 @@ "supports_function_calling": true, "input_cost_per_token": 2e-06, "output_cost_per_token": 1e-05, - "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost": 1e-07, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/anthropic/claude-fable-5-1": { @@ -75155,6 +75263,9 @@ "output_cost_per_token": 5e-05, "cache_read_input_token_cost": 1e-06, "cache_creation_input_token_cost": 1.25e-05, + "input_cost_per_token_above_272k_tokens": 2e-05, + "output_cost_per_token_above_272k_tokens": 7.5e-05, + "cache_read_input_token_cost_above_272k_tokens": 2e-06, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/openai/gpt-6-luna": { @@ -76622,6 +76733,9 @@ "input_cost_per_token": 1e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, + "input_cost_per_token_above_100k_tokens": 5e-07, + "output_cost_per_token_above_100k_tokens": 2.5e-06, + "cache_read_input_token_cost_above_100k_tokens": 5e-08, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "us-gov.anthropic.claude-haiku-5-5": { @@ -77096,6 +77210,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-4-argon": { + "deprecation_date": "2027-10-13", "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, @@ -77504,9 +77619,10 @@ "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/reflection%2Fbeam", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 5e-08 }, "baseten/deepseek-ai/DeepSeek-V4.1-Flash-1017": { "cache_read_input_token_cost": 7e-09, @@ -81777,5 +81893,55 @@ "supports_response_schema": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true + }, + "nebius/deepseek-ai/DeepSeek-V4.1-Flash-1017": { + "max_input_tokens": 1048576, + "input_cost_per_token": 3e-07, + "cache_read_input_token_cost": 6e-09, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/meta-models/muse-spark-open": { + "max_input_tokens": 1048576, + "input_cost_per_token": 1.25e-06, + "cache_read_input_token_cost": 1.5e-07, + "output_cost_per_token": 4.25e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_video_input": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/MiniMaxAI/MiniMax-M3.1-Flash": { + "max_input_tokens": 1048576, + "input_cost_per_token": 1.5e-07, + "cache_read_input_token_cost": 1.5e-08, + "output_cost_per_token": 6e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/nvidia/Nemotron-3.5-Super-VL": { + "max_input_tokens": 1048576, + "input_cost_per_token": 3e-07, + "cache_read_input_token_cost": 6e-08, + "output_cost_per_token": 9e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_video_input": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 974f1bb0c8b..5a46a0891fc 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -2135,12 +2135,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.5e-06, + "output_cost_per_token_batches": 1.25e-05 }, "anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2221,12 +2223,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "global.anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.5e-06, + "output_cost_per_token_batches": 1.25e-05 }, "global.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2307,12 +2311,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "us.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2393,12 +2399,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/eu-west-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "eu-west-1" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "eu.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2474,12 +2482,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-southeast-2/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "ap-southeast-4" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "au.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2560,7 +2570,9 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-northeast-1/index.json", + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "jp.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -3949,7 +3961,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true, + "supports_max_reasoning_effort": false, "prompt_cache_min_tokens": 512 }, "azure_ai/claude-fable-5-1": { @@ -3983,10 +3995,40 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true, + "supports_max_reasoning_effort": false, "prompt_cache_min_tokens": 512, "deprecation_date": "2027-12-05" }, + "azure_ai/claude-mythos-5-1": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 2.5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": false, + "prompt_cache_min_tokens": 512, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models" + }, "azure_ai/claude-opus-5": { "deprecation_date": "2027-07-08", "supports_mid_conversation_system": true, @@ -22269,7 +22311,7 @@ }, "mode": "chat", "output_cost_per_token": 3.74997e-06, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22287,7 +22329,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-7-flash": { "cache_creation_input_token_cost": 7.4998e-07, @@ -22301,7 +22345,7 @@ }, "mode": "chat", "output_cost_per_token": 3.74997e-06, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22319,7 +22363,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-6-flash": { "cache_creation_input_token_cost": 7.4998e-07, @@ -22335,7 +22381,7 @@ "mode": "chat", "output_cost_per_token": 3.74997e-06, "output_dbu_cost_per_token": 5.3571e-05, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22354,7 +22400,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-5-flash": { "cache_creation_input_token_cost": 1.87502e-06, @@ -22370,7 +22418,7 @@ "mode": "chat", "output_cost_per_token": 1.124998e-05, "output_dbu_cost_per_token": 0.000160714, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22388,7 +22436,9 @@ "input_cost_per_token_priority": 3.37498e-06, "output_cost_per_token_priority": 2.025002e-05, "cache_read_input_token_cost_priority": 3.3747e-07, - "cache_creation_input_token_cost_priority": 3.37498e-06 + "cache_creation_input_token_cost_priority": 3.37498e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-5-flash-lite": { "cache_creation_input_token_cost": 3.7499e-07, @@ -22404,7 +22454,7 @@ "mode": "chat", "output_cost_per_token": 3.12501e-06, "output_dbu_cost_per_token": 4.4643e-05, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22424,7 +22474,9 @@ "input_cost_per_token_priority": 6.7501e-07, "output_cost_per_token_priority": 5.62499e-06, "cache_read_input_token_cost_priority": 6.748e-08, - "cache_creation_input_token_cost_priority": 6.7501e-07 + "cache_creation_input_token_cost_priority": 6.7501e-07, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemma-3-12b": { "cache_creation_input_token_cost": 1.5001e-07, @@ -24167,7 +24219,7 @@ "max_output_tokens": 163840, "input_cost_per_token": 5e-07, "output_cost_per_token": 2.15e-06, - "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost": 3.5e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, @@ -24241,7 +24293,7 @@ "max_output_tokens": 163840, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 9.5e-07, - "cache_read_input_token_cost": 2.16e-07, + "cache_read_input_token_cost": 1.3e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, @@ -29639,6 +29691,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-lite-image": { + "deprecation_date": "2027-06-28", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, @@ -30638,6 +30691,7 @@ "google_maps_grounding_cost_per_query": 0.014 }, "vertex_ai/gemini-3.8-flash": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, @@ -30744,6 +30798,7 @@ "supports_native_streaming": true }, "vertex_ai/gemini-3.8-live": { + "deprecation_date": "2027-09-25", "gemini_audio_only_live": true, "input_cost_per_audio_token": 3e-06, "input_cost_per_image_token": 1e-06, @@ -32706,13 +32761,13 @@ "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_input_tokens": 131072, + "max_output_tokens": 57920, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-flash-preview", "supported_endpoints": [ "/v1/chat/completions" ], @@ -32730,7 +32785,8 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-11-06" }, "gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, @@ -32918,6 +32974,7 @@ "google_maps_grounding_cost_per_query": 0.014 }, "gemini-3.8-flash": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, @@ -44265,7 +44322,8 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/google%2Fgemma-3-27b-it" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 4e-08 }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { "max_input_tokens": 131072, @@ -44491,7 +44549,7 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash" }, "nebius/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, @@ -44499,7 +44557,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash-0731" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-08 }, "nebius/deepseek-ai/DeepSeek-V4-Pro": { "max_input_tokens": 1048576, @@ -44510,29 +44569,32 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.4e-07 }, "nebius/deepseek-ai/DeepSeek-V4-Pro-0813": { "input_cost_per_token": 1.32e-06, "litellm_provider": "nebius", - "max_input_tokens": 979000, + "max_input_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.96e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro-0813", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 4e-08 }, "nebius/deepseek-ai/DeepSeek-V4.1-Flash": { "input_cost_per_token": 3e-07, "litellm_provider": "nebius", - "max_input_tokens": 1048000, + "max_input_tokens": 1048576, "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://tokenfactory.nebius.com/endpoints?modals=endpoint-details&model-id=deepseek-ai/DeepSeek-V4.1-Flash", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1e-08 }, "nebius/MiniMaxAI/MiniMax-M2.5": { "max_input_tokens": 196608, @@ -44555,7 +44617,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M3" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 6e-08 }, "nebius/moonshotai/Kimi-K2.6": { "max_input_tokens": 262144, @@ -44567,7 +44630,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K2.6" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.6e-07 }, "nebius/moonshotai/Kimi-K2.7-Code": { "max_input_tokens": 262144, @@ -44579,10 +44643,11 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/moonshotai%2FKimi-K2.7-Code" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.5e-07 }, "nebius/moonshotai/Kimi-K3": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, @@ -44591,7 +44656,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K3" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 3e-07 }, "nebius/NousResearch/Hermes-4-405B": { "max_input_tokens": 131072, @@ -44602,7 +44668,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-405B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 2e-07 }, "nebius/NousResearch/Hermes-4-70B": { "max_input_tokens": 131072, @@ -44646,7 +44713,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNVIDIA-Nemotron-3-Nano-30B-A3B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-08 }, "nebius/nvidia/Nemotron-3-Nano-Omni": { "max_input_tokens": 262144, @@ -44669,7 +44737,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2Fnemotron-3-super-120b-a12b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 6e-08 }, "nebius/nvidia/Nemotron-3-Ultra-550b-a55b": { "max_input_tokens": 1048576, @@ -44680,7 +44749,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Ultra-550b-a55b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-07 }, "nebius/nvidia/Nemotron-3_5-Lightning": { "max_input_tokens": 1048576, @@ -44691,7 +44761,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3_5-Lightning" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 3e-08 }, "nebius/openai/gpt-oss-120b": { "max_input_tokens": 131072, @@ -44702,7 +44773,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/openai%2Fgpt-oss-120b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-07 }, "nebius/openbmb/MiniCPM-V-4_5": { "max_input_tokens": 32000, @@ -44712,7 +44784,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/openbmb%2FMiniCPM-V-4_5" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.3e-07 }, "nebius/Qwen/Qwen3-235B-A22B-Instruct-2507": { "max_input_tokens": 262144, @@ -44722,7 +44795,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-235B-A22B-Instruct-2507" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 5e-08 }, "nebius/Qwen/Qwen3-30B-A3B-Instruct-2507": { "max_input_tokens": 262144, @@ -44732,7 +44806,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-30B-A3B-Instruct-2507" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-07 }, "nebius/Qwen/Qwen3-Next-80B-A3B-Thinking": { "max_input_tokens": 128000, @@ -44754,7 +44829,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.5-397B-A17B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-07 }, "nebius/Qwen/Qwen3.8-27B": { "input_cost_per_token": 4.5e-07, @@ -44762,9 +44838,10 @@ "max_input_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.8-27B", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 1e-07 }, "nebius/zai-org/GLM-5.1": { "max_input_tokens": 202752, @@ -44775,7 +44852,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.1" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 2.6e-07 }, "nebius/zai-org/GLM-5.2": { "max_input_tokens": 1048576, @@ -44786,21 +44864,23 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.2" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.4e-07 }, "nebius/zai-org/GLM-5.3": { "input_cost_per_token": 1.4e-06, "litellm_provider": "nebius", - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 2.6e-07 }, "nebius/zai-org/GLM-5.3-Flash": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 5e-07, @@ -44808,8 +44888,9 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3-Flash", - "supports_vision": true + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "supports_vision": true, + "cache_read_input_token_cost": 3e-08 }, "nebius/BAAI/bge-en-icl": { "max_input_tokens": 32768, @@ -49813,7 +49894,7 @@ "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -49835,7 +49916,9 @@ "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-gov-west-1" - } + }, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05 }, "us-gov.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -52922,6 +53005,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-image": { + "deprecation_date": "2027-06-28", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, @@ -54150,7 +54234,7 @@ "voyage/voyage-lite-01": { "input_cost_per_token": 1e-07, "litellm_provider": "voyage", - "max_input_tokens": 4096, + "max_input_tokens": 4000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -64809,7 +64893,7 @@ "supports_vision": false }, "novita/deepseek/deepseek-v4-pro-0813": { - "cache_read_input_token_cost": 1.3200000000000002e-07, + "cache_read_input_token_cost": 4.4e-08, "input_cost_per_token": 1.32e-06, "litellm_provider": "novita", "max_input_tokens": 1048576, @@ -64921,13 +65005,13 @@ "supports_vision": false }, "novita/mindai/macaron-v1-venti": { - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 1.5e-06, + "cache_read_input_token_cost": 1.65e-07, + "input_cost_per_token": 8.25e-07, "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, "mode": "chat", - "output_cost_per_token": 4.5e-06, + "output_cost_per_token": 2.475e-06, "source": "https://novita.ai/pricing", "supports_function_calling": true, "supports_prompt_caching": true, @@ -65030,13 +65114,13 @@ "supports_vision": false }, "novita/mindai/macaron-v1-tall": { - "cache_read_input_token_cost": 8e-08, - "input_cost_per_token": 4.5000000000000003e-07, + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 2.475e-07, "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, "mode": "chat", - "output_cost_per_token": 2.6e-06, + "output_cost_per_token": 1.43e-06, "source": "https://novita.ai/pricing", "supports_function_calling": true, "supports_prompt_caching": true, @@ -66461,9 +66545,9 @@ "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { "max_output_tokens": 1048576, "max_input_tokens": 1048576, - "input_cost_per_token": 8e-08, + "input_cost_per_token": 6e-08, "output_cost_per_token": 1.8e-07, - "cache_read_input_token_cost": 1.6e-08, + "cache_read_input_token_cost": 1.5e-08, "supports_prompt_caching": true, "litellm_provider": "deepinfra", "mode": "chat", @@ -66842,8 +66926,8 @@ "deepinfra/google/gemma-4-31B-it": { "max_output_tokens": 262144, "max_input_tokens": 262144, - "input_cost_per_token": 1.3e-07, - "output_cost_per_token": 3.8e-07, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_function_calling": true, @@ -68554,7 +68638,10 @@ "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-gov-west-1" - } + }, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json" }, "bedrock/us-gov-west-1/anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -68829,7 +68916,10 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json" }, "bedrock/us-gov-east-1/anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -69852,6 +69942,14 @@ "cache_read_input_token_cost": 3e-08, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, + "perplexity/google/gemini-4-argon": { + "litellm_provider": "perplexity", + "mode": "responses", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, "perplexity/google/gemini-3.6-flash": { "litellm_provider": "perplexity", "mode": "responses", @@ -69979,6 +70077,14 @@ "cache_read_input_token_cost": 1.15e-09, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, + "perplexity/perplexity/nemotron-3.5-super-120b-a12b": { + "litellm_provider": "perplexity", + "mode": "responses", + "input_cost_per_token": 2e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 4e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, "perplexity/perplexity/nemotron-3-ultra-550b-a55b": { "litellm_provider": "perplexity", "mode": "responses", @@ -70104,12 +70210,12 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai", "max_output_tokens": 57920, - "max_input_tokens": 57920, + "max_input_tokens": 131072, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-1-1-flash", "supported_endpoints": [ "/v1beta/interactions" ], @@ -74505,12 +74611,14 @@ "input_cost_per_token": 5.5e-06, "cache_read_input_token_cost": 5.5e-07, "output_cost_per_token": 2.75e-05, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-northeast-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "ap-southeast-4" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "apac.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -75107,7 +75215,7 @@ "supports_function_calling": true, "input_cost_per_token": 2e-06, "output_cost_per_token": 1e-05, - "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost": 1e-07, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/anthropic/claude-fable-5-1": { @@ -75155,6 +75263,9 @@ "output_cost_per_token": 5e-05, "cache_read_input_token_cost": 1e-06, "cache_creation_input_token_cost": 1.25e-05, + "input_cost_per_token_above_272k_tokens": 2e-05, + "output_cost_per_token_above_272k_tokens": 7.5e-05, + "cache_read_input_token_cost_above_272k_tokens": 2e-06, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/openai/gpt-6-luna": { @@ -76622,6 +76733,9 @@ "input_cost_per_token": 1e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, + "input_cost_per_token_above_100k_tokens": 5e-07, + "output_cost_per_token_above_100k_tokens": 2.5e-06, + "cache_read_input_token_cost_above_100k_tokens": 5e-08, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "us-gov.anthropic.claude-haiku-5-5": { @@ -77096,6 +77210,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-4-argon": { + "deprecation_date": "2027-10-13", "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, @@ -77504,9 +77619,10 @@ "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/reflection%2Fbeam", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 5e-08 }, "baseten/deepseek-ai/DeepSeek-V4.1-Flash-1017": { "cache_read_input_token_cost": 7e-09, @@ -81777,5 +81893,55 @@ "supports_response_schema": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true + }, + "nebius/deepseek-ai/DeepSeek-V4.1-Flash-1017": { + "max_input_tokens": 1048576, + "input_cost_per_token": 3e-07, + "cache_read_input_token_cost": 6e-09, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/meta-models/muse-spark-open": { + "max_input_tokens": 1048576, + "input_cost_per_token": 1.25e-06, + "cache_read_input_token_cost": 1.5e-07, + "output_cost_per_token": 4.25e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_video_input": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/MiniMaxAI/MiniMax-M3.1-Flash": { + "max_input_tokens": 1048576, + "input_cost_per_token": 1.5e-07, + "cache_read_input_token_cost": 1.5e-08, + "output_cost_per_token": 6e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/nvidia/Nemotron-3.5-Super-VL": { + "max_input_tokens": 1048576, + "input_cost_per_token": 3e-07, + "cache_read_input_token_cost": 6e-08, + "output_cost_per_token": 9e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_video_input": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" } } diff --git a/tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py b/tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py index 9b20192c3f2..a5b66efe7cd 100644 --- a/tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py +++ b/tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py @@ -22,6 +22,7 @@ TOKEN_PRICED_NAMES: Final = ( "cohere-command-a", "grok-4-20-reasoning", "grok-4-20-non-reasoning", + "claude-mythos-5-1", ) GROK_4_20_NAMES: Final = ("grok-4-20-reasoning", "grok-4-20-non-reasoning") CATALOG_NAMES: Final = TOKEN_PRICED_NAMES + ("whisper",) From cba5bcb17e31906541afb1cc8e90e18a3b27a771 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 3 Nov 2026 11:51:44 -0800 Subject: [PATCH 009/122] feat(model-catalog): read successor_model, the provider's recommended replacement for a deprecating row (#49791) * feat(model-catalog): read successor_model, the provider's recommended replacement for a deprecating row * test(cost-map): allow successor_model in the cost map row schema --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/crates/model-catalog/src/wire/attributes.rs | 4 ++++ tests/unit/test_utils.py | 1 + 2 files changed, 5 insertions(+) diff --git a/litellm-rust/crates/model-catalog/src/wire/attributes.rs b/litellm-rust/crates/model-catalog/src/wire/attributes.rs index 307f25f0bda..d2ba5f609ee 100644 --- a/litellm-rust/crates/model-catalog/src/wire/attributes.rs +++ b/litellm-rust/crates/model-catalog/src/wire/attributes.rs @@ -93,6 +93,8 @@ attributes! { search_free_requests: SearchFreeRequests, /// URL of the provider pricing/model page this entry was taken from. source: String, + /// Model the provider recommends migrating to before `deprecation_date`, as a catalog key when one exists. + successor_model: String, /// Audio container formats the model can return. supported_audio_formats: Vec, /// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions. @@ -303,6 +305,8 @@ mod tests { #[case::known_mode("mode", json!("chat"), true)] #[case::unknown_mode("mode", json!("chatt"), false)] #[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)] + #[case::successor_model("successor_model", json!("gpt-6-luna"), true)] + #[case::successor_model_not_a_name("successor_model", json!(["gpt-6-luna"]), false)] #[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)] #[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)] #[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)] diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index c1c2593c4ef..f12280545b7 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1026,6 +1026,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "cache_read_input_image_token_cost": {"type": "number"}, "audio_transcription_config": {"type": "string"}, "deprecation_date": {"type": "string"}, + "successor_model": {"type": "string"}, "input_cost_per_audio_per_second": {"type": "number"}, "input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"}, "google_maps_grounding_cost_per_query": {"type": "number"}, From f52e5057f9f352bc7c48437246a512e8f2124be8 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 3 Nov 2026 11:52:52 -0800 Subject: [PATCH 010/122] refactor(ui): move teams, users and orgs text onto i18n (#49484) * refactor(ui): move teams, users and orgs text onto i18n Moves the page-authored English on /teams, /users and /organizations, plus the shared member table, user search modal, model select and budget schedule helpers they render, into react-i18next t() calls backed by en.json. The rendered English is unchanged Helpers that files outside this batch or untouched tests still call keep their signatures and take an optional t that defaults to the shared instance, the same as usageUnits.ts. Text that existing tests pin, values compared against the API, code samples, CSV column names and date formats stay as they were * refactor(ui): escape member ids in team member dialogs and merge split org and team strings The reset spend and reset budget dialogs interpolate the member id into a Trans string, so an id containing strong, i, p or br tags rendered as markup. Escaping the values and unescaping text nodes keeps it literal, as it was before the move to i18n. Also merges the org budget lines and the team metadata hint into whole sentences and drops two constants nothing imports anymore * refactor(ui): pass the member id into the team member reset dialogs as an element Escaping the id still let Trans fill in placeholders like {{budget}} written inside it a second time. Rendering the id through its own component keeps it out of the translated string, so any id shows exactly as it did before the move to i18n --- .../organizations/OrganizationFilters.tsx | 6 +- .../_components/OrganizationsPanel.tsx | 39 +- .../_components/OrganizationsTable.tsx | 15 +- .../_components/OrganizationsTableColumns.tsx | 71 +- .../users/_components/BulkEditUsers.tsx | 79 +- .../DefaultUserSettingsForm.tsx | 176 +- .../default-user-settings/schema.ts | 17 +- .../users/_components/user_edit_view.tsx | 100 +- .../users/_components/view_users.tsx | 38 +- .../_components/view_users/UsersTable.tsx | 56 +- .../view_users/UsersTableColumns.tsx | 110 +- .../_components/view_users/user_info_view.tsx | 187 +- .../src/components/CreateUserButton.tsx | 98 +- .../components/ModelSelect/ModelSelect.tsx | 47 +- .../src/components/TeamSSOSettings.tsx | 82 +- ui/litellm-dashboard/src/components/Teams.tsx | 319 ++-- .../TeamsPage/TeamMemberSpendBudgetCell.tsx | 29 +- .../src/components/TeamsPage/TeamsTable.tsx | 48 +- .../components/TeamsPage/teamTableColumns.tsx | 91 +- .../components/bulk_create_users_button.tsx | 220 +-- .../Filters/FiltersButton.tsx | 11 +- .../Filters/ResetFiltersButton.tsx | 6 +- .../common_components/MemberTable.tsx | 50 +- .../common_components/user_search_modal.tsx | 56 +- .../org-create/OrgCreateDialog.tsx | 62 +- .../org-settings/OrgSettingsForm.tsx | 66 +- .../organization/org-settings/schema.ts | 44 +- .../organization/organization_view.tsx | 152 +- .../components/team/AvailableTeamsPanel.tsx | 6 +- .../components/team/AvailableTeamsTable.tsx | 31 +- .../team/AvailableTeamsTableColumns.tsx | 33 +- .../src/components/team/EditMembership.tsx | 49 +- .../src/components/team/GuardrailsSelect.tsx | 25 +- .../src/components/team/LoggingSettings.tsx | 90 +- .../src/components/team/MyUserTab.tsx | 52 +- .../team/ResetMemberBudgetsDialog.tsx | 21 +- .../components/team/TeamAdminSettingsForm.tsx | 30 +- .../src/components/team/TeamInfo.tsx | 823 ++++---- .../src/components/team/TeamMemberTab.tsx | 142 +- .../components/team/TeamVirtualKeysTable.tsx | 190 +- .../src/components/team/memberFormValues.ts | 3 +- .../components/team/member_permissions.tsx | 30 +- .../components/team/teamAdminEditAccess.ts | 46 +- .../src/components/team/teamModelAccess.ts | 27 +- .../components/team/useMemberBudgetReset.ts | 17 +- ui/litellm-dashboard/src/i18n/locales/en.json | 1651 +++++++++++++++++ ui/litellm-dashboard/src/utils/budgetUtils.ts | 24 +- 47 files changed, 3957 insertions(+), 1608 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.tsx index 6ad2f00fdb0..cd5c796b569 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.tsx @@ -2,6 +2,7 @@ import { FilterInput } from "@/components/common_components/Filters/FilterInput" import { FiltersButton } from "@/components/common_components/Filters/FiltersButton"; import { ResetFiltersButton } from "@/components/common_components/Filters/ResetFiltersButton"; import { Search, User } from "lucide-react"; +import { useTranslation } from "react-i18next"; interface OrganizationFiltersProps { filters: FilterState; @@ -23,6 +24,7 @@ const OrganizationFilters = ({ onChange, onReset, }: OrganizationFiltersProps) => { + const { t } = useTranslation(); const hasActiveFilters = !!(filters.org_id || filters.org_alias); return ( @@ -30,7 +32,7 @@ const OrganizationFilters = ({ {/* Search and Filter Controls */}
onChange("org_alias", value)} icon={Search} @@ -50,7 +52,7 @@ const OrganizationFilters = ({ {showFilters && (
onChange("org_id", value)} icon={User} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx index 2810902bef6..819aa6c3334 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx @@ -4,6 +4,7 @@ import OrganizationFilters, { FilterState } from "@/app/(dashboard)/organization import { useQueryClient } from "@tanstack/react-query"; import { parseAsString, parseAsStringLiteral, useQueryStates } from "nuqs"; import React, { useState } from "react"; +import { Trans, useTranslation } from "react-i18next"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; import { toast } from "@/lib/toast"; import { organizationDeleteCall } from "@/components/networking"; @@ -28,6 +29,7 @@ const ORGANIZATION_DETAIL_STATE = { const ORGANIZATION_DETAIL_URL_KEYS = { tab: ORGANIZATION_TAB_URL_KEY }; const OrganizationsPanel: React.FC = ({ userRole, accessToken, premiumUser }) => { + const { t } = useTranslation(); const [{ org: selectedOrgId }, setOrganizationDetail] = useQueryStates(ORGANIZATION_DETAIL_STATE, { history: "push", urlKeys: ORGANIZATION_DETAIL_URL_KEYS, @@ -78,7 +80,7 @@ const OrganizationsPanel: React.FC = ({ userRole, acces try { setIsDeleting(true); await organizationDeleteCall(accessToken, orgToDelete); - toast.success("Organization deleted successfully"); + toast.success(t("organizations.panel.toasts.deleted")); setIsDeleteModalOpen(false); setOrgToDelete(null); @@ -99,16 +101,19 @@ const OrganizationsPanel: React.FC = ({ userRole, acces return (

- This is a LiteLLM Enterprise feature, and requires a valid key to use. Get a trial key{" "} - - here - - . + + ), + }} + />

); @@ -118,7 +123,7 @@ const OrganizationsPanel: React.FC = ({ userRole, acces
{(userRole === "Admin" || userRole === "Org Admin") && ( )} @@ -133,7 +138,7 @@ const OrganizationsPanel: React.FC = ({ userRole, acces /> ) : ( <> -

Click on an organization ID to view its details.

+

{t("organizations.panel.listHint")}

= ({ userRole, acces @@ -27,12 +29,12 @@ function EmptyState({ searchActive }: { searchActive: boolean }) {
- {searchActive ? "No matching organizations" : "No organizations yet"} + {searchActive ? t("organizations.table.empty.noMatchesTitle") : t("organizations.table.empty.noneTitle")}
{searchActive - ? "No organizations match your search. Try a different name or ID." - : "Create an organization to group teams, models, and budgets."} + ? t("organizations.table.empty.noMatchesDescription") + : t("organizations.table.empty.noneDescription")}
); @@ -47,12 +49,13 @@ const OrganizationsTable: React.FC = ({ onEditClick, onDeleteClick, }) => { + const { t } = useTranslation(); const { sorting, onSortingChange, pagination, onPaginationChange } = useOrganizationsTableState(); const columns = useMemo(() => { - const deps = { userRole, onOrganizationClick, onEditClick, onDeleteClick }; + const deps = { userRole, onOrganizationClick, onEditClick, onDeleteClick, t }; return getOrganizationsTableColumns(deps); - }, [userRole, onOrganizationClick, onEditClick, onDeleteClick]); + }, [userRole, onOrganizationClick, onEditClick, onDeleteClick, t]); return ( = ({ sorting={sorting} onSortingChange={onSortingChange} isLoading={isLoading} - loadingMessage="Loading organizations…" + loadingMessage={t("organizations.table.loading")} noDataMessage={} size="compact" /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx index dcbf141314b..b4464392ea8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx @@ -1,7 +1,9 @@ "use client"; import { ColumnDef } from "@tanstack/react-table"; +import type { TFunction } from "i18next"; import { MoreHorizontal, Pencil, Trash2 } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { DataTableMultiSortHeader, DataTableSortHeader } from "@/components/shared/DataTable"; import { DateCell, IdentityCell, ModelsCell, MoneyCell } from "@/components/shared/table_cells"; @@ -25,11 +27,18 @@ const getOrganizationBudget = (organization: Organization): OrganizationBudget = (organization.litellm_budget_table ?? {}) as OrganizationBudget; function OrganizationLimitsCell({ organization }: { organization: Organization }) { + const { t } = useTranslation(); const { tpm_limit, rpm_limit } = getOrganizationBudget(organization); return (
- TPM: {tpm_limit ?? "Unlimited"} - RPM: {rpm_limit ?? "Unlimited"} + + {t("organizations.table.limits.tpmLabel")} + {tpm_limit ?? t("organizations.table.limits.unlimited")} + + + {t("organizations.table.limits.rpmLabel")} + {rpm_limit ?? t("organizations.table.limits.unlimited")} +
); } @@ -41,10 +50,11 @@ interface OrganizationRowActionsProps { } function OrganizationRowActions({ organization, onEditClick, onDeleteClick }: OrganizationRowActionsProps) { + const { t } = useTranslation(); return ( @@ -56,7 +66,7 @@ function OrganizationRowActions({ organization, onEditClick, onDeleteClick }: Or onClick={() => onEditClick(organization.organization_id)} > - Edit + {t("organizations.table.actions.edit")} onDeleteClick(organization.organization_id)} > - Delete + {t("organizations.table.actions.delete")} @@ -76,6 +86,7 @@ export interface OrganizationsTableColumnsDeps { onOrganizationClick: (organizationId: string) => void; onEditClick: (organizationId: string) => void; onDeleteClick: (organizationId: string) => void; + t: TFunction; } export const getOrganizationsTableColumns = ({ @@ -83,12 +94,13 @@ export const getOrganizationsTableColumns = ({ onOrganizationClick, onEditClick, onDeleteClick, + t, }: OrganizationsTableColumnsDeps): ColumnDef[] => [ { id: "organization_id", accessorKey: "organization_id", - meta: { title: "Organization ID" }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.id") }, + header: ({ column }) => , size: 220, enableSorting: true, cell: ({ row }) => ( @@ -103,8 +115,8 @@ export const getOrganizationsTableColumns = ({ { id: "organization_alias", accessorKey: "organization_alias", - meta: { title: "Organization Name" }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.name") }, + header: ({ column }) => , size: 200, enableSorting: true, cell: ({ row }) => { @@ -120,8 +132,8 @@ export const getOrganizationsTableColumns = ({ id: "created_at", accessorKey: "created_at", sortingFn: "datetime", - meta: { title: "Created" }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.created") }, + header: ({ column }) => , size: 130, enableSorting: true, cell: ({ row }) => , @@ -129,8 +141,8 @@ export const getOrganizationsTableColumns = ({ { id: "spend", accessorKey: "spend", - meta: { title: "Spend (USD)", numeric: true }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.spend"), numeric: true }, + header: ({ column }) => , size: 120, enableSorting: true, cell: ({ row }) => , @@ -140,12 +152,17 @@ export const getOrganizationsTableColumns = ({ accessorFn: (organization) => getOrganizationBudget(organization).max_budget ?? Number.POSITIVE_INFINITY, sortUndefined: "last", sortDescFirst: true, - meta: { title: "Budget (USD)", numeric: true }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.budget"), numeric: true }, + header: ({ column }) => , size: 120, enableSorting: true, cell: ({ row }) => ( - + ), }, { @@ -156,21 +173,21 @@ export const getOrganizationsTableColumns = ({ : undefined, sortUndefined: "last", sortDescFirst: false, - meta: { title: "Models", skeleton: "chips" }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.models"), skeleton: "chips" }, + header: ({ column }) => , size: 260, enableSorting: true, cell: ({ row }) => , }, { id: "limits", - meta: { title: "TPM / RPM Limits" }, + meta: { title: t("organizations.table.columns.limits") }, header: ({ table }) => ( ), @@ -183,11 +200,15 @@ export const getOrganizationsTableColumns = ({ accessorFn: (organization) => organization.members?.length ?? 0, sortUndefined: "last", sortDescFirst: true, - meta: { title: "Members", numeric: true }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.members"), numeric: true }, + header: ({ column }) => , size: 100, enableSorting: true, - cell: ({ row }) => {row.original.members?.length ?? 0} Members, + cell: ({ row }) => ( + + {t("organizations.table.memberCount", { count: row.original.members?.length ?? 0 })} + + ), }, ...(["tpm_limit", "rpm_limit"] as const).map( (field): ColumnDef => ({ @@ -201,7 +222,7 @@ export const getOrganizationsTableColumns = ({ { id: "actions", meta: { className: "text-right", headerClassName: "text-right" }, - header: () => Actions, + header: () => {t("organizations.table.columns.actions")}, size: 64, enableSorting: false, enableHiding: false, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.tsx index 67a648dacd1..b38dd5c6d5e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.tsx @@ -1,4 +1,5 @@ import React, { useId, useState } from "react"; +import { Trans, useTranslation } from "react-i18next"; import { userBulkUpdateUserCall, teamBulkMemberAddCall, Member } from "@/components/networking"; import { UserEditView } from "./user_edit_view"; import { toast } from "@/lib/toast"; @@ -46,6 +47,7 @@ const BulkEditUserModal: React.FC = ({ userModels, allowAllUsers = false, }) => { + const { t } = useTranslation(); const { premiumUser } = useAuthorized(); const [loading, setLoading] = useState(false); const [selectedTeams, setSelectedTeams] = useState([]); @@ -89,7 +91,7 @@ const BulkEditUserModal: React.FC = ({ const handleSubmit = async (formValues: any) => { if (!accessToken) { - toast.fromError("Access token not found"); + toast.fromError(t("users.bulkEdit.toasts.noAccessToken")); return; } @@ -125,7 +127,7 @@ const BulkEditUserModal: React.FC = ({ const hasTeamAdditions = addToTeams && selectedTeams.length > 0; if (!hasUserUpdates && !hasTeamAdditions) { - toast.fromError("Please modify at least one field or select teams to add users to"); + toast.fromError(t("users.bulkEdit.toasts.nothingToUpdate")); return; } @@ -135,10 +137,10 @@ const BulkEditUserModal: React.FC = ({ if (hasUserUpdates) { if (updateAllUsers) { const result = await userBulkUpdateUserCall(accessToken, updatePayload, undefined, true); - successMessages.push(`Updated all users (${result.total_requested} total)`); + successMessages.push(t("users.bulkEdit.toasts.updatedAll", { total: result.total_requested })); } else { await userBulkUpdateUserCall(accessToken, updatePayload, userIds); - successMessages.push(`Updated ${userIds.length} user(s)`); + successMessages.push(t("users.bulkEdit.toasts.updated", { count: userIds.length })); } } @@ -190,11 +192,13 @@ const BulkEditUserModal: React.FC = ({ if (successfulTeams.length > 0) { const totalAdditions = successfulTeams.reduce((sum, r) => sum + r.successfulAdditions, 0); - successMessages.push(`Added users to ${successfulTeams.length} team(s) (${totalAdditions} total additions)`); + successMessages.push( + t("users.bulkEdit.toasts.addedToTeams", { count: successfulTeams.length, additions: totalAdditions }), + ); } if (failedTeams.length > 0) { - toast.warning(`Failed to add users to ${failedTeams.length} team(s)`); + toast.warning(t("users.bulkEdit.toasts.teamAddFailed", { count: failedTeams.length })); } } @@ -212,7 +216,7 @@ const BulkEditUserModal: React.FC = ({ onCancel(); } catch (error) { console.error("Bulk operation failed:", error); - toast.fromError("Failed to perform bulk operations"); + toast.fromError(t("users.bulkEdit.toasts.bulkFailed")); } finally { setLoading(false); } @@ -223,7 +227,7 @@ const BulkEditUserModal: React.FC = ({ - {updateAllUsers ? "Bulk Edit All Users" : `Bulk Edit ${selectedUsers.length} User(s)`} + {updateAllUsers ? t("users.bulkEdit.titleAll") : t("users.bulkEdit.title", { count: selectedUsers.length })} {allowAllUsers && ( @@ -233,17 +237,15 @@ const BulkEditUserModal: React.FC = ({ id={updateAllUsersId} checked={updateAllUsers} onCheckedChange={(checked) => setUpdateAllUsers(checked === true)} - aria-label="Update ALL users in the system" + aria-label={t("users.bulkEdit.updateAll")} />
{updateAllUsers && (
- - ⚠️ This will apply changes to ALL users in the system, not just the selected ones. - + {t("users.bulkEdit.updateAllWarning")}
)} @@ -251,15 +253,19 @@ const BulkEditUserModal: React.FC = ({ {!updateAllUsers && (
-
Selected Users ({selectedUsers.length}):
+
+ {t("users.bulkEdit.selectedUsers", { count: selectedUsers.length })} +
- User ID - Email - Current Role - Budget + {t("users.bulkEdit.table.userId")} + {t("users.bulkEdit.table.email")} + {t("users.bulkEdit.table.currentRole")} + + {t("users.bulkEdit.table.budget")} + @@ -268,12 +274,19 @@ const BulkEditUserModal: React.FC = ({ {user.user_id.length > 20 ? `${user.user_id.slice(0, 20)}...` : user.user_id} - {user.user_email || "No email"} + + {user.user_email || t("users.bulkEdit.table.noEmail")} + {possibleUIRoles?.[user.user_role]?.ui_label || user.user_role} - + ))} @@ -287,15 +300,14 @@ const BulkEditUserModal: React.FC = ({

- Instructions: Fill in the fields below with the values you want to apply to all selected - users. You can bulk edit: role, budget, models, and metadata. You can also add users to teams. + }} />

{/* Team Management Section */} - Team Management + {t("users.bulkEdit.teamManagement.title")}
@@ -304,10 +316,10 @@ const BulkEditUserModal: React.FC = ({ id={addToTeamsId} checked={addToTeams} onCheckedChange={(checked) => setAddToTeams(checked === true)} - aria-label="Add selected users to teams" + aria-label={t("users.bulkEdit.teamManagement.addToTeams")} />
@@ -315,12 +327,12 @@ const BulkEditUserModal: React.FC = ({ <>
= ({
) => setTeamBudget(event.target.value === "" ? null : Number(event.target.value)) @@ -348,13 +360,12 @@ const BulkEditUserModal: React.FC = ({ step={0.01} /> - Leave empty for unlimited budget within team limits + {t("users.bulkEdit.teamManagement.teamBudgetHint")}
- Users will be added with "user" role by default. All users will be added to each selected - team. + {t("users.bulkEdit.teamManagement.defaultRoleNote")} )} @@ -379,7 +390,9 @@ const BulkEditUserModal: React.FC = ({ {loading && (
- Updating {updateAllUsers ? "all users" : selectedUsers.length} user(s)... + {updateAllUsers + ? t("users.bulkEdit.updatingAll") + : t("users.bulkEdit.updating", { count: selectedUsers.length })}
)} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx index 0239844851f..a617d5f84cb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx @@ -1,8 +1,10 @@ "use client"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import type { ParseKeys } from "i18next"; import * as React from "react"; import { useFieldArray, type Control } from "react-hook-form"; +import { useTranslation } from "react-i18next"; import { useInfiniteTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import { ModelSelect, MODEL_SENTINEL_OPTIONS } from "@/components/ModelSelect/ModelSelect"; @@ -29,21 +31,21 @@ import { const NO_RESET = "never"; -const BUDGET_DURATION_OPTIONS = [ - { value: NO_RESET, label: "No reset" }, - { value: "1h", label: "hourly" }, - { value: "24h", label: "daily" }, - { value: "7d", label: "weekly" }, - { value: "30d", label: "monthly" }, -] as const; +const BUDGET_DURATION_OPTIONS: readonly { value: string; labelKey: ParseKeys }[] = [ + { value: NO_RESET, labelKey: "users.defaultSettings.budgetDurations.noReset" }, + { value: "1h", labelKey: "users.defaultSettings.budgetDurations.hourly" }, + { value: "24h", labelKey: "users.defaultSettings.budgetDurations.daily" }, + { value: "7d", labelKey: "users.defaultSettings.budgetDurations.weekly" }, + { value: "30d", labelKey: "users.defaultSettings.budgetDurations.monthly" }, +]; -const TEAM_ROLE_OPTIONS = [ - { value: "user", label: "User" }, - { value: "admin", label: "Admin" }, -] as const; +const TEAM_ROLE_OPTIONS: readonly { value: "user" | "admin"; labelKey: ParseKeys }[] = [ + { value: "user", labelKey: "users.defaultSettings.teamRoles.user" }, + { value: "admin", labelKey: "users.defaultSettings.teamRoles.admin" }, +]; -const MODEL_SENTINEL_LABELS: ReadonlyMap = new Map( - MODEL_SENTINEL_OPTIONS.map(({ value, label }) => [value, label]), +const MODEL_SENTINEL_LABEL_KEYS: ReadonlyMap = new Map( + MODEL_SENTINEL_OPTIONS.map(({ value, labelKey }) => [value, labelKey]), ); const TEAMS_PAGE_SIZE = 50; @@ -71,6 +73,7 @@ interface RoleOption { type SettingsControl = Control; const TeamPickerField = ({ control, index }: { control: SettingsControl; index: number }) => { + const { t } = useTranslation(); const [search, setSearch] = React.useState(""); const { data, fetchNextPage, hasNextPage, isFetchingNextPage, isLoading } = useInfiniteTeams( TEAMS_PAGE_SIZE, @@ -90,7 +93,7 @@ const TeamPickerField = ({ control, index }: { control: SettingsControl; index: ); return ( - + {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( { + const { t } = useTranslation(); const { fields, append, remove } = useFieldArray({ control, name: "teams" }); + const teamRoleItems = TEAM_ROLE_OPTIONS.map((option) => ({ value: option.value, label: t(option.labelKey) })); return (
-

Default Teams

-

- New users are added to these teams. Only teams that already exist can be selected. -

+

{t("users.defaultSettings.fields.teams")}

+

{t("users.defaultSettings.teams.description")}

{fields.map((field, index) => (
-

Team {index + 1}

+

{t("users.defaultSettings.teams.rowTitle", { number: index + 1 })}

- + {({ ref, ...budgetField }) => ( - + )} - + {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( - onChange(selected ?? "user")}> { - {TEAM_ROLE_OPTIONS.map((option) => ( + {teamRoleItems.map((option) => ( {option.label} @@ -172,7 +186,7 @@ const TeamsField = ({ control }: { control: SettingsControl }) => { ))}
); @@ -191,30 +205,41 @@ interface SettingsViewProps { } const SettingsView = ({ values, roleOptions }: SettingsViewProps) => { + const { t } = useTranslation(); const roleLabel = roleOptions.find((option) => option.value === values.user_role)?.label ?? values.user_role; const durationValue = values.budget_duration === "" ? NO_RESET : values.budget_duration; - const durationLabel = - BUDGET_DURATION_OPTIONS.find((option) => option.value === durationValue)?.label ?? values.budget_duration; + const durationOption = BUDGET_DURATION_OPTIONS.find((option) => option.value === durationValue); + const durationLabel = durationOption === undefined ? values.budget_duration : t(durationOption.labelKey); + const notSet = t("users.defaultSettings.view.notSet"); return (
- {roleLabel === "" ? "Not set" : roleLabel} - {values.max_budget === "" ? "Not set" : values.max_budget} - {durationLabel} - + {roleLabel === "" ? notSet : roleLabel} + + {values.max_budget === "" ? notSet : values.max_budget} + + {durationLabel} + {values.models.length === 0 - ? "Not set" - : values.models.map((model) => MODEL_SENTINEL_LABELS.get(model) ?? model).join(", ")} + ? notSet + : values.models + .map((model) => { + const labelKey = MODEL_SENTINEL_LABEL_KEYS.get(model); + return labelKey === undefined ? model : t(labelKey); + }) + .join(", ")}
-

Default Teams

+

{t("users.defaultSettings.fields.teams")}

{values.teams.length === 0 ? ( -

None

+

{t("users.defaultSettings.view.none")}

) : ( values.teams.map((team) => (

{team.team_id} - {team.max_budget_in_team !== "" && <> · ${team.max_budget_in_team} max budget} + {team.max_budget_in_team !== "" && ( + <> · {t("users.defaultSettings.view.teamMaxBudget", { budget: team.max_budget_in_team })} + )} <> · {team.user_role}

)) @@ -233,20 +258,25 @@ interface SettingsFormProps { } const SettingsForm = ({ initialValues, roleOptions, updateSettings, onCancel, onSaved }: SettingsFormProps) => { + const { t } = useTranslation(); const queryClient = useQueryClient(); + const budgetDurationItems = BUDGET_DURATION_OPTIONS.map((option) => ({ + value: option.value, + label: t(option.labelKey), + })); const form = useZodForm(defaultUserSettingsSchema, { defaultValues: initialValues }); const { isDirty } = form.formState; const mutation = useMutation({ mutationFn: (values: DefaultUserSettingsSubmitValues) => updateSettings(buildBody(values)), onSuccess: (_result, values) => { - toast.success("Default user settings updated successfully"); + toast.success(t("users.defaultSettings.toasts.updated")); queryClient.invalidateQueries({ queryKey: SETTINGS_QUERY_KEY }); form.reset(values); onSaved(); }, onError: (error: unknown) => - toast.fromError(error instanceof Error ? error.message : "Failed to update default user settings"), + toast.fromError(error instanceof Error ? error.message : t("users.defaultSettings.toasts.updateFailed")), }); const onSubmit = form.handleSubmit((values) => mutation.mutate(values)); @@ -257,8 +287,8 @@ const SettingsForm = ({ initialValues, roleOptions, updateSettings, onCancel, on {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( } @@ -295,12 +325,12 @@ const SettingsForm = ({ initialValues, roleOptions, updateSettings, onCancel, on {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( } )} {!isBulkEdit && ( - + {({ ref, value, ...control }) => } )} - + {({ ref, value, ...control }) => } @@ -242,8 +247,8 @@ export function UserEditView({ control={form.control} name="user_role" label={labelWithHint( - "Global Proxy Role", - "This is the role that the user will globally on the proxy. This role is independent of any team/org specific roles.", + t("users.editView.fields.globalProxyRole"), + t("users.editView.fields.globalProxyRoleHint"), )} > {({ id, value, onChange }) => ( @@ -271,8 +276,8 @@ export function UserEditView({ control={form.control} name="models" label={labelWithHint( - "Personal Models", - "Select which models this user can access outside of team-scope. Choose 'All Proxy Models' to grant access to all models available on the proxy.", + t("users.editView.fields.personalModels"), + t("users.editView.fields.personalModelsHint"), )} > {({ value, onChange }) => ( @@ -280,7 +285,7 @@ export function UserEditView({ options={modelOptions} value={value} onValueChange={onChange} - placeholder="Select models" + placeholder={t("users.editView.fields.modelsPlaceholder")} disabled={!all_admin_roles.includes(userRole || "")} /> )} @@ -291,10 +296,10 @@ export function UserEditView({ name="max_budget" label={ <> - Max Budget (USD) + {t("users.editView.fields.maxBudget")} } @@ -308,19 +313,19 @@ export function UserEditView({ value={value ?? ""} onChange={(event) => onChange(event.target.value)} onWheel={(event) => event.currentTarget.blur()} - placeholder="Enter a numerical value" + placeholder={t("users.editView.fields.maxBudgetPlaceholder")} disabled={unlimitedBudget} /> )} - + {({ id, value, onChange }) => ( )} @@ -330,10 +335,7 @@ export function UserEditView({ {({ ref, value, onChange, ...control }) => ( onChange(event.target.value)} onWheel={(event) => event.currentTarget.blur()} - placeholder="Unlimited" + placeholder={t("users.editView.fields.rateLimitPlaceholder")} /> )} @@ -353,10 +355,7 @@ export function UserEditView({ {({ ref, value, onChange, ...control }) => ( onChange(event.target.value)} onWheel={(event) => event.currentTarget.blur()} - placeholder="Unlimited" + placeholder={t("users.editView.fields.rateLimitPlaceholder")} /> )} @@ -385,13 +384,19 @@ export function UserEditView({ onChange={setModelMaxBudget} availableModels={userModels} usage={userData.user_info?.model_max_budget_usage} - hint="Cap this user's spend on individual models, each with its own reset window. Applies across every key the user holds." + hint={t("users.editView.fields.modelMaxBudgetHint")} /> )} - + {({ ref, value, ...control }) => ( -