From 21c442759e3acf75f076def7ce47b7d64bc5201e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:22:15 -0500 Subject: [PATCH 01/48] feat(router): time-windowed team reservation of deployments via model_info.access_windows (#42398) * feat(router): time-windowed team reservation of deployments via model_info.access_windows Deployments can declare model_info.access_windows, a list of local wall-clock windows (IANA timezone, cross-midnight allowed) that reserve the deployment for the listed team_ids. While a window is active the router drops the deployment for every other request, including no-team and admin requests, on every candidate path (model name, model id, specific_deployment, early-resolve, wildcard, litellm_params.model lookup, fallbacks). If every candidate is reserved the request fails with a 400 naming the window end instead of falling back. Outside a window routing is unchanged and reserved deployments stay visible in /model/info and /v1/models. Malformed windows (bad time, unknown timezone, empty team_ids, start equal to end, offset-aware times) fail proxy startup with a clear error since the proxy router runs with ignore_invalid_deployments=True Resolves LIT-8308 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): cover _filter_reserved_deployments directly for coverage gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(router): keep reservation filtering immutable Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): drop strategy markers before reservation filtering Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 18 ++ litellm/router.py | 97 ++++++- litellm/router_utils/access_windows.py | 75 ++++++ litellm/types/router.py | 41 +++ .../proxy/proxy_server/test_proxy_config.py | 57 +++++ .../router_utils/test_access_windows.py | 190 ++++++++++++++ tests/test_litellm/test_router.py | 241 +++++++++++++++++- tests/test_litellm/types/test_router.py | 45 ++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 19 ++ 9 files changed, 769 insertions(+), 14 deletions(-) create mode 100644 litellm/router_utils/access_windows.py create mode 100644 tests/test_litellm/router_utils/test_access_windows.py diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 18bc02467d1..efc428fa4c2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -132,6 +132,7 @@ from litellm.proxy.common_utils.callback_utils import ( strip_callback_config, ) from litellm.proxy.common_utils.realtime_utils import _realtime_request_body +from litellm.router_utils.access_windows import access_windows_config_error from litellm.router_utils.add_retry_fallback_headers import ( get_fallback_errors_from_headers, get_hidden_params_dict, @@ -4852,6 +4853,22 @@ def validate_deployment_complexity_router_placement(model: Mapping[str, object]) raise ValueError(f"model {model.get('model_name', '')!r}: {violation}") +def validate_deployment_access_windows(model: Mapping[str, object]) -> None: + """ + Reject a malformed `model_info.access_windows` instead of silently dropping the deployment. + + Checked here rather than on `ModelInfo` because the proxy builds its router with + `ignore_invalid_deployments=True`, so a rejection further down turns a bad + deployment into a silently missing model instead of a refusal to start. + """ + model_info: Final = model.get("model_info") + if not isinstance(model_info, Mapping): + return + error: Final = access_windows_config_error(model_info, model_name=str(model.get("model_name", ""))) + if error is not None: + raise ValueError(error) + + def validate_auto_router_capability_limits(model_list: Sequence[Mapping[str, object]], *, limit: int | None) -> None: """ Refuse to start when config.yaml defines more auto-routers claiming a licensed capability than allowed. @@ -6552,6 +6569,7 @@ class ProxyConfig: model["litellm_params"][k] = get_secret(v) validate_deployment_max_agentic_loops(model) validate_deployment_complexity_router_placement(model) + validate_deployment_access_windows(model) pin_complexity_router_model_id(model) complexity_router_config = model["litellm_params"].get("complexity_router_config") if isinstance(complexity_router_config, dict): diff --git a/litellm/router.py b/litellm/router.py index 06a65b85cc3..80e8eb0c090 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -31,6 +31,7 @@ from collections.abc import ( MutableMapping, Sequence, ) +from datetime import datetime, timezone from functools import lru_cache, partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeAlias, TypeVar, Union, cast @@ -138,6 +139,7 @@ from litellm.router_strategy.tag_based_routing import ( get_deployments_for_tag, is_valid_deployment_tag, ) +from litellm.router_utils.access_windows import access_windows_config_error, filter_reserved_deployments from litellm.router_utils.add_retry_fallback_headers import ( _HiddenParamsHost, add_fallback_headers_to_response, @@ -8774,6 +8776,9 @@ class Router: ) if ptu_error is not None and is_ptu_cost_attribution_enabled(): raise ValueError(ptu_error) + access_windows_error: Final = access_windows_config_error(_model_info, model_name=_model_name) + if access_windows_error is not None: + raise ValueError(access_windows_error) zeroed_pricing: Final = zeroed_ptu_pricing(_model_info, _litellm_params) if config_sourced else None litellm_params: Final[LiteLLM_Params] = LiteLLM_Params( **( # pyright: ignore[reportArgumentType] # untyped merged dict; already true for every field here @@ -12535,12 +12540,30 @@ class Router: request_team_id: Final = get_request_team_id(request_kwargs) # check if aliases set on litellm model alias map if specific_deployment is True: - return model, self._get_deployment_by_litellm_model(model=model) + return model, self._drop_strategy_markers( + model, + self._filter_reserved_deployments( + model=model, + healthy_deployments=self._get_deployment_by_litellm_model(model=model), + request_team_id=request_team_id, + ), + ) elif model not in self.model_names and self.has_model_id(model): deployment: Final = self.get_deployment(model_id=model) if deployment is not None: deployment_model: Final = deployment.litellm_params.model - return deployment_model, deployment.model_dump(exclude_none=True) + return deployment_model, cast( # cast-ok: contract requires a plain dict for a single deployment + dict, + self._filter_reserved_deployments( + model=deployment_model, + healthy_deployments=( + cast( # cast-ok: model_dump of a router deployment + DeploymentTypedDict, deployment.model_dump(exclude_none=True) + ), + ), + request_team_id=request_team_id, + )[0], + ) raise ValueError( f"LiteLLM Router: Trying to call specific deployment, but Model ID :{model} does not exist in Model ID map" ) @@ -12558,8 +12581,26 @@ class Router: ) if early is not None: if not isinstance(early[1], list): - return early - return early[0], self._drop_strategy_markers(early[0], early[1]) + return early[0], cast( # cast-ok: contract requires a plain dict for a single deployment + dict, + self._filter_reserved_deployments( + model=early[0], + healthy_deployments=( + cast( # cast-ok: early resolve returns a router deployment + DeploymentTypedDict, early[1] + ), + ), + request_team_id=request_team_id, + )[0], + ) + return early[0], self._drop_strategy_markers( + early[0], + self._filter_reserved_deployments( + model=early[0], + healthy_deployments=early[1], + request_team_id=request_team_id, + ), + ) ## get healthy deployments ### get all deployments @@ -12569,10 +12610,14 @@ class Router: else self._get_all_deployments(model_name=model, team_id=request_team_id) ) _pre_model_access_group_filter_len: Final = len(healthy_deployments) - healthy_deployments = self._filter_deployments_by_model_access_groups( + healthy_deployments = self._filter_reserved_deployments( model=model, - healthy_deployments=healthy_deployments, - request_kwargs=request_kwargs, + healthy_deployments=self._filter_deployments_by_model_access_groups( + model=model, + healthy_deployments=healthy_deployments, + request_kwargs=request_kwargs, + request_team_id=request_team_id, + ), request_team_id=request_team_id, ) _access_group_filter_emptied_candidates = ( @@ -12585,10 +12630,14 @@ class Router: # _get_deployment_by_litellm_model does not re-apply that filter. if _pre_model_access_group_filter_len == 0: _litellm_model_deployments: Final = self._get_deployment_by_litellm_model(model=model) - healthy_deployments = self._filter_deployments_by_model_access_groups( + healthy_deployments = self._filter_reserved_deployments( model=model, - healthy_deployments=_litellm_model_deployments, - request_kwargs=request_kwargs, + healthy_deployments=self._filter_deployments_by_model_access_groups( + model=model, + healthy_deployments=_litellm_model_deployments, + request_kwargs=request_kwargs, + request_team_id=request_team_id, + ), request_team_id=request_team_id, ) # If the litellm-model lookup produced candidates that access-group @@ -12616,10 +12665,14 @@ class Router: # Re-assign model to the fallback and try to get deployments again model = fallback_model healthy_deployments = self._get_all_deployments(model_name=model, team_id=request_team_id) - healthy_deployments = self._filter_deployments_by_model_access_groups( + healthy_deployments = self._filter_reserved_deployments( model=model, - healthy_deployments=healthy_deployments, - request_kwargs=request_kwargs, + healthy_deployments=self._filter_deployments_by_model_access_groups( + model=model, + healthy_deployments=healthy_deployments, + request_kwargs=request_kwargs, + request_team_id=request_team_id, + ), request_team_id=request_team_id, ) @@ -12653,6 +12706,24 @@ class Router: ) return selectable + def _filter_reserved_deployments( + self, + model: str, + healthy_deployments: Sequence[DeploymentTypedDict], + request_team_id: str | None, + ) -> tuple[DeploymentTypedDict, ...]: + result: Final = filter_reserved_deployments( + self._drop_strategy_markers(model, healthy_deployments), request_team_id, now=datetime.now(timezone.utc) + ) + if result.blocking_window is not None and len(result.deployments) == 0: + raise litellm.BadRequestError( + message=f"Deployment {model} is reserved for another team until " + f"{result.blocking_window.end:%H:%M} {result.blocking_window.timezone}", + model=model, + llm_provider="", + ) + return result.deployments + def _filter_deployments_by_model_access_groups( self, model: str, diff --git a/litellm/router_utils/access_windows.py b/litellm/router_utils/access_windows.py new file mode 100644 index 00000000000..5a44584b33d --- /dev/null +++ b/litellm/router_utils/access_windows.py @@ -0,0 +1,75 @@ +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Final, Generic, TypeVar +from zoneinfo import ZoneInfo + +from pydantic import TypeAdapter, ValidationError + +from litellm.types.router import ModelAccessWindow + +_WINDOWS_ADAPTER: Final = TypeAdapter(tuple[ModelAccessWindow, ...]) + +_DeploymentT = TypeVar("_DeploymentT", bound=Mapping[str, object]) + + +def parse_access_windows(model_info: Mapping[str, object]) -> tuple[ModelAccessWindow, ...]: + raw: Final = model_info.get("access_windows") + if raw is None: + return () + return _WINDOWS_ADAPTER.validate_python(raw) + + +def access_windows_config_error(model_info: Mapping[str, object], *, model_name: str) -> str | None: + if model_info.get("access_windows") is None: + return None + try: + parse_access_windows(model_info) + except ValidationError as exc: + first: Final = exc.errors()[0] + loc: Final = ".".join(str(part) for part in first["loc"]) + return f"model '{model_name}': invalid model_info.access_windows: {loc}: {first['msg']}" + return None + + +def is_window_active(window: ModelAccessWindow, now: datetime) -> bool: + aware: Final = now if now.tzinfo is not None else now.replace(tzinfo=timezone.utc) + local: Final = aware.astimezone(ZoneInfo(window.timezone)).time() + if window.start < window.end: + return window.start <= local < window.end + return local >= window.start or local < window.end + + +@dataclass(frozen=True, slots=True) +class ReservationFilterResult(Generic[_DeploymentT]): + deployments: tuple[_DeploymentT, ...] + blocking_window: ModelAccessWindow | None + + +def _reservation_blocking_window( + deployment: Mapping[str, object], request_team_id: str | None, now: datetime +) -> ModelAccessWindow | None: + model_info: Final = deployment.get("model_info") + if not isinstance(model_info, Mapping): + return None + active: Final = tuple(window for window in parse_access_windows(model_info) if is_window_active(window, now)) + if not active: + return None + if request_team_id is not None and any(request_team_id in window.team_ids for window in active): + return None + return active[0] + + +def filter_reserved_deployments( + healthy_deployments: Sequence[_DeploymentT], + request_team_id: str | None, + now: datetime, +) -> ReservationFilterResult[_DeploymentT]: + checks: Final = tuple( + (deployment, _reservation_blocking_window(deployment, request_team_id, now)) + for deployment in healthy_deployments + ) + return ReservationFilterResult( + deployments=tuple(deployment for deployment, blocking in checks if blocking is None), + blocking_window=next((blocking for _, blocking in checks if blocking is not None), None), + ) diff --git a/litellm/types/router.py b/litellm/types/router.py index 8ca27a9fb66..5017e7073f7 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -7,6 +7,7 @@ import enum from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Any, ClassVar, Final, Generic, Literal, TypeVar, get_type_hints +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import httpx from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator @@ -163,6 +164,44 @@ def _as_utc(value: datetime.datetime | None) -> datetime.datetime | None: return value.astimezone(datetime.timezone.utc) +class ModelAccessWindow(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + start: datetime.time + end: datetime.time + timezone: str + team_ids: tuple[str, ...] = Field(min_length=1) + + @field_validator("start", "end") + @classmethod + def _naive_wall_clock(cls, value: datetime.time) -> datetime.time: + if value.tzinfo is not None: + raise ValueError("start and end must be local wall-clock times without a UTC offset") + return value + + @field_validator("timezone") + @classmethod + def _known_iana_timezone(cls, value: str) -> str: + try: + ZoneInfo(value) + except (ZoneInfoNotFoundError, ValueError) as exc: + raise ValueError(f"unknown IANA timezone '{value}'") from exc + return value + + @field_validator("team_ids") + @classmethod + def _non_empty_team_ids(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if any(not team_id for team_id in value): + raise ValueError("team_ids entries must be non-empty") + return value + + @model_validator(mode="after") + def _start_differs_from_end(self) -> "ModelAccessWindow": + if self.start == self.end: + raise ValueError("start and end must differ") + return self + + class ModelInfo(MirroredPricingParams): id: str | None # Allow id to be optional on input, but it will always be present as a str in the model instance db_model: bool = False # used for proxy - to separate models which are stored in the db vs. config. @@ -188,6 +227,8 @@ class ModelInfo(MirroredPricingParams): # admin-toggled pause flag; mirrors LiteLLM_ProxyModelTable.blocked blocked: bool | None = None + access_windows: tuple[ModelAccessWindow, ...] | None = None + # Bounds live on the model rather than litellm.constants: names there reach # litellm/__init__ through several modules' star re-exports, and a Final rebound that # way trips the basedpyright gate. diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 462489f48b0..832ac6402d9 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -32,6 +32,7 @@ from litellm.proxy.proxy_server import ( _scrub_guardrail_inner, resolve_complexity_router_plugins, resolve_routing_plugins, + validate_deployment_access_windows, validate_deployment_complexity_router_placement, validate_deployment_max_agentic_loops, validate_auto_router_capability_limits, @@ -4714,3 +4715,59 @@ def test_websearch_interception_settings_can_be_named_in_supported_db_objects(mo monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": ["models"]}) assert proxy_server.should_load_db_object(object_type="websearch_interception_settings") is False + + +def test_validate_deployment_access_windows_rejects_malformed_time(): + model = { + "model_name": "gpt-4o-shared", + "litellm_params": {"model": "gpt-4o"}, + "model_info": { + "access_windows": [ + {"start": "25:00", "end": "06:00", "timezone": "America/New_York", "team_ids": ["t"]} + ] + }, + } + + with pytest.raises(ValueError, match="access_windows") as exc_info: + validate_deployment_access_windows(model) + + assert "gpt-4o-shared" in str(exc_info.value) + + +def test_validate_deployment_access_windows_rejects_unknown_timezone(): + model = { + "model_name": "gpt-4o-shared", + "litellm_params": {"model": "gpt-4o"}, + "model_info": { + "access_windows": [ + {"start": "22:00", "end": "06:00", "timezone": "Mars/Olympus", "team_ids": ["t"]} + ] + }, + } + + with pytest.raises(ValueError, match="Mars/Olympus"): + validate_deployment_access_windows(model) + + +def test_validate_deployment_access_windows_accepts_valid_and_absent(): + assert ( + validate_deployment_access_windows( + { + "model_name": "gpt-4o-shared", + "litellm_params": {"model": "gpt-4o"}, + "model_info": { + "access_windows": [ + {"start": "22:00", "end": "06:00", "timezone": "America/New_York", "team_ids": ["t"]} + ] + }, + } + ) + is None + ) + assert validate_deployment_access_windows({"model_name": "m", "litellm_params": {"model": "m"}}) is None + assert ( + validate_deployment_access_windows( + {"model_name": "m", "litellm_params": {"model": "m"}, "model_info": {"id": "x"}} + ) + is None + ) diff --git a/tests/test_litellm/router_utils/test_access_windows.py b/tests/test_litellm/router_utils/test_access_windows.py new file mode 100644 index 00000000000..a797a788b4e --- /dev/null +++ b/tests/test_litellm/router_utils/test_access_windows.py @@ -0,0 +1,190 @@ +from datetime import datetime, time, timezone +from typing import Final + + +from litellm.router_utils.access_windows import ( + access_windows_config_error, + filter_reserved_deployments, + is_window_active, +) +from litellm.types.router import ModelAccessWindow + +_NIGHT_NY: Final = ModelAccessWindow( + start=time(22, 0), + end=time(6, 0), + timezone="America/New_York", + team_ids=("team-nightly",), +) + + +def _window(start: str, end: str, tz: str = "UTC", team_ids=("team-a",)) -> ModelAccessWindow: + return ModelAccessWindow( + start=time.fromisoformat(start), + end=time.fromisoformat(end), + timezone=tz, + team_ids=tuple(team_ids), + ) + + +def _deployment(windows: object = None) -> dict: + if windows is None: + return {"model_info": {}} + return {"model_info": {"access_windows": windows}} + + +def test_same_day_window_active_and_inactive(): + window: Final = _window("09:00", "17:00") + assert is_window_active(window, datetime(2026, 3, 9, 12, 0, tzinfo=timezone.utc)) is True + assert is_window_active(window, datetime(2026, 3, 9, 20, 0, tzinfo=timezone.utc)) is False + + +def test_cross_midnight_window(): + window: Final = _window("22:00", "06:00") + assert is_window_active(window, datetime(2026, 3, 9, 23, 0, tzinfo=timezone.utc)) is True + assert is_window_active(window, datetime(2026, 3, 10, 5, 59, tzinfo=timezone.utc)) is True + assert is_window_active(window, datetime(2026, 3, 9, 12, 0, tzinfo=timezone.utc)) is False + + +def test_start_boundary_inclusive_and_end_boundary_exclusive(): + window: Final = _window("22:00", "06:00") + assert is_window_active(window, datetime(2026, 3, 9, 22, 0, tzinfo=timezone.utc)) is True + assert is_window_active(window, datetime(2026, 3, 10, 6, 0, tzinfo=timezone.utc)) is False + + +def test_dst_spring_forward_gap_uses_real_local_time(): + window: Final = ModelAccessWindow( + start=time(1, 30), + end=time(3, 30), + timezone="America/New_York", + team_ids=("team-a",), + ) + assert is_window_active(window, datetime(2026, 3, 8, 6, 30, tzinfo=timezone.utc)) is True + assert is_window_active(window, datetime(2026, 3, 8, 7, 0, tzinfo=timezone.utc)) is True + assert is_window_active(window, datetime(2026, 3, 8, 7, 30, tzinfo=timezone.utc)) is False + + +def test_naive_now_is_treated_as_utc(): + window: Final = _window("09:00", "17:00") + assert is_window_active(window, datetime(2026, 3, 9, 12, 0)) is True + + +def test_team_in_second_window_is_kept(): + deployments: Final = ( + _deployment([ + {"start": "01:00", "end": "02:00", "timezone": "UTC", "team_ids": ["team-other"]}, + {"start": "20:00", "end": "23:59", "timezone": "UTC", "team_ids": ["team-a"]}, + ]), + ) + result: Final = filter_reserved_deployments( + deployments, "team-a", now=datetime(2026, 3, 9, 21, 0, tzinfo=timezone.utc) + ) + assert result.deployments == deployments + assert result.blocking_window is None + + +def test_unlisted_team_is_dropped_with_blocking_window(): + deployments: Final = (_deployment([_NIGHT_NY.model_dump()]),) + result: Final = filter_reserved_deployments( + deployments, "team-b", now=datetime(2026, 3, 10, 4, 0, tzinfo=timezone.utc) + ) + assert result.deployments == () + assert result.blocking_window == _NIGHT_NY + + +def test_missing_team_id_is_dropped(): + result: Final = filter_reserved_deployments( + (_deployment([_NIGHT_NY.model_dump()]),), + None, + now=datetime(2026, 3, 10, 4, 0, tzinfo=timezone.utc), + ) + assert result.deployments == () + assert result.blocking_window == _NIGHT_NY + + +def test_listed_team_is_kept(): + deployments: Final = (_deployment([_NIGHT_NY.model_dump()]),) + result: Final = filter_reserved_deployments( + deployments, "team-nightly", now=datetime(2026, 3, 10, 4, 0, tzinfo=timezone.utc) + ) + assert result.deployments == deployments + assert result.blocking_window is None + + +def test_deployment_without_windows_kept_for_anyone(): + deployments: Final = (_deployment(),) + result: Final = filter_reserved_deployments( + deployments, None, now=datetime(2026, 3, 10, 4, 0, tzinfo=timezone.utc) + ) + assert result.deployments == deployments + assert result.blocking_window is None + + +def test_inactive_window_keeps_deployment_for_unlisted_team(): + deployments: Final = (_deployment([_NIGHT_NY.model_dump()]),) + result: Final = filter_reserved_deployments( + deployments, "team-b", now=datetime(2026, 3, 10, 16, 0, tzinfo=timezone.utc) + ) + assert result.deployments == deployments + assert result.blocking_window is None + + +def test_unreserved_deployment_survives_for_other_team(): + reserved: Final = _deployment([_NIGHT_NY.model_dump()]) + open_deployment: Final = _deployment() + result: Final = filter_reserved_deployments( + (reserved, open_deployment), + "team-b", + now=datetime(2026, 3, 10, 4, 0, tzinfo=timezone.utc), + ) + assert result.deployments == (open_deployment,) + assert result.blocking_window == _NIGHT_NY + + +def test_config_error_unknown_timezone(): + error: Final = access_windows_config_error( + {"access_windows": [{"start": "22:00", "end": "06:00", "timezone": "Mars/Olympus", "team_ids": ["t"]}]}, + model_name="nightly-model", + ) + assert error is not None + assert "nightly-model" in error + assert "access_windows" in error + assert "Mars/Olympus" in error + + +def test_config_error_bad_time(): + error: Final = access_windows_config_error( + {"access_windows": [{"start": "25:00", "end": "06:00", "timezone": "UTC", "team_ids": ["t"]}]}, + model_name="m", + ) + assert error is not None + assert "access_windows" in error + + +def test_config_error_empty_team_ids(): + error: Final = access_windows_config_error( + {"access_windows": [{"start": "22:00", "end": "06:00", "timezone": "UTC", "team_ids": []}]}, + model_name="m", + ) + assert error is not None + + +def test_config_error_start_equals_end(): + error: Final = access_windows_config_error( + {"access_windows": [{"start": "22:00", "end": "22:00", "timezone": "UTC", "team_ids": ["t"]}]}, + model_name="m", + ) + assert error is not None + + +def test_config_error_none_when_absent(): + assert access_windows_config_error({}, model_name="m") is None + assert access_windows_config_error({"access_windows": None}, model_name="m") is None + + +def test_config_error_offset_aware_time(): + error: Final = access_windows_config_error( + {"access_windows": [{"start": "22:00+05:00", "end": "06:00", "timezone": "UTC", "team_ids": ["t"]}]}, + model_name="m", + ) + assert error is not None + assert "UTC offset" in error diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 0df6a181957..f5344c01f3a 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -9,7 +9,7 @@ import sys import threading import warnings from collections.abc import Awaitable, Callable, Mapping -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -17620,3 +17620,242 @@ class TestMemberAutoRouterInference: monkeypatch.setitem(sys.modules, "fastapi", None) monkeypatch.delitem(sys.modules, "litellm.proxy.auth.auto_router_checks", raising=False) assert (await self._route(router, {"metadata": {"user_api_key_team_id": "router-team"}})).model == "restricted-model" + + +def _access_window_offsets(start_hours: float, end_hours: float, team_ids: list) -> dict: + now_utc = datetime.now(timezone.utc) + return { + "start": (now_utc + timedelta(hours=start_hours)).strftime("%H:%M"), + "end": (now_utc + timedelta(hours=end_hours)).strftime("%H:%M"), + "timezone": "UTC", + "team_ids": team_ids, + } + + +def _reserved_model_list(windows_for_reserved=None, windows_for_open=None) -> list: + reserved: dict = { + "model_name": "gpt-4o-ptu", + "litellm_params": {"model": "gpt-4o", "mock_response": "reserved"}, + "model_info": {"id": "reserved-deployment"}, + } + if windows_for_reserved is not None: + reserved["model_info"]["access_windows"] = windows_for_reserved + unreserved: dict = { + "model_name": "gpt-4o-ptu", + "litellm_params": {"model": "gpt-4o", "mock_response": "open"}, + "model_info": {"id": "open-deployment"}, + } + if windows_for_open is not None: + unreserved["model_info"]["access_windows"] = windows_for_open + return [reserved, unreserved] + + +def test_access_windows_hide_reserved_deployment_from_other_teams(): + router = Router( + model_list=_reserved_model_list( + windows_for_reserved=[_access_window_offsets(-1, 1, ["team-a"])], + ), + ) + _, deployments = router._common_checks_available_deployment( + model="gpt-4o-ptu", + request_kwargs={"metadata": {"user_api_key_team_id": "team-b"}}, + ) + assert [d["model_info"]["id"] for d in deployments] == ["open-deployment"] + + +def test_access_windows_raise_when_only_reserved_deployments_remain(): + router = Router(model_list=_reserved_model_list( + windows_for_reserved=[_access_window_offsets(-1, 1, ["team-a"])], + windows_for_open=[_access_window_offsets(-1, 1, ["team-a"])], + )[:1]) + for request_kwargs in ({"metadata": {"user_api_key_team_id": "team-b"}}, {}): + with pytest.raises(litellm.BadRequestError, match="reserved for another team"): + router._common_checks_available_deployment(model="gpt-4o-ptu", request_kwargs=request_kwargs) + _, deployments = router._common_checks_available_deployment( + model="gpt-4o-ptu", + request_kwargs={"metadata": {"user_api_key_team_id": "team-a"}}, + ) + assert [d["model_info"]["id"] for d in deployments] == ["reserved-deployment"] + + +def test_reserved_deployments_drop_strategy_markers_before_filtering(): + router = Router(model_list=_reserved_model_list()[:1]) + marker = {"model_name": "gpt-4o-ptu", "litellm_params": {"model": "auto_router/semantic"}} + reserved = { + "model_name": "gpt-4o-ptu", + "litellm_params": {"model": "gpt-4o"}, + "model_info": {"access_windows": [_access_window_offsets(-1, 1, ["team-a"])]}, + } + with pytest.raises(litellm.BadRequestError, match="reserved for another team"): + router._filter_reserved_deployments( + model="gpt-4o-ptu", + healthy_deployments=[marker, reserved], + request_team_id="team-b", + ) + + +def test_access_windows_invalid_timezone_fails_router_construction(): + with pytest.raises(ValueError, match=r"gpt-4o-ptu.*access_windows"): + Router( + model_list=[ + { + "model_name": "gpt-4o-ptu", + "litellm_params": {"model": "gpt-4o", "mock_response": "x"}, + "model_info": { + "access_windows": [ + { + "start": "22:00", + "end": "06:00", + "timezone": "Mars/Olympus", + "team_ids": ["team-a"], + } + ] + }, + } + ], + ) + + +def test_access_windows_inactive_window_leaves_deployments_available(): + router = Router( + model_list=_reserved_model_list( + windows_for_reserved=[_access_window_offsets(2, 3, ["team-a"])], + ), + ) + _, deployments = router._common_checks_available_deployment( + model="gpt-4o-ptu", + request_kwargs={"metadata": {"user_api_key_team_id": "team-b"}}, + ) + assert {d["model_info"]["id"] for d in deployments} == {"reserved-deployment", "open-deployment"} + + +def test_access_windows_apply_when_calling_by_model_id(): + router = Router(model_list=_reserved_model_list( + windows_for_reserved=[_access_window_offsets(-1, 1, ["team-a"])], + )) + with pytest.raises(litellm.BadRequestError, match="reserved for another team"): + router._common_checks_available_deployment( + model="reserved-deployment", + request_kwargs={"metadata": {"user_api_key_team_id": "team-b"}}, + ) + _, deployment = router._common_checks_available_deployment( + model="reserved-deployment", + request_kwargs={"metadata": {"user_api_key_team_id": "team-a"}}, + ) + assert deployment["model_info"]["id"] == "reserved-deployment" + + +def test_access_windows_apply_when_calling_by_litellm_model_name(): + router = Router( + model_list=[ + { + "model_name": "gpt-4o-ptu", + "litellm_params": { + "model": "openai/gpt-5.6-bypass-probe", + "mock_response": "reserved", + }, + "model_info": { + "id": "reserved-litellm-model", + "access_windows": [_access_window_offsets(-1, 1, ["team-a"])], + }, + } + ], + ) + with pytest.raises(litellm.BadRequestError, match="reserved for another team"): + router._common_checks_available_deployment( + model="openai/gpt-5.6-bypass-probe", + request_kwargs={"metadata": {"user_api_key_team_id": "team-b"}}, + ) + _, deployments = router._common_checks_available_deployment( + model="openai/gpt-5.6-bypass-probe", + request_kwargs={"metadata": {"user_api_key_team_id": "team-a"}}, + ) + assert [d["model_info"]["id"] for d in deployments] == ["reserved-litellm-model"] + + +def test_access_windows_apply_to_specific_deployment_calls(): + router = Router( + model_list=[ + { + "model_name": "gpt-4o-ptu", + "litellm_params": { + "model": "openai/gpt-5.6-specific-probe", + "mock_response": "reserved", + }, + "model_info": { + "id": "reserved-specific", + "access_windows": [_access_window_offsets(-1, 1, ["team-a"])], + }, + } + ], + ) + with pytest.raises(litellm.BadRequestError, match="reserved for another team"): + router._common_checks_available_deployment( + model="openai/gpt-5.6-specific-probe", + specific_deployment=True, + request_kwargs={"metadata": {"user_api_key_team_id": "team-b"}}, + ) + _, deployments = router._common_checks_available_deployment( + model="openai/gpt-5.6-specific-probe", + specific_deployment=True, + request_kwargs={"metadata": {"user_api_key_team_id": "team-a"}}, + ) + assert [d["model_info"]["id"] for d in deployments] == ["reserved-specific"] + + +def test_access_windows_apply_to_wildcard_early_resolve(): + router = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "mock_response": "reserved"}, + "model_info": { + "id": "reserved-wildcard", + "access_windows": [_access_window_offsets(-1, 1, ["team-a"])], + }, + } + ], + ) + with pytest.raises(litellm.BadRequestError, match="reserved for another team"): + router._common_checks_available_deployment( + model="openai/gpt-probe-wildcard", + request_kwargs={"metadata": {"user_api_key_team_id": "team-b"}}, + ) + _, deployments = router._common_checks_available_deployment( + model="openai/gpt-probe-wildcard", + request_kwargs={"metadata": {"user_api_key_team_id": "team-a"}}, + ) + assert [d["model_info"]["id"] for d in deployments] == ["reserved-wildcard"] + + +def test_access_windows_filter_reserved_deployments_method(): + router = Router(model_list=_reserved_model_list()) + reserved: dict = { + "model_info": { + "id": "reserved-deployment", + "access_windows": [_access_window_offsets(-1, 1, ["team-a"])], + } + } + open_deployment: dict = {"model_info": {"id": "open-deployment"}} + assert [ + d["model_info"]["id"] + for d in router._filter_reserved_deployments( + model="gpt-4o-ptu", + healthy_deployments=[reserved, open_deployment], + request_team_id="team-b", + ) + ] == ["open-deployment"] + with pytest.raises(litellm.BadRequestError, match="reserved for another team"): + router._filter_reserved_deployments( + model="gpt-4o-ptu", + healthy_deployments=[reserved], + request_team_id="team-b", + ) + assert [ + d["model_info"]["id"] + for d in router._filter_reserved_deployments( + model="gpt-4o-ptu", + healthy_deployments=[reserved, open_deployment], + request_team_id="team-a", + ) + ] == ["reserved-deployment", "open-deployment"] diff --git a/tests/test_litellm/types/test_router.py b/tests/test_litellm/types/test_router.py index 7fcb76b638f..df744a77fe4 100644 --- a/tests/test_litellm/types/test_router.py +++ b/tests/test_litellm/types/test_router.py @@ -146,3 +146,48 @@ def test_aws_session_tags_round_trip_as_sts_shaped_pairs(): def test_aws_session_tags_reject_shapes_sts_would_refuse(aws_session_tags): with pytest.raises(ValidationError, match="aws_session_tags"): LiteLLM_Params(model="bedrock/anthropic.claude-opus-5", aws_session_tags=aws_session_tags) + + +def test_model_info_parses_access_windows_time_strings(): + import datetime + + info = ModelInfo( + id="x", + access_windows=[ + { + "start": "22:00", + "end": "06:00", + "timezone": "America/New_York", + "team_ids": ["team-nightly"], + } + ], + ) + window = info.access_windows[0] + assert window.start == datetime.time(22, 0) + assert window.end == datetime.time(6, 0) + assert window.timezone == "America/New_York" + assert window.team_ids == ("team-nightly",) + + +@pytest.mark.parametrize( + "access_windows", + [ + [{"start": "25:00", "end": "06:00", "timezone": "UTC", "team_ids": ["t"]}], + [{"start": "22:00", "end": "06:00", "timezone": "Mars/Olympus", "team_ids": ["t"]}], + [{"start": "22:00", "end": "06:00", "timezone": "UTC", "team_ids": []}], + ], + ids=["invalid-time", "unknown-timezone", "empty-team-ids"], +) +def test_model_info_rejects_invalid_access_windows(access_windows): + with pytest.raises(ValidationError): + ModelInfo(id="x", access_windows=access_windows) + + +def test_model_info_rejects_offset_aware_access_window_times(): + with pytest.raises(ValidationError): + ModelInfo( + id="x", + access_windows=[ + {"start": "22:00+05:00", "end": "06:00", "timezone": "UTC", "team_ids": ["t"]} + ], + ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 38f8ac365ff..68d3ab364d6 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -33722,6 +33722,23 @@ export interface components { [key: string]: string | string[]; }; }; + /** ModelAccessWindow */ + ModelAccessWindow: { + /** + * End + * Format: time + */ + end: string; + /** + * Start + * Format: time + */ + start: string; + /** Team Ids */ + team_ids: string[]; + /** Timezone */ + timezone: string; + }; /** ModelDeprecationInfo */ ModelDeprecationInfo: { /** @@ -41879,6 +41896,8 @@ export interface components { }; /** ModelInfo */ litellm__types__router__ModelInfo: { + /** Access Windows */ + access_windows?: components["schemas"]["ModelAccessWindow"][] | null; /** Allow Fail Open */ allow_fail_open?: boolean | null; /** Base Model */ From 2ea43214c96a61dae71715da08db57ebea7c7701 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:22:43 -0500 Subject: [PATCH 02/48] feat(terraform): expose server_metadata on litellm_key so undeclared metadata is visible (#42453) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- terraform/provider/CHANGELOG.md | 1 + terraform/provider/docs/resources/key.md | 2 ++ terraform/provider/litellm/resource_key.go | 27 ++++++++++++++++ .../provider/litellm/resource_key_test.go | 32 +++++++++++++++++++ 4 files changed, 62 insertions(+) diff --git a/terraform/provider/CHANGELOG.md b/terraform/provider/CHANGELOG.md index 8f40ed6dfb7..079bb7d8667 100644 --- a/terraform/provider/CHANGELOG.md +++ b/terraform/provider/CHANGELOG.md @@ -16,6 +16,7 @@ longer signal it. ### Added +- **key**: Computed `server_metadata` attribute on `litellm_key` exposing every metadata entry the proxy stores, so metadata created outside Terraform is visible in state and drift on it shows on refresh, while `metadata` keeps tracking only the declared entries and updates keep preserving undeclared ones - **team_member_add**: `tpm_limit`, `rpm_limit`, `budget_duration`, and `allowed_models` attributes on `litellm_team_member_add`, applied to every member of the resource; `budget_duration` and `allowed_models` ride on `/team/member_add`, while the limits are sent through `/team/member_update`, which is where the proxy accepts them - **team**: Optional `team_id` argument on `litellm_team`, so teams can be created with a stable, human-readable ID instead of a provider-generated UUID; changing it forces replacement - `litellm_jwt_key_mapping` accepts `token_id` as an alternative to `key`, so a diff --git a/terraform/provider/docs/resources/key.md b/terraform/provider/docs/resources/key.md index 0ef0688830f..af2f38b5352 100644 --- a/terraform/provider/docs/resources/key.md +++ b/terraform/provider/docs/resources/key.md @@ -120,6 +120,8 @@ In addition to all arguments above, the following attributes are exported: * `key` - The generated API key. This is the actual key value that will be used for authentication. +* `server_metadata` - Map of every metadata entry the proxy stores for this key, including entries not declared in `metadata`, so drift on them is visible on refresh. Entries already exposed as their own attributes (`model_rpm_limit`, `model_tpm_limit`, `tags`, `guardrails`, `enforced_params`, `allowed_passthrough_routes`, `rpm_limit_type`, `tpm_limit_type`, `prompts`) are omitted and non-string values are JSON encoded. Terraform never writes it; `metadata` still tracks only the entries declared in the configuration. + * `spend` - The current spend for this key. This reflects the total amount spent using this key so far. ## State Management diff --git a/terraform/provider/litellm/resource_key.go b/terraform/provider/litellm/resource_key.go index 39546d588df..b471cab73e8 100644 --- a/terraform/provider/litellm/resource_key.go +++ b/terraform/provider/litellm/resource_key.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "log" + "slices" "github.com/hashicorp/go-cty/cty" "github.com/hashicorp/terraform-plugin-sdk/v2/diag" @@ -61,6 +62,12 @@ func resourceKey() *schema.Resource { Optional: true, Elem: &schema.Schema{Type: schema.TypeString}, }, + "server_metadata": { + Type: schema.TypeMap, + Computed: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Description: "Every metadata entry the proxy stores for this key, including ones not declared in metadata. Read-only, so drift on undeclared entries shows up on refresh without Terraform taking ownership of them. Entries the provider already exposes as their own attributes (model_rpm_limit, model_tpm_limit, tags, guardrails, enforced_params, allowed_passthrough_routes, rpm_limit_type, tpm_limit_type, prompts) are omitted, and non-string values are JSON encoded", + }, "tpm_limit": { Type: schema.TypeInt, Optional: true, @@ -304,6 +311,7 @@ func resourceKeyRead(ctx context.Context, d *schema.ResourceData, m interface{}) return nil } + d.Set("server_metadata", serverKeyMetadata(key.Metadata)) key.Metadata = declaredKeyMetadata(key.Metadata, d.Get("metadata").(map[string]interface{})) mapKeyToResourceData(d, key) return nil @@ -395,6 +403,25 @@ func declaredKeyMetadata(server, declared map[string]interface{}) map[string]int return result } +func serverKeyMetadata(server map[string]interface{}) map[string]string { + result := make(map[string]string, len(server)) + for k, v := range server { + if slices.Contains(keyFieldsStoredInMetadata, k) { + continue + } + if s, ok := v.(string); ok { + result[k] = s + continue + } + encoded, err := json.Marshal(v) + if err != nil { + continue + } + result[k] = string(encoded) + } + return result +} + func mergeKeyMetadata(server, oldDeclared, newDeclared map[string]interface{}) map[string]interface{} { result := make(map[string]interface{}, len(server)+len(newDeclared)) for k, v := range server { diff --git a/terraform/provider/litellm/resource_key_test.go b/terraform/provider/litellm/resource_key_test.go index fe708edd3d3..b6e67360ad0 100644 --- a/terraform/provider/litellm/resource_key_test.go +++ b/terraform/provider/litellm/resource_key_test.go @@ -600,6 +600,12 @@ func TestKeyUpdateWithoutMetadataChangePreservesServerMetadata(t *testing.T) { if got := newState.Attributes["metadata.a"]; got != "1" { t.Errorf("metadata.a = %q, want 1", got) } + if got := newState.Attributes["server_metadata.server_side"]; got != "x" { + t.Errorf("server_metadata.server_side = %q, want x", got) + } + if _, present := proxy.updates[0]["server_metadata"]; present { + t.Errorf("computed server_metadata was sent on /key/update: %v", proxy.updates[0]["server_metadata"]) + } } func TestKeyUpdateWithMetadataChangeMergesOverServerMetadata(t *testing.T) { @@ -654,6 +660,32 @@ func TestKeyReadKeepsOnlyDeclaredMetadata(t *testing.T) { } } +func TestKeyReadExposesUndeclaredMetadataInServerMetadata(t *testing.T) { + proxy := &fakeKeyProxy{metadata: map[string]interface{}{ + "a": "1", + "server_side": "x", + "model_rpm_limit": map[string]interface{}{"gpt-4o-mini": float64(5)}, + "nested": map[string]interface{}{"k": "v"}, + }} + srv := httptest.NewServer(proxy.handler()) + defer srv.Close() + client := NewClient(srv.URL, "test-key", true) + + d := newKeyResourceData(t, map[string]interface{}{"metadata": map[string]interface{}{"a": "1"}}) + d.SetId("hash-1") + if diags := resourceKeyRead(context.Background(), d, client); diags.HasError() { + t.Fatalf("Read returned error: %v", diags) + } + + if got, want := d.Get("metadata"), map[string]interface{}{"a": "1"}; !reflect.DeepEqual(got, want) { + t.Errorf("metadata in state = %v, want %v", got, want) + } + want := map[string]interface{}{"a": "1", "server_side": "x", "nested": `{"k":"v"}`} + if got := d.Get("server_metadata"); !reflect.DeepEqual(got, want) { + t.Errorf("server_metadata in state = %v, want %v", got, want) + } +} + func TestKeyUpdateSendsChangedDuration(t *testing.T) { proxy := &fakeKeyProxy{metadata: map[string]interface{}{}} srv := httptest.NewServer(proxy.handler()) From 0d0f73cd14caa72de23bb1218a138d47a9627b8c Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Tue, 22 Sep 2026 18:27:50 +0000 Subject: [PATCH 03/48] fix(mcp): restore legacy SSE and bounded cancellation cleanup (#42382) * fix(mcp): restore legacy SSE and bounded cancellation cleanup * fix(mcp): drain termination despite repeated task cancellation * fix(mcp): admit virtual keys on legacy SSE message routes * fix(mcp): preserve cancellation through late transport failures * fix(mcp): preserve process exits during cleanup * test(mcp): drain idle sockets before server shutdown --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/experimental_mcp_client/client.py | 249 +++++++--- .../mcp_server/auth/user_api_key_auth_mcp.py | 2 + .../proxy/_experimental/mcp_server/server.py | 70 ++- .../_experimental/mcp_server/sse_transport.py | 139 +----- litellm/proxy/_types.py | 3 + tests/mcp_tests/test_mcp_server.py | 29 +- .../test_mcp_client.py | 461 +++++++++++++++++- .../auth/test_user_api_key_auth_mcp.py | 5 + .../mcp_server/test_mcp_server.py | 295 ++++++----- .../proxy/auth/test_route_checks.py | 27 + 10 files changed, 900 insertions(+), 380 deletions(-) diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 49434befd4e..1ccae8de35f 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -7,11 +7,11 @@ import base64 import hashlib import json import os -from collections.abc import Awaitable, Callable, Generator, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Generator, Sequence from contextlib import AbstractAsyncContextManager from functools import partial from types import MappingProxyType -from typing import Any, Final, TypeAlias, TypeVar +from typing import Final, TypeAlias, TypeVar, cast import anyio import httpx2 @@ -121,14 +121,14 @@ def _strip_header_whitespace(headers: dict[str, str]) -> dict[str, str]: } -def _first_non_cancelled_cause(exc: BaseException) -> BaseException | None: +def _first_non_cancelled_cause(exc: BaseException, cleanup_errors: tuple[Exception, ...] = ()) -> BaseException | None: queue: Final[list[BaseException]] = [exc] while queue: current = queue.pop(0) nested = getattr(current, "exceptions", None) if nested: queue.extend(nested) - elif not isinstance(current, asyncio.CancelledError): + elif not isinstance(current, asyncio.CancelledError) and not any(current is error for error in cleanup_errors): return current return None @@ -159,7 +159,59 @@ _ListPage = TypeVar("_ListPage", bound=PaginatedResult) _ListItem = TypeVar("_ListItem") +async def _run_bounded_cleanup(operation: Callable[[], Awaitable[TSessionResult]], deadline: float) -> TSessionResult: + async def run() -> TSessionResult: + with anyio.fail_after(max(0, deadline - anyio.current_time()), shield=True): + return await operation() + + # A cancelled asyncio.gather repeatedly forwards Task.cancel, bypassing AnyIO shields. + # Isolate only cleanup, and drain it before propagating the caller's cancellation. + task: Final = asyncio.create_task(run()) + interrupted: asyncio.CancelledError | None = None + with anyio.CancelScope(shield=True): + while not task.done(): + try: + await asyncio.shield(task) + except asyncio.CancelledError as exc: + interrupted = exc + except Exception: + break + if interrupted is not None: + if not task.cancelled(): + task.exception() + raise interrupted + return task.result() + + +class _MCPResponseStream(httpx2.AsyncByteStream): + def __init__(self, stream: httpx2.AsyncByteStream, record_error: Callable[[Exception], None]) -> None: + self._stream: Final = stream + self._record_error: Final = record_error + + async def __aiter__(self) -> AsyncIterator[bytes]: + try: + async for chunk in self._stream: + yield chunk + except Exception as error: + self._record_error(error) + raise + + async def aclose(self) -> None: + try: + await self._stream.aclose() + except Exception as error: + self._record_error(error) + raise + + class _MCPHTTPClient(httpx2.AsyncClient): + cleanup_scope: anyio.CancelScope | None = None + cleanup_errors: tuple[Exception, ...] = () + + def _record_cleanup_error(self, error: Exception) -> None: + if self.cleanup_scope is not None and self.cleanup_scope.shield: + self.cleanup_errors += (error,) + async def send( self, request: httpx2.Request, @@ -168,11 +220,29 @@ class _MCPHTTPClient(httpx2.AsyncClient): auth: AuthTypes | UseClientDefault | None = httpx2.USE_CLIENT_DEFAULT, follow_redirects: bool | UseClientDefault = httpx2.USE_CLIENT_DEFAULT, ) -> httpx2.Response: - response: Final = await super().send(request, stream=stream, auth=auth, follow_redirects=follow_redirects) - if request.method == "POST" and response.is_error and response.status_code != 404: - await response.aclose() - response.raise_for_status() - return response + if request.method == "DELETE" and self.cleanup_scope is not None: + + async def terminate() -> httpx2.Response: + termination: Final = await super(_MCPHTTPClient, self).send( + request, stream=stream, auth=auth, follow_redirects=follow_redirects + ) + await termination.aread() + return termination + + return await _run_bounded_cleanup(terminate, self.cleanup_scope.deadline) + try: + response: Final = await super().send(request, stream=stream, auth=auth, follow_redirects=follow_redirects) + if request.method == "POST" and response.is_error and response.status_code != 404: + await response.aclose() + response.raise_for_status() + if stream: + response.stream = _MCPResponseStream( + cast(httpx2.AsyncByteStream, response.stream), self._record_cleanup_error + ) + return response + except Exception as error: + self._record_cleanup_error(error) + raise class MCPSigV4Auth(httpx2.Auth): @@ -458,6 +528,7 @@ class MCPClient: self, transport_ctx: _TransportContext, operation: Callable[[ClientSession], Awaitable[TSessionResult]], + http_client: httpx2.AsyncClient | None = None, ) -> TSessionResult: """ Execute an operation within a transport and session context. @@ -466,69 +537,97 @@ class MCPClient: so that upstream MCP servers can request LLM inference (sampling), user input (elicitation), or send log messages. """ - transport: Final = await transport_ctx.__aenter__() in_flight_error: BaseException | None = None - try: - read_stream: Final = transport[0] - write_stream: Final = transport[1] - stream_error: Final[asyncio.Future[Exception]] = asyncio.get_running_loop().create_future() - - async def receive_message( - message: ServerNotification | Exception, - ) -> None: - if not isinstance(message, (ValueError, httpx2.HTTPError, OSError)): - return - if not stream_error.done(): - stream_error.set_result(message) - # The SDK closes pending requests when its message handler raises. - raise RuntimeError("MCP response stream failed") - - # Build session kwargs with optional callbacks - session_kwargs: Final[dict[str, Any]] = {} - if self._sampling_callback is not None: - session_kwargs["sampling_callback"] = self._sampling_callback - if self._elicitation_callback is not None: - session_kwargs["elicitation_callback"] = self._elicitation_callback - if self._logging_callback is not None: - session_kwargs["logging_callback"] = self._logging_callback - # The SDK drops a response stream that ends without a JSON-RPC reply, so nothing else - # ever fails the request. - session_ctx: Final = ClientSession( - read_stream, - write_stream, - read_timeout_seconds=self.timeout, - message_handler=receive_message, - **session_kwargs, - ) - session: Final = await session_ctx.__aenter__() + with anyio.CancelScope() as cleanup_scope: + if isinstance(http_client, _MCPHTTPClient): + http_client.cleanup_scope = cleanup_scope try: - init_result: Final = await session.initialize() - self._last_initialize_instructions = None - if init_result is not None: - ins: Final = getattr(init_result, "instructions", None) - if isinstance(ins, str) and ins.strip(): - self._last_initialize_instructions = ins.strip() - return await operation(session) - except MCPError: - if stream_error.done(): - raise stream_error.result() - raise - finally: + transport: Final = await transport_ctx.__aenter__() try: - await session_ctx.__aexit__(None, None, None) + read_stream: Final = transport[0] + write_stream: Final = transport[1] + stream_error: Final[asyncio.Future[Exception]] = asyncio.get_running_loop().create_future() + + async def receive_message( + message: ServerNotification | Exception, + ) -> None: + if not isinstance(message, (ValueError, httpx2.HTTPError, OSError)): + return + if not stream_error.done(): + stream_error.set_result(message) + # The SDK closes pending requests when its message handler raises. + raise RuntimeError("MCP response stream failed") + + session_kwargs: Final = { + name: callback + for name, callback in ( + ("sampling_callback", self._sampling_callback), + ("elicitation_callback", self._elicitation_callback), + ("logging_callback", self._logging_callback), + ) + if callback is not None + } + # The SDK drops a response stream that ends without a JSON-RPC reply, so nothing else + # ever fails the request. + session_ctx: Final = ClientSession( + read_stream, + write_stream, + read_timeout_seconds=self.timeout, + message_handler=receive_message, + **session_kwargs, + ) + session: Final = await session_ctx.__aenter__() + try: + init_result: Final = await session.initialize() + instructions: Final = getattr(init_result, "instructions", None) + self._last_initialize_instructions = ( + instructions.strip() or None if isinstance(instructions, str) else None + ) + result: Final = await operation(session) + except BaseException as operation_error: + in_flight_error = operation_error + if isinstance(operation_error, MCPError) and stream_error.done(): + raise stream_error.result() + raise + finally: + cleanup_scope.shield = True + cleanup_scope.deadline = anyio.current_time() + 5 + try: + await session_ctx.__aexit__(None, None, None) + except (Exception, asyncio.CancelledError) as e: + verbose_logger.debug("Error during session context exit: %s", e) + if in_flight_error is None and isinstance(e, asyncio.CancelledError): + raise except BaseException as e: - verbose_logger.debug("Error during session context exit: %s", e) - except BaseException as e: - in_flight_error = e - raise - finally: - try: - await transport_ctx.__aexit__(None, None, None) - except BaseException as exit_error: - verbose_logger.debug("Error during transport context exit: %s", exit_error) - root_cause: Final = _first_non_cancelled_cause(exit_error) - if root_cause is not None and isinstance(in_flight_error, asyncio.CancelledError): - raise root_cause from in_flight_error + in_flight_error = e + raise + finally: + cleanup_scope.shield = True + cleanup_scope.deadline = min(cleanup_scope.deadline, anyio.current_time() + 5) + try: + await transport_ctx.__aexit__(None, None, None) + except (Exception, asyncio.CancelledError) as exit_error: + verbose_logger.debug("Error during transport context exit: %s", exit_error) + if in_flight_error is None and isinstance(exit_error, asyncio.CancelledError): + raise + root_cause: Final = _first_non_cancelled_cause( + exit_error, http_client.cleanup_errors if isinstance(http_client, _MCPHTTPClient) else () + ) + if root_cause is not None and isinstance(in_flight_error, asyncio.CancelledError): + raise root_cause from in_flight_error + finally: + cleanup_scope.shield = False + if isinstance(http_client, _MCPHTTPClient): + http_client.cleanup_errors = () + http_client.cleanup_scope = None + await anyio.lowlevel.checkpoint_if_cancelled() + if cleanup_scope.cancel_called: + raise ( + in_flight_error + if in_flight_error is not None + else asyncio.CancelledError("MCP session cleanup timed out") + ) + return result async def run_with_session( self, @@ -542,10 +641,11 @@ class MCPClient: (call_tool / list_tools under raise_on_error), so an expected pass-through re-auth does not emit a warning per call; every other caller keeps the operator-visible warning.""" http_client: httpx2.AsyncClient | None = None + close_cancellation: asyncio.CancelledError | None = None try: self._last_initialize_instructions = None transport_ctx, http_client = self._create_transport_context() - return await self._execute_session_operation(transport_ctx, operation) + result: Final = await self._execute_session_operation(transport_ctx, operation, http_client=http_client) except Exception as e: read_timeout: Final = as_mcp_read_timeout(e) if read_timeout is not None: @@ -561,9 +661,16 @@ class MCPClient: finally: if http_client is not None: try: - await http_client.aclose() - except BaseException as e: + await _run_bounded_cleanup(http_client.aclose, anyio.current_time() + 1) + except (Exception, asyncio.CancelledError) as e: verbose_logger.debug("Error during http_client cleanup: %s", e) + if isinstance(e, asyncio.CancelledError): + close_cancellation = e + + if close_cancellation is not None: + raise close_cancellation + await anyio.lowlevel.checkpoint_if_cancelled() + return result def update_auth_value(self, mcp_auth_value: str | dict[str, str]) -> None: """ diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 60dc91a69cc..d6478421834 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -663,6 +663,8 @@ class MCPRequestHandler: # with ``server.py::_get_mcp_servers_in_path``, which also accepts the # un-rewritten form (some entry points may skip the # ``dynamic_mcp_route`` rewrite). + if path.rstrip("/") in ("/mcp/sse", "/mcp/sse/messages"): + return [] segments: Final = [s for s in path.split("/") if s] if len(segments) >= 2 and segments[1] == "mcp" and segments[0] != "mcp": return [segments[0]] diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 433b693fcae..fe508fc22c7 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -21,6 +21,7 @@ from fastapi import FastAPI, HTTPException from pydantic import ConfigDict, TypeAdapter, ValidationError from starlette.requests import Request as StarletteRequest from starlette.responses import JSONResponse +from starlette.routing import Route from starlette.types import Message, Receive, Scope, Send from litellm._logging import verbose_logger @@ -474,6 +475,8 @@ if MCP_AVAILABLE: AuthContextMiddleware, auth_context_var, ) + from mcp.server.auth.middleware.bearer_auth import AuthenticatedUser + from mcp.server.auth.provider import AccessToken from mcp.server.context import ServerRequestContext from mcp.server.lowlevel.server import NotificationOptions from mcp.server.models import InitializationOptions @@ -573,7 +576,7 @@ if MCP_AVAILABLE: version=LITELLM_MCP_SERVER_VERSION, ) server.create_initialization_options = types.MethodType(_gateway_create_initialization_options, server) - sse: Final[SseServerTransport] = SseServerTransport("/mcp/sse/messages") + sse: Final[SseServerTransport] = SseServerTransport("/sse/messages") # Create session managers session_manager_stateless: Final = StreamableHTTPSessionManager( @@ -629,18 +632,9 @@ if MCP_AVAILABLE: # Keep this alias so existing references to session_manager still work session_manager: Final = session_manager_stateless - # Create SSE session manager - sse_session_manager: Final = StreamableHTTPSessionManager( - app=server, - event_store=None, - json_response=False, # Use SSE responses for this endpoint - stateless=True, - ) - # Context managers for proper lifecycle management _session_manager_cm = None _session_manager_stateful_cm = None - _sse_session_manager_cm = None _stateful_auth_context_cleanup_task: asyncio.Task | None = None async def _purge_expired_stateful_session_auth_contexts( @@ -732,7 +726,6 @@ if MCP_AVAILABLE: _SESSION_MANAGERS_INITIALIZED, \ _session_manager_cm, \ _session_manager_stateful_cm, \ - _sse_session_manager_cm, \ _stateful_auth_context_cleanup_task # Use async lock to prevent concurrent initialization @@ -745,12 +738,10 @@ if MCP_AVAILABLE: # Start the session managers with context managers _session_manager_cm = session_manager_stateless.run() _session_manager_stateful_cm = session_manager_stateful.run() - _sse_session_manager_cm = sse_session_manager.run() # Enter the context managers await _session_manager_cm.__aenter__() await _session_manager_stateful_cm.__aenter__() - await _sse_session_manager_cm.__aenter__() _stateful_auth_context_cleanup_task = asyncio.create_task(_cleanup_expired_stateful_session_auth_contexts()) _SESSION_MANAGERS_INITIALIZED = True @@ -762,7 +753,6 @@ if MCP_AVAILABLE: _SESSION_MANAGERS_INITIALIZED, \ _session_manager_cm, \ _session_manager_stateful_cm, \ - _sse_session_manager_cm, \ _stateful_auth_context_cleanup_task if _SESSION_MANAGERS_INITIALIZED: @@ -773,8 +763,6 @@ if MCP_AVAILABLE: _stateful_auth_context_cleanup_task.cancel() with contextlib.suppress(asyncio.CancelledError): await _stateful_auth_context_cleanup_task - if _sse_session_manager_cm: - await _sse_session_manager_cm.__aexit__(None, None, None) if _session_manager_stateful_cm: await _session_manager_stateful_cm.__aexit__(None, None, None) if _session_manager_cm: @@ -784,7 +772,6 @@ if MCP_AVAILABLE: _session_manager_cm = None _session_manager_stateful_cm = None - _sse_session_manager_cm = None _stateful_auth_context_cleanup_task = None _SESSION_MANAGERS_INITIALIZED = False @@ -1054,6 +1041,8 @@ if MCP_AVAILABLE: """ import re + if path.rstrip("/") in ("/mcp/sse", "/mcp/sse/messages"): + return None mcp_servers_from_path: list[str] | None = None segments: Final = [s for s in path.split("/") if s] if len(segments) >= 2 and segments[1] == "mcp" and segments[0] != "mcp": @@ -2286,7 +2275,24 @@ if MCP_AVAILABLE: async def handle_sse_mcp(scope: Scope, receive: Receive, send: Send) -> None: """Handle MCP requests through SSE.""" try: - path: Final[str] = scope.get("path", "") + bad_version: Final = unsupported_protocol_version(scope) + if bad_version is not None: + supported: Final = ", ".join(sorted(HANDSHAKE_PROTOCOL_VERSIONS)) + await JSONResponse( + status_code=400, + content={ # mutable-ok: JSON-RPC error payload + "jsonrpc": "2.0", + "id": None, + "error": { + "code": INVALID_REQUEST, + "message": f"Unsupported MCP-Protocol-Version {bad_version}; supported: {supported}", + }, + }, + )(scope, receive, send) + return + from litellm.proxy.auth.auth_utils import get_request_route + + path: Final = get_request_route(StarletteRequest(scope)) ( user_api_key_auth, mcp_auth_header, @@ -2353,9 +2359,14 @@ if MCP_AVAILABLE: client_ip=_sse_client_ip, ) - if not _SESSION_MANAGERS_INITIALIZED: - await initialize_session_managers() - await asyncio.sleep(0.1) + owner: Final = _owner_fingerprint_for(user_api_key_auth, oauth2_headers, _sse_client_ip) + transport_scope: Final[Scope] = { + **scope, + "user": AuthenticatedUser(AccessToken(token=owner, client_id=owner, scopes=[])), + } + if scope["method"] == "POST": + await sse.handle_post_message(transport_scope, receive, send) + return async with _gateway_initialize_instructions_request_scope( user_api_key_auth, @@ -2364,7 +2375,8 @@ if MCP_AVAILABLE: scoped_server_endpoint=scoped_server_endpoint, is_initialize=scope.get("method") == "GET", ): - await sse_session_manager.handle_request(scope, receive, send) + async with sse.connect_sse(transport_scope, receive, send) as (read_stream, write_stream): + await server.run(read_stream, write_stream, server.create_initialization_options()) except MCPUpstreamAuthError as e: # Upstream delegated auth returned 401; surface it to the client so # standards-compliant MCP clients trigger the upstream OAuth flow. @@ -2387,7 +2399,6 @@ if MCP_AVAILABLE: # Try to send a graceful error response for non-HTTP exceptions try: # Send a proper HTTP error response instead of letting the exception bubble up - from starlette.responses import JSONResponse from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR error_response: Final = JSONResponse( @@ -2418,11 +2429,22 @@ if MCP_AVAILABLE: """ return {"enabled": MCP_AVAILABLE} + class _LegacySseEndpoint: + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + await handle_sse_mcp(scope, receive, send) + + for sse_path, sse_method in ( + ("/sse", "GET"), + ("/sse/", "GET"), + ("/sse/messages", "POST"), + ("/sse/messages/", "POST"), + ): + app.router.routes.append(Route(sse_path, endpoint=_LegacySseEndpoint(), methods=[sse_method])) + # Mount the MCP handlers app.mount("/", handle_streamable_http_mcp) app.mount("/mcp", handle_streamable_http_mcp) app.mount("/{mcp_server_name}/mcp", handle_streamable_http_mcp) - app.mount("/sse", handle_sse_mcp) app.add_middleware(AuthContextMiddleware) ######################################################## diff --git a/litellm/proxy/_experimental/mcp_server/sse_transport.py b/litellm/proxy/_experimental/mcp_server/sse_transport.py index 2a08a5f8f7a..839d2eebbbe 100644 --- a/litellm/proxy/_experimental/mcp_server/sse_transport.py +++ b/litellm/proxy/_experimental/mcp_server/sse_transport.py @@ -1,138 +1,3 @@ -""" -This is a modification of code from: https://github.com/SecretiveShell/MCP-Bridge/blob/master/mcp_bridge/mcp_server/sse_transport.py +from mcp.server.sse import SseServerTransport -Credit to the maintainers of SecretiveShell for their SSE Transport implementation - -""" - -from contextlib import asynccontextmanager -from typing import Any, Final -from urllib.parse import quote -from uuid import UUID, uuid4 - -import anyio -from anyio.streams.memory import MemoryObjectReceiveStream, MemoryObjectSendStream -from fastapi.requests import Request -from fastapi.responses import Response -from mcp import types -from pydantic import ValidationError -from sse_starlette import EventSourceResponse -from starlette.types import Receive, Scope, Send - -from litellm._logging import verbose_logger - - -class SseServerTransport: - """ - SSE server transport for MCP. This class provides _two_ ASGI applications, - suitable to be used with a framework like Starlette and a server like Hypercorn: - - 1. connect_sse() is an ASGI application which receives incoming GET requests, - and sets up a new SSE stream to send server messages to the client. - 2. handle_post_message() is an ASGI application which receives incoming POST - requests, which should contain client messages that link to a - previously-established SSE session. - """ - - _endpoint: str - _read_stream_writers: dict[UUID, MemoryObjectSendStream[types.JSONRPCMessage | Exception]] - - def __init__(self, endpoint: str) -> None: - """ - Creates a new SSE server transport, which will direct the client to POST - messages to the relative or absolute URL given. - """ - - super().__init__() - self._endpoint = endpoint - self._read_stream_writers = {} - verbose_logger.debug("SseServerTransport initialized with endpoint: %s", endpoint) - - @asynccontextmanager - async def connect_sse(self, request: Request): - if request.scope["type"] != "http": - verbose_logger.error("connect_sse received non-HTTP request") - raise ValueError("connect_sse can only handle HTTP requests") - - verbose_logger.debug("Setting up SSE connection") - read_stream: MemoryObjectReceiveStream[types.JSONRPCMessage | Exception] - read_stream_writer: MemoryObjectSendStream[types.JSONRPCMessage | Exception] - - write_stream: MemoryObjectSendStream[types.JSONRPCMessage] - write_stream_reader: MemoryObjectReceiveStream[types.JSONRPCMessage] - - read_stream_writer, read_stream = anyio.create_memory_object_stream(0) - write_stream, write_stream_reader = anyio.create_memory_object_stream(0) - - session_id: Final = uuid4() - session_uri: Final = f"{quote(self._endpoint)}?session_id={session_id.hex}" - self._read_stream_writers[session_id] = read_stream_writer - verbose_logger.debug("Created new session with ID: %s", session_id) - - sse_stream_writer: MemoryObjectSendStream[dict[str, Any]] - sse_stream_reader: MemoryObjectReceiveStream[dict[str, Any]] - sse_stream_writer, sse_stream_reader = anyio.create_memory_object_stream(0, dict[str, Any]) - - async def sse_writer(): - verbose_logger.debug("Starting SSE writer") - async with sse_stream_writer, write_stream_reader: - await sse_stream_writer.send({"event": "endpoint", "data": session_uri}) - verbose_logger.debug("Sent endpoint event: %s", session_uri) - - async for message in write_stream_reader: - verbose_logger.debug("Sending message via SSE: %s", message) - await sse_stream_writer.send( - { - "event": "message", - "data": message.model_dump_json(by_alias=True, exclude_none=True), - } - ) - - async with anyio.create_task_group() as tg: - response: Final = EventSourceResponse(content=sse_stream_reader, data_sender_callable=sse_writer) - verbose_logger.debug("Starting SSE response task") - tg.start_soon(response, request.scope, request.receive, request._send) - - verbose_logger.debug("Yielding read and write streams") - yield (read_stream, write_stream) - - async def handle_post_message(self, scope: Scope, receive: Receive, send: Send) -> Response: - verbose_logger.debug("Handling POST message") - request: Final = Request(scope, receive) - - session_id_param: Final = request.query_params.get("session_id") - if session_id_param is None: - verbose_logger.warning("Received request without session_id") - response = Response("session_id is required", status_code=400) - return response - - try: - session_id: Final = UUID(hex=session_id_param) - verbose_logger.debug("Parsed session ID: %s", session_id) - except ValueError: - verbose_logger.warning("Received invalid session ID: %s", session_id_param) - response = Response("Invalid session ID", status_code=400) - return response - - writer: Final = self._read_stream_writers.get(session_id) - if not writer: - verbose_logger.warning("Could not find session for ID: %s", session_id) - response = Response("Could not find session", status_code=404) - return response - - json: Final = await request.json() - verbose_logger.debug("Received JSON: %s", json) - - try: - message: Final = types.JSONRPCMessage.model_validate(json) - verbose_logger.debug("Validated client message: %s", message) - except ValidationError as err: - verbose_logger.error("Failed to parse message: %s", err) - response = Response("Could not parse message", status_code=400) - await writer.send(err) - return response - - verbose_logger.debug("Sending message to writer: %s", message) - response = Response("Accepted", status_code=202) - await writer.send(message) - return response +__all__ = ("SseServerTransport",) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 8b2c81fea77..76df8790b92 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -533,6 +533,9 @@ class LiteLLMRoutes(enum.Enum): mcp_inference_routes = [ "/mcp", "/mcp/", + "/mcp/sse/", + "/mcp/sse/messages", + "/mcp/sse/messages/", "/mcp/proxy", "/mcp/{subpath}", "/mcp/tools", diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index 2b92367f186..d38a11ec927 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -456,9 +456,15 @@ async def test_sse_mcp_handler_mock(): """Test the SSE MCP handler functionality""" from litellm.proxy._types import UserAPIKeyAuth - # Mock the SSE session manager and its methods - mock_sse_session_manager = AsyncMock() - mock_sse_session_manager.handle_request = AsyncMock() + read_stream, write_stream = MagicMock(), MagicMock() + + @asynccontextmanager + async def connect_sse(scope, receive, send): + yield read_stream, write_stream + + mock_sse = MagicMock() + mock_sse.connect_sse.side_effect = connect_sse + run = AsyncMock() # Mock scope, receive, send with proper ASGI scope format mock_scope = { @@ -483,13 +489,14 @@ async def test_sse_mcp_handler_mock(): ) with ( + patch("litellm.proxy._experimental.mcp_server.server.server.run", run), patch( "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.server.sse_session_manager", - mock_sse_session_manager, + "litellm.proxy._experimental.mcp_server.server.sse", + mock_sse, ), patch( "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", @@ -504,10 +511,8 @@ async def test_sse_mcp_handler_mock(): # Call the handler await handle_sse_mcp(mock_scope, mock_receive, mock_send) - # Verify SSE session manager handle_request was called - mock_sse_session_manager.handle_request.assert_called_once_with( - mock_scope, mock_receive, mock_send - ) + assert run.await_args.args[:2] == (read_stream, write_stream) + assert mock_sse.connect_sse.call_args.args[0]["path"] == "/mcp/sse" @pytest.mark.asyncio @@ -545,7 +550,10 @@ async def test_sse_mcp_handler_propagates_passthrough_401(): True, ), patch( - "litellm.proxy._experimental.mcp_server.server.sse_session_manager", + "litellm.proxy._experimental.mcp_server.server.sse", + ) as transport, + patch( + "litellm.proxy._experimental.mcp_server.server.server.run", AsyncMock(), ), patch( @@ -569,6 +577,7 @@ async def test_sse_mcp_handler_propagates_passthrough_401(): with pytest.raises(HTTPException) as excinfo: await handle_sse_mcp(mock_scope, mock_receive, mock_send) + transport.connect_sse.assert_not_called() assert excinfo.value.status_code == 401 assert excinfo.value.headers and "WWW-Authenticate" in excinfo.value.headers diff --git a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py index 6c20ef135ba..b260d240f29 100644 --- a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py +++ b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py @@ -552,6 +552,50 @@ class TestExecuteSessionOperationSurfacesTransportError: with pytest.raises(asyncio.CancelledError): await client._execute_session_operation(transport_ctx, _op) + @pytest.mark.asyncio + @pytest.mark.parametrize("failure_phase", ("early", "late", "mixed")) + @patch("litellm.experimental_mcp_client.client.ClientSession") + async def test_response_close_preserves_cancellation_and_original_errors(self, session_class, failure_phase): + closed: Final = asyncio.Event() + close_error: Final = httpx2.ReadError("response close failed") + connect_error: Final = httpx2.ConnectError("another request failed before cancellation") + cancelled: Final = asyncio.CancelledError("caller cancelled") + + class FailingCloseStream(httpx2.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b"pending" + + async def aclose(self) -> None: + closed.set() + raise close_error + + client: Final = MCPClient(server_url="https://example.com/mcp") + async with client._create_httpx_client_factory( + transport=httpx2.MockTransport(lambda _: httpx2.Response(200, stream=FailingCloseStream())) + )() as http_client: + response: Final = await http_client.send(http_client.build_request("POST", client.server_url), stream=True) + + async def initialize(): + if failure_phase == "early": + await response.aclose() + raise cancelled + + async def close_transport(*args): + if failure_phase == "early": + return + try: + await response.aclose() + except httpx2.ReadError as error: + failures: Final = [error, connect_error] if failure_phase == "mixed" else [error] + raise _FakeExceptionGroup("transport", [_FakeExceptionGroup("reader", failures)]) + + self._make_session(session_class, initialize) + expected: Final = close_error if failure_phase == "early" else connect_error if failure_phase == "mixed" else cancelled + with pytest.raises(type(expected)) as caught: + await client._execute_session_operation(self._make_transport(close_transport), AsyncMock(), http_client) + assert caught.value is expected + assert closed.is_set() + @pytest.mark.asyncio @patch("litellm.experimental_mcp_client.client.ClientSession") async def test_cleanup_error_after_success_is_swallowed(self, mock_session_cls): @@ -568,6 +612,94 @@ class TestExecuteSessionOperationSurfacesTransportError: assert result == "done" + @pytest.mark.asyncio + @patch("litellm.experimental_mcp_client.client.ClientSession") + async def test_session_entry_failure_still_closes_transport(self, session_class): + failure: Final = RuntimeError("session dispatcher did not start") + session_class.return_value.__aenter__ = AsyncMock(side_effect=failure) + closed: Final = asyncio.Event() + + async def close_transport(*args): + await anyio.lowlevel.checkpoint() + closed.set() + + transport: Final = self._make_transport(close_transport) + client: Final = MCPClient(server_url="https://example.com/mcp") + with pytest.raises(RuntimeError) as caught: + await client._execute_session_operation(transport, AsyncMock()) + assert caught.value is failure + assert closed.is_set() + + @pytest.mark.asyncio + @pytest.mark.parametrize("original_error", (False, True)) + @patch("litellm.experimental_mcp_client.client.ClientSession") + async def test_session_exit_cancellation_preserves_original_failure(self, session_class, original_error): + self._make_session(session_class, AsyncMock(return_value=None)) + cancelled: Final = asyncio.CancelledError("cancelled while closing session") + session_class.return_value.__aexit__ = AsyncMock(side_effect=cancelled) + original: Final = RuntimeError("operation failed") + transport: Final = self._make_transport(None) + client: Final = MCPClient(server_url="https://example.com/mcp") + + async def operation(session): + if original_error: + raise original + return "done" + + with pytest.raises(RuntimeError if original_error else asyncio.CancelledError) as caught: + await client._execute_session_operation(transport, operation) + assert caught.value is (original if original_error else cancelled) + transport.__aexit__.assert_awaited_once() + + @pytest.mark.asyncio + @pytest.mark.parametrize("signal_type", (SystemExit, KeyboardInterrupt)) + @pytest.mark.parametrize("phase", ("session", "transport")) + @patch("litellm.experimental_mcp_client.client.ClientSession") + async def test_cleanup_preserves_process_exit(self, session_class, phase, signal_type): + self._make_session(session_class, AsyncMock(return_value=None)) + signal: Final = signal_type("process stopping") + if phase == "session": + session_class.return_value.__aexit__ = AsyncMock(side_effect=signal) + transport: Final = self._make_transport(signal if phase == "transport" else None) + client: Final = MCPClient(server_url="https://example.com/mcp") + with pytest.raises(signal_type) as caught: + await client._execute_session_operation(transport, AsyncMock(return_value="done")) + assert caught.value is signal + transport.__aexit__.assert_awaited_once() + + @pytest.mark.asyncio + @patch("litellm.experimental_mcp_client.client.ClientSession") + async def test_session_and_termination_share_one_cleanup_deadline(self, session_class): + self._make_session(session_class, AsyncMock(return_value=None)) + deleting: Final = asyncio.Event() + + async def close_session(*args): + await anyio.sleep(1) + + async def respond(request: httpx2.Request) -> httpx2.Response: + deleting.set() + await anyio.sleep_forever() + raise AssertionError("termination unexpectedly resumed") + + client: Final = MCPClient(server_url="https://example.com/mcp") + http_client: Final = client._create_httpx_client_factory(transport=httpx2.MockTransport(respond))() + session_class.return_value.__aexit__ = AsyncMock(side_effect=close_session) + + async def close_transport(*args): + await http_client.delete(client.server_url) + + before: Final = anyio.current_time() + try: + with pytest.raises(asyncio.CancelledError): + await client._execute_session_operation( + self._make_transport(close_transport), AsyncMock(return_value="completed"), http_client=http_client + ) + assert deleting.is_set() + assert 4.8 <= anyio.current_time() - before < 5.8 + finally: + await http_client.aclose() + + class TestMCPClientResolvedAuth: """A pre-resolved httpx2.Auth is attached to the upstream client's auth= slot.""" @@ -736,7 +868,7 @@ async def test_run_with_session_quiet_on_error_demotes_warning_to_debug(): async def _op(_session): raise boom - async def _fake_exec(_transport_ctx, _operation): + async def _fake_exec(_transport_ctx, _operation, http_client=None): raise boom with patch.object(client, "_create_transport_context", return_value=(object(), None)): @@ -1486,7 +1618,9 @@ async def test_http_response_handler_preserves_success_and_http_errors(status_co client: Final = MCPClient(server_url="https://example.com/mcp", timeout=30) async with client._create_httpx_client_factory(transport=httpx2.MockTransport(respond))() as http_client: operation: Final = client._execute_session_operation( - streamable_http_client(client.server_url, http_client=http_client), lambda session: session.list_tools() + streamable_http_client(client.server_url, http_client=http_client), + lambda session: session.list_tools(), + http_client=http_client, ) if status_code == 200: result: Final = await asyncio.wait_for(operation, timeout=3) @@ -1816,13 +1950,14 @@ async def test_interrupted_http_response_preserves_the_transport_failure() -> No def respond(request: httpx2.Request) -> httpx2.Response: return httpx2.Response(200, headers={"Content-Type": "application/json"}, stream=_InterruptedHTTPBody()) - async with httpx2.AsyncClient(transport=httpx2.MockTransport(respond)) as http_client: - client: Final = MCPClient(server_url="https://example.com/mcp", timeout=30) + client: Final = MCPClient(server_url="https://example.com/mcp", timeout=30) + async with client._create_httpx_client_factory(transport=httpx2.MockTransport(respond))() as http_client: with pytest.raises(httpx2.RemoteProtocolError, match="secret-incomplete-response"): await asyncio.wait_for( client._execute_session_operation( streamable_http_client(client.server_url, http_client=http_client), lambda session: session.list_tools(), + http_client=http_client, ), timeout=3, ) @@ -1906,7 +2041,7 @@ async def test_optional_discovery_capabilities_and_errors( "jsonrpc": "2.0", "id": payload.id, "result": { - "protocolVersion": payload.params["protocolVersion"], + "protocolVersion": (payload.params or {})["protocolVersion"], "capabilities": {} if outcome == "absent" else {advertised if outcome == "other_capability" else capability: {}}, @@ -1985,7 +2120,7 @@ async def test_optional_discovery_uses_each_sessions_capabilities(supports_first return httpx2.Response(202) result: Final = ( { - "protocolVersion": payload.params["protocolVersion"], + "protocolVersion": (payload.params or {})["protocolVersion"], "capabilities": next(capabilities), "serverInfo": {"name": "changing", "version": "1"}, } @@ -2030,7 +2165,7 @@ async def test_optional_discovery_preserves_cancellation(method: str) -> None: "jsonrpc": "2.0", "id": payload.id, "result": { - "protocolVersion": payload.params["protocolVersion"], + "protocolVersion": (payload.params or {})["protocolVersion"], "capabilities": {"resources": {}, "prompts": {}}, "serverInfo": {"name": "pending", "version": "1"}, }, @@ -2112,7 +2247,7 @@ async def test_optional_discovery_collects_all_pages(method: str, session_id: st "jsonrpc": "2.0", "id": payload.id, "result": { - "protocolVersion": payload.params["protocolVersion"], + "protocolVersion": (payload.params or {})["protocolVersion"], "capabilities": {"prompts": {}, "resources": {}}, "serverInfo": {"name": "paged", "version": "1"}, }, @@ -2200,7 +2335,7 @@ async def test_optional_discovery_rejects_incomplete_walks( "jsonrpc": "2.0", "id": payload.id, "result": { - "protocolVersion": payload.params["protocolVersion"], + "protocolVersion": (payload.params or {})["protocolVersion"], "capabilities": {"prompts": {}, "resources": {}}, "serverInfo": {"name": "interrupted", "version": "1"}, }, @@ -2281,7 +2416,7 @@ async def test_optional_discovery_allows_exhaustion_at_page_cap(method: str, mon return httpx2.Response(202) if payload.method == "initialize": result: Final = { - "protocolVersion": payload.params["protocolVersion"], + "protocolVersion": (payload.params or {})["protocolVersion"], "capabilities": {"prompts": {}, "resources": {}}, "serverInfo": {"name": "empty-pages", "version": "1"}, } @@ -2455,3 +2590,309 @@ def test_public_mcp_import_preserves_incompatible_sdk_error() -> None: assert not isinstance(caught.value, ModuleNotFoundError) assert caught.value.__cause__ is None assert "litellm[mcp]" not in str(caught.value) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("grouped", (False, True)) +@pytest.mark.parametrize("raise_on_error", (False, True)) +@pytest.mark.parametrize("termination", ("ok", "failure", "hang")) +async def test_outer_deadline_delivers_session_termination(termination: str, grouped: bool, raise_on_error: bool) -> None: + deleted: Final = asyncio.Event() + started: Final = asyncio.Event() + + async def respond(request: httpx2.Request) -> httpx2.Response: + await anyio.lowlevel.checkpoint() + if request.method == "DELETE": + first_termination: Final = not deleted.is_set() + deleted.set() + if termination == "hang" and first_termination: + await anyio.sleep_forever() + return httpx2.Response(500 if termination == "failure" else 200) + if request.method == "GET": + return httpx2.Response(405) + payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content) + if not isinstance(payload, JSONRPCRequest): + return httpx2.Response(202) + if payload.method == "initialize": + return httpx2.Response( + 200, + headers={"mcp-session-id": "cancel-owned-session"}, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "result": { + "protocolVersion": "2025-11-25", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "cancellation-peer", "version": "1"}, + }, + }, + ) + if payload.method == "tools/list": + return httpx2.Response(200, json={"jsonrpc": "2.0", "id": payload.id, "result": {"tools": []}}) + started.set() + await anyio.sleep_forever() + raise AssertionError("cancelled request resumed") + + client: Final = _MockTransportClient(respond, server_url="https://example.com/mcp", timeout=30) + + async def invoke(): + with anyio.fail_after(0.2): + pending: Final = client.call_tool(CallToolRequestParams(name="slow", arguments={}), raise_on_error=raise_on_error) + if grouped: + await asyncio.gather(pending) + else: + await pending + + before: Final = anyio.current_time() + with pytest.raises(TimeoutError): + await invoke() + assert started.is_set() + assert deleted.is_set(), "Cancellation must deliver DELETE before returning to the caller" + + assert anyio.current_time() - before < 6.5 + assert await client.list_tools(raise_on_error=True) == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("original_error", (False, True)) +async def test_task_cancellation_during_cleanup_preserves_failure(original_error: bool) -> None: + deleting: Final = asyncio.Event() + drained: Final = asyncio.Event() + original: Final = RuntimeError("operation failed before teardown") + + async def respond(request: httpx2.Request) -> httpx2.Response: + if request.method == "DELETE": + deleting.set() + try: + await anyio.sleep_forever() + finally: + drained.set() + if request.method == "GET": + return httpx2.Response(405) + payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content) + if not isinstance(payload, JSONRPCRequest): + return httpx2.Response(202) + return httpx2.Response( + 200, + headers={"mcp-session-id": "cleanup-session"}, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "result": { + "protocolVersion": (payload.params or {})["protocolVersion"], + "capabilities": {}, + "serverInfo": {"name": "cleanup-peer", "version": "1"}, + }, + }, + ) + + async def operation(session: mcp_client_module.ClientSession) -> str: + if original_error: + raise original + return "completed" + + client: Final = _MockTransportClient(respond, server_url="https://example.com/mcp", timeout=30) + task: Final = asyncio.create_task(client.run_with_session(operation)) + await asyncio.wait_for(deleting.wait(), 2) + for _ in range(3): + task.cancel() + await asyncio.sleep(0) + with pytest.raises(RuntimeError if original_error else asyncio.CancelledError) as caught: + await task + assert drained.is_set(), "Caller must wait for termination cleanup to finish" + if original_error: + assert caught.value is original + else: + assert task.cancelled() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("original_error", (False, True)) +@pytest.mark.parametrize("cancel_mode", ("task", "scope")) +async def test_http_close_cancellation_cannot_turn_into_success(original_error: bool, cancel_mode: str) -> None: + closing: Final = asyncio.Event() + drained: Final = asyncio.Event() + original: Final = RuntimeError("failed before HTTP close") + + class ClosingHTTPClient(httpx2.AsyncClient): + async def aclose(self) -> None: + closing.set() + try: + await anyio.sleep_forever() + finally: + drained.set() + + class ClosingMCPClient(MCPClient): + def _create_transport_context(self): + http_client: Final = ClosingHTTPClient(transport=httpx2.MockTransport(lambda _: httpx2.Response(200))) + return streamable_http_client(self.server_url, http_client=http_client), http_client + + async def _execute_session_operation(self, transport_ctx, operation, http_client=None): + if original_error: + raise original + return "completed" + + client: Final = ClosingMCPClient(server_url="https://example.com/mcp") + + async def invoke() -> str: + with anyio.fail_after(0.05 if cancel_mode == "scope" else None): + return await client.run_with_session(AsyncMock()) + + task: Final = asyncio.create_task(invoke()) + await asyncio.wait_for(closing.wait(), 2) + if cancel_mode == "task": + task.cancel() + cancellation_type: Final = asyncio.CancelledError if cancel_mode == "task" else TimeoutError + with pytest.raises(RuntimeError if original_error else cancellation_type) as caught: + await task + assert drained.is_set(), "Caller must wait for HTTP closure to finish" + if original_error: + assert caught.value is original + elif cancel_mode == "task": + assert task.cancelled() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cancel_mode", ("scope", "task", "wait_for", "read_timeout")) +@pytest.mark.parametrize("concurrency", (1, 5)) +@pytest.mark.parametrize("termination", ("ok", "hang", "hang_body")) +@pytest.mark.parametrize("raise_on_error", (False, True)) +async def test_cancellation_delivers_termination_over_tcp( + cancel_mode: str, concurrency: int, termination: str, raise_on_error: bool +) -> None: + started: Final = asyncio.Event() + terminations: Final[list[bytes]] = [] + starts: Final[list[bytes]] = [] + stop: Final = asyncio.Event() + connections: Final[list[asyncio.Task[None]]] = [] + + async def handle_connection(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + connection: Final = asyncio.current_task() + assert connection is not None + connections.append(connection) + try: + request_line: Final = await reader.readline() + if not request_line: + return + method: Final = request_line.split()[0] + headers: Final = await reader.readuntil(b"\r\n\r\n") + length: Final = next( + ( + int(line.split(b":", 1)[1]) + for line in headers.splitlines() + if line.lower().startswith(b"content-length:") + ), + 0, + ) + try: + body: Final = await reader.readexactly(length) + except asyncio.IncompleteReadError: + return + if method == b"DELETE": + terminations.append(body) + if termination != "ok": + await stop.wait() + return + writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + elif method == b"GET": + writer.write(b"HTTP/1.1 405 Method Not Allowed\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + else: + payload: Final = json.loads(body) + if payload["method"] == "tools/call": + if termination == "hang_body": + writer.write( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n" + b"Content-Length: 200\r\nConnection: close\r\n\r\n" + ) + await writer.drain() + starts.append(body) + if len(starts) == concurrency: + started.set() + await stop.wait() + return + if payload["method"] == "initialize": + response: Final = json.dumps( + { + "jsonrpc": "2.0", + "id": payload["id"], + "result": { + "protocolVersion": "2025-06-18", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "tcp-peer", "version": "1"}, + }, + } + ).encode() + writer.write( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nMcp-Session-Id: tcp-session\r\n" + + f"Content-Length: {len(response)}\r\nConnection: close\r\n\r\n".encode() + + response + ) + else: + writer.write(b"HTTP/1.1 202 Accepted\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + await writer.drain() + finally: + writer.close() + await writer.wait_closed() + + listener: Final = await asyncio.start_server(handle_connection, "127.0.0.1", 0) + port: Final = listener.sockets[0].getsockname()[1] + client: Final = MCPClient( + server_url=f"http://127.0.0.1:{port}/mcp", timeout=2 if cancel_mode == "read_timeout" else 0.5 if termination != "ok" else 30 + ) + + async def calls(): + results: Final = await asyncio.gather( + *( + client.call_tool(CallToolRequestParams(name="slow", arguments={}), raise_on_error=raise_on_error) + for _ in range(concurrency) + ), + return_exceptions=cancel_mode == "read_timeout", + ) + if cancel_mode == "read_timeout": + if raise_on_error: + assert all(isinstance(result, TimeoutError) for result in results) + else: + assert all(isinstance(result, CallToolResult) and result.is_error for result in results) + return results + + async def invoke(): + if cancel_mode == "scope": + with anyio.fail_after(0.2): + return await calls() + return await calls() + + try: + task: Final = asyncio.create_task(invoke()) + await asyncio.wait_for(started.wait(), 3) + if cancel_mode == "task": + task.cancel() + expected_error: Final = ( + TimeoutError + if cancel_mode == "read_timeout" + else asyncio.CancelledError + if cancel_mode == "task" + else TimeoutError + ) + if cancel_mode == "read_timeout": + done, _ = await asyncio.wait((task,), timeout=8) + assert task in done, "Read timeout and bounded cleanup must complete without external cancellation" + await task + elif cancel_mode == "wait_for": + with pytest.raises(expected_error): + await asyncio.wait_for(task, 0.2) + else: + with pytest.raises(expected_error): + await task + assert len(starts) == concurrency + assert len(terminations) == concurrency, "Each cancelled call must send DELETE over a fresh TCP connection" + finally: + stop.set() + if not task.done(): + task.cancel() + await asyncio.wait((task,), timeout=8) + listener.close() + for connection in connections: + connection.cancel() + closed: Final = await asyncio.wait_for(asyncio.gather(*connections, return_exceptions=True), 2) + assert all(result is None or isinstance(result, asyncio.CancelledError) for result in closed), closed + await asyncio.wait_for(listener.wait_closed(), 2) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 35e7055bbc0..0012be23428 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -3057,6 +3057,11 @@ class TestMCPDelegateAuthToUpstream: ) cases = [ + ("/mcp/sse", []), + ("/mcp/sse/", []), + ("/mcp/sse/messages", []), + ("/mcp/sse/messages/", []), + ("/sse/mcp", ["sse"]), # Single server, single segment. ("/mcp/foo", ["foo"]), # Server name with one embedded slash (two segments). diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index b715fe67e20..6588ab4c87b 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -1147,109 +1147,39 @@ async def test_mcp_read_resource_success(): assert result is read_result -def test_normalize_resource_contents_passes_metadata(): - """Test that _normalize_resource_contents preserves meta from ResourceContents (MCP 1.26.0+).""" - try: - from litellm.proxy._experimental.mcp_server.server import ( - _normalize_resource_contents, - ) - except ImportError: - pytest.skip("MCP server not available") +@pytest.mark.asyncio +@pytest.mark.parametrize( + "kind,metadata", + (("text", {"version": "1.0", "source": "test"}), ("blob", {"encoding": "base64"}), ("text", {}), ("text", None)), +) +async def test_read_resource_preserves_content_metadata(_mcp_request_ctx, kind, metadata): + from mcp.types import ReadResourceRequestParams + from litellm.proxy._experimental.mcp_server import operations, server - meta = {"version": "1.0", "source": "test"} - contents = [ - TextResourceContents( - uri="https://example.com/resource", - text="hello world", - mimeType="text/plain", - meta=meta, - ) - ] + uri: Final = "https://example.com/resource" + caller: Final = UserAPIKeyAuth(user_id="resource-caller") + upstream_server: Final = MCPServer(server_id="catalog", name="catalog", transport=MCPTransport.http) + content: Final = ( + TextResourceContents(uri=uri, text="hello world", mimeType="text/plain", meta=metadata) + if kind == "text" + else BlobResourceContents(uri=uri, blob="aGVsbG8=", mimeType="image/png", meta=metadata) + ) + with ( + patch.object(server, "get_or_extract_auth_context", AsyncMock(return_value=(caller, None, ["catalog"], None, None, None, None))), + patch.object(operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[upstream_server])), + patch.object(operations.global_mcp_server_manager, "read_resource_from_server", AsyncMock(return_value=ReadResourceResult(contents=[content]))), + ): + result: Final = await server.read_resource(_mcp_request_ctx(), ReadResourceRequestParams(uri=uri)) - result = _normalize_resource_contents(contents) - - assert len(result) == 1 - assert result[0].content == "hello world" - assert result[0].mime_type == "text/plain" - assert result[0].meta == meta - - -def test_normalize_resource_contents_blob_with_metadata(): - """Test that _normalize_resource_contents preserves meta for BlobResourceContents.""" - try: - from litellm.proxy._experimental.mcp_server.server import ( - _normalize_resource_contents, - ) - except ImportError: - pytest.skip("MCP server not available") - - meta = {"encoding": "base64"} - contents = [ - BlobResourceContents( - uri="https://example.com/image.png", - blob="aGVsbG8=", - mimeType="image/png", - meta=meta, - ) - ] - - result = _normalize_resource_contents(contents) - - assert len(result) == 1 - assert result[0].content == "aGVsbG8=" - assert result[0].mime_type == "image/png" - assert result[0].meta == meta - - -def test_normalize_resource_contents_preserves_empty_metadata(): - """Test that empty dict meta is preserved (truthiness bug fix).""" - try: - from litellm.proxy._experimental.mcp_server.server import ( - _normalize_resource_contents, - ) - except ImportError: - pytest.skip("MCP server not available") - - empty_meta: dict = {} - contents = [ - TextResourceContents( - uri="https://example.com/resource", - text="hi", - mimeType="text/plain", - meta=empty_meta, - ) - ] - - result = _normalize_resource_contents(contents) - - assert len(result) == 1 - assert result[0].meta == empty_meta - assert result[0].meta is not None - assert result[0].meta == {} - - -def test_normalize_resource_contents_without_metadata(): - """Test that _normalize_resource_contents works when meta is absent (backward compat).""" - try: - from litellm.proxy._experimental.mcp_server.server import ( - _normalize_resource_contents, - ) - except ImportError: - pytest.skip("MCP server not available") - - contents = [ - TextResourceContents( - uri="https://example.com/resource", - text="hello", - mimeType="text/plain", - ) - ] - - result = _normalize_resource_contents(contents) - - assert len(result) == 1 - assert result[0].content == "hello" - assert result[0].meta is None + assert result.model_dump(mode="json", by_alias=True, exclude_none=True) == { + "cacheScope": "private", "resultType": "complete", "ttlMs": 0, + "contents": [{ + "uri": uri, + "mimeType": "text/plain" if kind == "text" else "image/png", + "text" if kind == "text" else "blob": "hello world" if kind == "text" else "aGVsbG8=", + **({"_meta": metadata} if metadata is not None else {}), + }], + } @pytest.mark.asyncio @@ -1859,14 +1789,12 @@ async def test_concurrent_initialize_session_managers(): original_initialized = mcp_server._SESSION_MANAGERS_INITIALIZED original_session_cm = mcp_server._session_manager_cm original_stateful_cm = mcp_server._session_manager_stateful_cm - original_sse_cm = mcp_server._sse_session_manager_cm original_cleanup_task = mcp_server._stateful_auth_context_cleanup_task try: mcp_server._SESSION_MANAGERS_INITIALIZED = False mcp_server._session_manager_cm = None mcp_server._session_manager_stateful_cm = None - mcp_server._sse_session_manager_cm = None # Create mock context managers for all three session managers mock_cm_stateless = AsyncMock() @@ -1877,10 +1805,6 @@ async def test_concurrent_initialize_session_managers(): mock_cm_stateful.__aenter__ = AsyncMock() mock_cm_stateful.__aexit__ = AsyncMock() - mock_cm_sse = AsyncMock() - mock_cm_sse.__aenter__ = AsyncMock() - mock_cm_sse.__aexit__ = AsyncMock() - with ( patch.object( mcp_server.session_manager_stateless, @@ -1892,11 +1816,6 @@ async def test_concurrent_initialize_session_managers(): "run", return_value=mock_cm_stateful, ) as mock_stateful_run, - patch.object( - mcp_server.sse_session_manager, - "run", - return_value=mock_cm_sse, - ) as mock_sse_run, patch("litellm.proxy._experimental.mcp_server.operations.verbose_logger"), ): # Create multiple concurrent tasks that call initialize_session_managers @@ -1918,10 +1837,6 @@ async def test_concurrent_initialize_session_managers(): assert mock_stateful_run.call_count == 1, ( f"Expected 1 call to session_manager_stateful.run(), got {mock_stateful_run.call_count}" ) - assert mock_sse_run.call_count == 1, ( - f"Expected 1 call to sse_session_manager.run(), got {mock_sse_run.call_count}" - ) - # The context managers should only be entered once each assert mock_cm_stateless.__aenter__.call_count == 1, ( f"Expected 1 call to stateless __aenter__, got {mock_cm_stateless.__aenter__.call_count}" @@ -1929,10 +1844,6 @@ async def test_concurrent_initialize_session_managers(): assert mock_cm_stateful.__aenter__.call_count == 1, ( f"Expected 1 call to stateful __aenter__, got {mock_cm_stateful.__aenter__.call_count}" ) - assert mock_cm_sse.__aenter__.call_count == 1, ( - f"Expected 1 call to sse __aenter__, got {mock_cm_sse.__aenter__.call_count}" - ) - # State should be properly set assert mcp_server._SESSION_MANAGERS_INITIALIZED is True @@ -1948,7 +1859,6 @@ async def test_concurrent_initialize_session_managers(): mcp_server._SESSION_MANAGERS_INITIALIZED = original_initialized mcp_server._session_manager_cm = original_session_cm mcp_server._session_manager_stateful_cm = original_stateful_cm - mcp_server._sse_session_manager_cm = original_sse_cm mcp_server._stateful_auth_context_cleanup_task = original_cleanup_task @@ -2263,7 +2173,7 @@ async def test_sse_endpoint_applies_the_same_client_allowlist( new_callable=AsyncMock, ), patch.object( # test-quality-ok: SSE manager is a module singleton; the downstream call is the observable - mcp_module.sse_session_manager, "handle_request", side_effect=handle_request + mcp_module.sse, "handle_post_message", side_effect=handle_request ), ): if admitted: @@ -6651,17 +6561,22 @@ class TestGatewayCreateInitializationOptions: ) captured = {} - async def record_request(scope, receive, send): + @contextlib.asynccontextmanager + async def connect_sse(scope, receive, send): + yield (None, None) + + async def record_request(read_stream, write_stream, options): captured["server_name"] = server.create_initialization_options().server_name scope = { "type": "http", - "method": "POST", + "method": "GET", "path": "/mcp/grafana", "headers": [], } with ( + patch.object(mcp_server.sse, "connect_sse", connect_sse), patch( "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, @@ -6697,8 +6612,8 @@ class TestGatewayCreateInitializationOptions: True, ), patch.object( - mcp_server.sse_session_manager, - "handle_request", + mcp_server.server, + "run", side_effect=record_request, ), ): @@ -10260,7 +10175,10 @@ async def test_active_request_ctx_var_feeds_auth_resolution_recording(_mcp_reque ("1999-01-01", True), ], ) -async def test_streamable_http_rejects_modern_protocol_version(header_value: str, expected_rejected: bool) -> None: +@pytest.mark.parametrize("handler", ("handle_streamable_http_mcp", "handle_sse_mcp")) +async def test_streamable_http_rejects_modern_protocol_version( + header_value: str, expected_rejected: bool, handler: str +) -> None: from litellm.proxy._experimental.mcp_server import server as mcp_module from litellm.proxy._experimental.mcp_server.server import unsupported_protocol_version @@ -10283,7 +10201,7 @@ async def test_streamable_http_rejects_modern_protocol_version(header_value: str async def send(message: Message) -> None: sent.append(message) - await mcp_module.handle_streamable_http_mcp(scope, receive, send) + await getattr(mcp_module, handler)(scope, receive, send) start = next(m for m in sent if m["type"] == "http.response.start") assert start["status"] == 400 @@ -10333,3 +10251,124 @@ async def test_tool_listing_preserves_permission_denial_when_failure_logging_fai logger.post_call_failure_hook.assert_awaited_once() assert logger.post_call_failure_hook.await_args.kwargs["original_exception"] is denial assert logger.post_call_failure_hook.await_args.kwargs["user_api_key_dict"] == auth + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prefix,suffix", (("", ""), ("/gateway", "/"))) +async def test_legacy_sse_mount_emits_message_endpoint(prefix: str, suffix: str) -> None: + from starlette.applications import Starlette + from starlette.routing import Mount + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing + from litellm.proxy._experimental.mcp_server import server as mcp_server + + app: Final = Starlette(routes=[Mount("/mcp", app=mcp_server.app)]) + incoming: Final[asyncio.Queue[Message]] = asyncio.Queue() + outgoing: Final[asyncio.Queue[Message]] = asyncio.Queue() + await incoming.put({"type": "http.request", "body": b"", "more_body": False}) + path: Final = f"{prefix}/mcp/sse{suffix}" + scope: Final[Scope] = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "GET", + "scheme": "http", + "path": path, + "raw_path": path.encode(), + "query_string": b"", + "root_path": prefix, + "server": ("localhost", 80), + "client": ("127.0.0.1", 1234), + "headers": [(b"accept", b"text/event-stream")], + } + auth: Final = UserAPIKeyAuth(api_key="test-owner") + with ( + patch.object( + mcp_server, "extract_mcp_auth_context", AsyncMock(return_value=(auth, None, None, None, None, None)) + ), + patch.object(mcp_server, "_raise_preemptive_401_for_unauthenticated_servers", AsyncMock()), + patch.object(mcp_server, "_check_passthrough_upstream_auth", AsyncMock()), + patch.object(mcp_server.operations, "_raise_if_initialize_grants_no_mcp_servers", AsyncMock()), + patch.object(mcp_server, "_SESSION_MANAGERS_INITIALIZED", True), + ): + task: Final = asyncio.create_task(app(scope, incoming.get, outgoing.put)) + try: + start: Final = await asyncio.wait_for(outgoing.get(), 2) + assert start["type"] == "http.response.start" + assert start["status"] == 200 + endpoint_frame: Final = await asyncio.wait_for(outgoing.get(), 2) + frame: Final = endpoint_frame["body"].decode() + assert "event: endpoint" in frame + endpoint: Final = frame.split("data: ", 1)[1].splitlines()[0] + assert endpoint.startswith(f"{prefix}/mcp/sse/messages?session_id=") + message_path, query = endpoint.split("?", 1) + + async def post(body: bytes) -> int: + messages: Final[asyncio.Queue[Message]] = asyncio.Queue() + requests: Final[asyncio.Queue[Message]] = asyncio.Queue() + await requests.put({"type": "http.request", "body": body, "more_body": False}) + post_scope: Final[Scope] = { + **scope, + "method": "POST", + "path": message_path + suffix, + "raw_path": (message_path + suffix).encode(), + "query_string": query.encode(), + "root_path": prefix, + "headers": [(b"content-type", b"application/json")], + } + await asyncio.wait_for(app(post_scope, requests.get, messages.put), 2) + return (await messages.get())["status"] + + initialization: Final = json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": LATEST_HANDSHAKE_VERSION, + "capabilities": {}, + "clientInfo": {"name": "legacy-client", "version": "1"}, + }, + } + ).encode() + assert await post(initialization) == 202 + reply: Final = (await asyncio.wait_for(outgoing.get(), 2))["body"].decode() + initialized: Final = json.loads(reply.split("data: ", 1)[1].splitlines()[0]) + assert initialized["id"] == 1 + assert initialized["result"]["serverInfo"]["name"] == "litellm-mcp-server" + + assert await post(b'{"jsonrpc":"2.0","method":"notifications/initialized"}') == 202 + for request_id, marker in ((2, "first-post"), (3, "second-post")): + post_auth: Final = UserAPIKeyAuth(api_key="test-owner", user_id=marker) + listing: Final = AsyncMock(return_value=AggregateToolListing(tools=[], outcomes={})) + with ( + patch.object( + mcp_server, + "extract_mcp_auth_context", + AsyncMock(return_value=(post_auth, None, [marker], {marker: {"Authorization": marker}}, {"Authorization": marker}, {"x-request-marker": marker})), + ), + patch.object(mcp_server.operations, "_get_tools_from_mcp_servers", listing), + ): + assert ( + await post(json.dumps({"jsonrpc": "2.0", "id": request_id, "method": "tools/list"}).encode()) + == 202 + ) + listed_frame: Final = (await asyncio.wait_for(outgoing.get(), 2))["body"].decode() + listed: Final = json.loads(listed_frame.split("data: ", 1)[1].splitlines()[0]) + assert listed["id"] == request_id + assert listed["result"]["tools"] == [] + listing.assert_awaited_once() + assert listing.await_args.kwargs["user_api_key_auth"].user_id == marker + assert listing.await_args.kwargs["mcp_servers"] == [marker] + assert listing.await_args.kwargs["mcp_server_auth_headers"] == {marker: {"Authorization": marker}} + assert listing.await_args.kwargs["oauth2_headers"] == {"Authorization": marker} + assert listing.await_args.kwargs["raw_headers"] == {"x-request-marker": marker} + + stranger: Final = UserAPIKeyAuth(api_key="different-owner") + with patch.object( + mcp_server, "extract_mcp_auth_context", AsyncMock(return_value=(stranger, None, None, None, None, None)) + ): + assert await post(initialization) == 404 + finally: + await incoming.put({"type": "http.disconnect"}) + await asyncio.wait_for(task, 2) + assert await post(initialization) == 404 diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 8da93ba341b..e5179387f82 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -653,6 +653,10 @@ def test_virtual_key_llm_api_routes_denies_spend_logs_v2(): "/mcp-rest/tools/call", "/mcp/tools/list", "/token", + "/mcp/sse", + "/mcp/sse/", + "/mcp/sse/messages", + "/mcp/sse/messages/", ], ) def test_mcp_inference_routes_classified_as_llm_api(route): @@ -4310,3 +4314,26 @@ def test_project_delete_route_stays_proxy_admin_only(): valid_token=valid_token, request_data={}, ) + + +@pytest.mark.parametrize("route", ("/mcp/sse", "/mcp/sse/", "/mcp/sse/messages", "/mcp/sse/messages/")) +@pytest.mark.parametrize("route_group", ("mcp_routes", "llm_api_routes", "openai_routes")) +def test_legacy_sse_respects_virtual_key_route_permissions(route: str, route_group: str) -> None: + token: Final = UserAPIKeyAuth( + user_id="sse-caller", user_role=LitellmUserRoles.INTERNAL_USER, allowed_routes=[route_group] + ) + request: Final = Request({"type": "http", "method": "POST" if "messages" in route else "GET", "path": route}) + if route_group == "openai_routes": + with pytest.raises(HTTPException) as caught: + RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=token, request=request) + assert caught.value.status_code == 403 + return + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=None, + _user_role=LitellmUserRoles.INTERNAL_USER, + route=route, + request=request, + valid_token=token, + request_data={}, + ) + assert RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=token, request=request) From 8332f0848aa18d6a7e049b74c8f32e4519ec922e Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 11:29:38 -0700 Subject: [PATCH 04/48] chore(prices): sync Google Gemini prices: 13 models (#42500) gemini/gemini-2.5-flash: cache_read_input_token_cost_batches gemini/gemini-2.5-flash-lite: cache_read_input_token_cost_batches gemini/gemini-2.5-pro: cache_read_input_token_cost_batches gemini/gemini-3-flash-preview: cache_read_input_token_cost_batches gemini/gemini-3.1-flash-lite: cache_read_input_token_cost_batches gemini/gemini-3.1-pro-preview: cache_read_input_token_cost_batches gemini/gemini-3.1-pro-preview-customtools: cache_read_input_token_cost_batches gemini/gemini-3.5-flash: cache_read_input_token_cost_batches gemini/gemini-3.5-flash-lite: cache_read_input_token_cost_batches gemini/gemini-3.6-flash: cache_read_input_token_cost_batches gemini/gemini-3.7-flash: cache_read_input_token_cost_batches gemini/gemini-3.8-flash: cache_read_input_token_cost_batches gemini/gemini-robotics-er-2-preview: cache_read_input_token_cost_batches Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 13 +++++++++++++ model_prices_and_context_window.json | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8d68e325198..589c4099bd7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -28389,6 +28389,7 @@ }, "gemini/gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_audio_token": 2e-06, "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, @@ -28794,6 +28795,7 @@ "gemini/gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_audio_token": 1e-06, @@ -29205,6 +29207,7 @@ "gemini/gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, "input_cost_per_audio_token": 3e-07, @@ -29550,6 +29553,7 @@ "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, "cache_read_input_token_cost_flex": 1.25e-07, "cache_read_input_token_cost_priority": 2.25e-07, "input_cost_per_token": 1.25e-06, @@ -29745,6 +29749,7 @@ "gemini/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-05-07", @@ -29806,6 +29811,7 @@ }, "gemini/gemini-3.5-flash-lite": { "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 2e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, "input_cost_per_token": 3e-07, @@ -29865,6 +29871,7 @@ "gemini/gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 5e-07, @@ -29926,6 +29933,7 @@ "gemini/gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "gemini", @@ -29987,6 +29995,7 @@ "gemini/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30046,6 +30055,7 @@ "gemini/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30106,6 +30116,7 @@ "gemini/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30248,6 +30259,7 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 2e-07, "cache_read_input_token_cost_flex": 2e-07, "search_context_cost_per_query": { "search_context_size_low": 0.014, @@ -30310,6 +30322,7 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 2e-07, "cache_read_input_token_cost_flex": 2e-07, "search_context_cost_per_query": { "search_context_size_low": 0.014, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8d68e325198..589c4099bd7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -28389,6 +28389,7 @@ }, "gemini/gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_audio_token": 2e-06, "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, @@ -28794,6 +28795,7 @@ "gemini/gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_audio_token": 1e-06, @@ -29205,6 +29207,7 @@ "gemini/gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, "input_cost_per_audio_token": 3e-07, @@ -29550,6 +29553,7 @@ "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, "cache_read_input_token_cost_flex": 1.25e-07, "cache_read_input_token_cost_priority": 2.25e-07, "input_cost_per_token": 1.25e-06, @@ -29745,6 +29749,7 @@ "gemini/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-05-07", @@ -29806,6 +29811,7 @@ }, "gemini/gemini-3.5-flash-lite": { "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 2e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, "input_cost_per_token": 3e-07, @@ -29865,6 +29871,7 @@ "gemini/gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 5e-07, @@ -29926,6 +29933,7 @@ "gemini/gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "gemini", @@ -29987,6 +29995,7 @@ "gemini/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30046,6 +30055,7 @@ "gemini/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30106,6 +30116,7 @@ "gemini/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30248,6 +30259,7 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 2e-07, "cache_read_input_token_cost_flex": 2e-07, "search_context_cost_per_query": { "search_context_size_low": 0.014, @@ -30310,6 +30322,7 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 2e-07, "cache_read_input_token_cost_flex": 2e-07, "search_context_cost_per_query": { "search_context_size_low": 0.014, From 2bab39e374648ec18e751f32161840b823d59d0f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 11:31:15 -0700 Subject: [PATCH 05/48] fix(realtime): surface an upstream handshake refusal as an error event and policy close (#42388) * fix(realtime): surface an upstream handshake refusal as an error event and policy close Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(realtime): tidy the handshake refusal e2e Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(realtime): keep upstream exception text out of the Azure client error Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(realtime): map handshake refusal close codes with a lookup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/realtime_errors.py | 34 +++++++- litellm/llms/azure/realtime/handler.py | 23 ++++- litellm/llms/custom_httpx/llm_http_handler.py | 14 +-- litellm/llms/openai/realtime/handler.py | 5 +- litellm/proxy/proxy_server.py | 5 +- .../realtime/realtime_client.py | 4 +- .../realtime/test_realtime_e2e.py | 42 ++++++++- .../test_realtime_errors.py | 53 ++++++++++++ .../llms/azure/realtime/__init__.py | 1 + .../llms/azure/realtime/test_handler.py | 86 +++++++++++++++++++ .../realtime/test_openai_realtime_handler.py | 49 +++++++++++ 11 files changed, 299 insertions(+), 17 deletions(-) create mode 100644 tests/test_litellm/llms/azure/realtime/__init__.py create mode 100644 tests/test_litellm/llms/azure/realtime/test_handler.py diff --git a/litellm/litellm_core_utils/realtime_errors.py b/litellm/litellm_core_utils/realtime_errors.py index 3c064728a66..07f7148815c 100644 --- a/litellm/litellm_core_utils/realtime_errors.py +++ b/litellm/litellm_core_utils/realtime_errors.py @@ -9,13 +9,20 @@ frame itself fail, which is how a loud failure turns back into a silent one. """ import json -from typing import Final +from types import MappingProxyType +from typing import Final, Protocol from litellm.types.realtime import RealtimeErrorDetail, RealtimeErrorEvent WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123 +class _ClientWebSocket(Protocol): + async def send_text(self, data: str) -> None: ... + + async def close(self, code: int = ..., reason: str | None = ...) -> None: ... + + def realtime_error_event(message: str, error_type: str) -> str: detail: Final[RealtimeErrorDetail] = {"type": error_type, "message": message} event: Final[RealtimeErrorEvent] = {"type": "error", "error": detail} @@ -37,3 +44,28 @@ def client_close_code(upstream_code: int) -> int: if upstream_code in EXTERNAL_CLOSE_CODES or 3000 <= upstream_code < 5000: return upstream_code return int(CloseCode.INTERNAL_ERROR) + + +def upstream_handshake_close_code(status_code: int) -> int: + from websockets.frames import CloseCode + + refusal_codes: Final = MappingProxyType( + { + 401: int(CloseCode.POLICY_VIOLATION), + 403: int(CloseCode.POLICY_VIOLATION), + 429: int(CloseCode.TRY_AGAIN_LATER), + } + ) + return refusal_codes.get(status_code, int(CloseCode.INTERNAL_ERROR)) + + +async def close_after_upstream_handshake_refusal(websocket: _ClientWebSocket, status_code: int) -> None: + message: Final = f"Upstream realtime handshake rejected with HTTP {status_code}" + try: + await websocket.send_text(realtime_error_event(message, error_type="server_error")) + except Exception: # noqa: BLE001 # best-effort notice: a dead client socket must not skip the close below + pass + await websocket.close( + code=upstream_handshake_close_code(status_code), + reason=websocket_close_reason(message, fallback="Upstream handshake rejected"), + ) diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index 146915dd6fd..df74975ad0f 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -8,11 +8,15 @@ from collections.abc import Mapping from types import MappingProxyType from typing import Any, Final, Protocol, cast -from litellm._logging import _redact_string, verbose_proxy_logger +from litellm._logging import verbose_proxy_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.types.realtime import RealtimeQueryParams from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from ....litellm_core_utils.realtime_errors import ( + close_after_upstream_handshake_refusal, + realtime_error_event, +) from ....litellm_core_utils.realtime_streaming import ( RealTimeStreaming, ScopedWebSocket, @@ -49,7 +53,9 @@ def azure_realtime_protocol_for_client( class _ProxyClientWebSocket(Protocol): - """Client-facing websocket handle: this path only closes it after a failed handshake.""" + """Client-facing websocket handle: this path only writes to it after a failed handshake.""" + + async def send_text(self, data: str) -> None: ... async def close(self, code: int = ..., reason: str | None = ...) -> None: ... @@ -181,7 +187,16 @@ class AzureOpenAIRealtime(AzureChatCompletion): ) await realtime_streaming.bidirectional_forward() - except websockets.exceptions.InvalidStatusCode as e: - await websocket.close(code=e.status_code, reason=_redact_string(str(e))) + except websockets.exceptions.InvalidStatus as e: + verbose_proxy_logger.exception("Error in AzureOpenAIRealtime.async_realtime") + await close_after_upstream_handshake_refusal(websocket, e.response.status_code) except Exception: verbose_proxy_logger.exception("Error in AzureOpenAIRealtime.async_realtime") + try: + await websocket.send_text(realtime_error_event("Internal server error", error_type="server_error")) + except Exception: # noqa: BLE001 # best-effort notice: a dead client socket must not skip the close below + pass + try: + await websocket.close(code=1011, reason="Internal server error") + except Exception: # noqa: BLE001 # the lower layer may have closed the socket already; closing twice is not an error + pass diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 333ce523e34..c9e9906e931 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -45,7 +45,11 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( ) from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields -from litellm.litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason +from litellm.litellm_core_utils.realtime_errors import ( + close_after_upstream_handshake_refusal, + realtime_error_event, + websocket_close_reason, +) from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.anthropic_messages.transformation import ( @@ -6384,9 +6388,9 @@ class BaseLLMHTTPHandler: await realtime_streaming.bidirectional_forward() - except websockets.exceptions.InvalidStatusCode as e: + except websockets.exceptions.InvalidStatus as e: verbose_logger.exception("Error connecting to backend: %s", e) - await websocket.close(code=e.status_code, reason=_redact_string(str(e))) + await close_after_upstream_handshake_refusal(websocket, e.response.status_code) except Exception as e: verbose_logger.exception("Error connecting to backend: %s", e) redacted_error: Final = _redact_string(str(e)) @@ -6799,9 +6803,9 @@ class BaseLLMHTTPHandler: ) return await streaming.bidirectional_forward() - except websockets.exceptions.InvalidStatusCode as e: + except websockets.exceptions.InvalidStatus as e: verbose_logger.exception("Error connecting to responses WS backend: %s", e) - await websocket.close(code=e.status_code, reason=_redact_string(str(e))) + await close_after_upstream_handshake_refusal(websocket, e.response.status_code) except Exception as e: verbose_logger.exception("Error in responses WS: %s", e) try: diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index e3ecbac1a53..bdc3a6c7908 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -12,6 +12,7 @@ from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.types.realtime import RealtimeQueryParams from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from ....litellm_core_utils.realtime_errors import close_after_upstream_handshake_refusal from ....litellm_core_utils.realtime_streaming import ( RealtimeEventNormalizer, RealTimeStreaming, @@ -175,8 +176,8 @@ class OpenAIRealtime(OpenAIChatCompletion): ) await realtime_streaming.bidirectional_forward() - except websockets.exceptions.InvalidStatusCode as e: - await websocket.close(code=e.status_code, reason=_redact_string(str(e))) + except websockets.exceptions.InvalidStatus as e: + await close_after_upstream_handshake_refusal(websocket, e.response.status_code) except Exception as e: try: await websocket.close(code=1011, reason=_redact_string(f"Internal server error: {e}")) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index efc428fa4c2..10110dd4f76 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -310,6 +310,7 @@ from litellm.litellm_core_utils.core_helpers import ( from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.realtime_errors import ( + close_after_upstream_handshake_refusal, realtime_error_event, websocket_close_reason, ) @@ -12531,9 +12532,9 @@ async def realtime_websocket_endpoint( user_model=user_model, ) await llm_call - except websockets.exceptions.InvalidStatusCode as e: + except websockets.exceptions.InvalidStatus as e: verbose_proxy_logger.exception("Invalid status code") - await websocket.close(code=e.status_code, reason="Invalid status code") + await close_after_upstream_handshake_refusal(websocket, e.response.status_code) except Exception as e: verbose_proxy_logger.exception("Internal server error") redacted_error: Final = _redact_string(str(e)) diff --git a/tests/e2e/llm_translation/realtime/realtime_client.py b/tests/e2e/llm_translation/realtime/realtime_client.py index 7c4a9cc4af9..7da7ceac4d3 100644 --- a/tests/e2e/llm_translation/realtime/realtime_client.py +++ b/tests/e2e/llm_translation/realtime/realtime_client.py @@ -286,7 +286,7 @@ def function_call_item(events: tuple[ReceivedEvent, ...]) -> OutputItem | None: # ---- session + client -------------------------------------------------- -def _as_text(message: str | bytes) -> str: +def as_text(message: str | bytes) -> str: return message.decode("utf-8") if isinstance(message, bytes) else message @@ -304,7 +304,7 @@ class RealtimeSession: collected: list[ReceivedEvent] = [] while time.monotonic() < deadline: try: - text = _as_text( + text = as_text( self.connection.recv(timeout=deadline - time.monotonic()) ) except TimeoutError: diff --git a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py index f99fa8d86b3..d7870b26497 100644 --- a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py @@ -13,8 +13,9 @@ failure. See REALTIME_COVERAGE_MATRIX.md. """ import pytest +from lifecycle import ResourceManager +from models import LiteLLMParamsBody from pydantic import BaseModel - from realtime_client import ( PROVIDERS, ConversationItemCreate, @@ -27,14 +28,17 @@ from realtime_client import ( RealtimeProvider, ResponseCreate, ResponseDone, + ServerEnvelope, SessionConfig, SessionUpdate, + as_text, function_call_item, parse_last, realtime_model, transcript, user_message, ) +from websockets.exceptions import ConnectionClosedError pytestmark = pytest.mark.e2e @@ -147,3 +151,39 @@ def test_tool_call_round_trip( second = session.collect_until("response.done", timeout=60) assert "72" in transcript(second), "follow-up did not use the tool result" + + +_REFUSED_UPSTREAMS = ( + RealtimeProvider( + "azure-bad-key", + "azure-realtime-refused", + LiteLLMParamsBody( + model="azure/gpt-realtime", + api_key="invalid-e2e-key", + api_version="2025-08-28", + realtime_protocol="GA", + ), + ), +) + + +@pytest.mark.parametrize("provider", _REFUSED_UPSTREAMS, ids=[p.id for p in _REFUSED_UPSTREAMS]) +def test_upstream_handshake_refusal_is_an_error_event_and_policy_close( + client: RealtimeClient, + resources: ResourceManager, + scoped_key: str, + provider: RealtimeProvider, +) -> None: + model_name, model_id = client.provision(provider) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + with client.connect(key=scoped_key, model=model_name) as session: + first = ServerEnvelope.model_validate_json( + as_text(session.connection.recv(timeout=15)) + ) + assert first.type == "error", first + with pytest.raises(ConnectionClosedError) as closed: + session.connection.recv(timeout=15) + + assert closed.value.rcvd is not None + assert closed.value.rcvd.code == 1008, closed.value diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_errors.py b/tests/test_litellm/litellm_core_utils/test_realtime_errors.py index 1d2cf905f4e..999c660c286 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_errors.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_errors.py @@ -1,13 +1,17 @@ import json +from typing import cast import pytest from litellm.litellm_core_utils.realtime_errors import ( WEBSOCKET_CLOSE_REASON_MAX_BYTES, client_close_code, + close_after_upstream_handshake_refusal, realtime_error_event, + upstream_handshake_close_code, websocket_close_reason, ) +from litellm.types.realtime import RealtimeErrorEvent def test_realtime_error_event_shape(): @@ -52,3 +56,52 @@ def test_websocket_close_reason_truncates_multibyte_message_by_bytes(): ) def test_client_close_code_only_forwards_codes_a_server_may_send(upstream_code, expected): assert client_close_code(upstream_code) == expected + + +@pytest.mark.parametrize( + ("status_code", "expected"), + [(401, 1008), (403, 1008), (429, 1013), (500, 1011)], +) +def test_upstream_handshake_close_code_maps_http_status_to_close_code(status_code: int, expected: int): + assert upstream_handshake_close_code(status_code) == expected + + +class _RecordingWebSocket: + def __init__(self) -> None: + self.sent: list[str] = [] + self.closed: tuple[int, str | None] | None = None + + async def send_text(self, data: str) -> None: + self.sent.append(data) + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + self.closed = (code, reason) + + +@pytest.mark.asyncio +async def test_close_after_upstream_handshake_refusal_sends_error_event_then_policy_close(): + websocket = _RecordingWebSocket() + + await close_after_upstream_handshake_refusal(websocket, 401) + + assert len(websocket.sent) == 1 + event = cast(RealtimeErrorEvent, json.loads(websocket.sent[0])) + assert event["type"] == "error" + assert event["error"]["type"] == "server_error" + assert "401" in event["error"]["message"] + assert websocket.closed is not None + assert websocket.closed[0] == 1008 + assert websocket.closed[1] + + +@pytest.mark.asyncio +async def test_close_after_upstream_handshake_refusal_still_closes_when_send_fails(): + class _DeadWebSocket(_RecordingWebSocket): + async def send_text(self, data: str) -> None: + raise RuntimeError("socket gone") + + websocket = _DeadWebSocket() + + await close_after_upstream_handshake_refusal(websocket, 500) + + assert websocket.closed == (1011, "Upstream realtime handshake rejected with HTTP 500") diff --git a/tests/test_litellm/llms/azure/realtime/__init__.py b/tests/test_litellm/llms/azure/realtime/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/tests/test_litellm/llms/azure/realtime/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/test_litellm/llms/azure/realtime/test_handler.py b/tests/test_litellm/llms/azure/realtime/test_handler.py new file mode 100644 index 00000000000..edf1b8b290f --- /dev/null +++ b/tests/test_litellm/llms/azure/realtime/test_handler.py @@ -0,0 +1,86 @@ +import json +from typing import cast +from unittest.mock import MagicMock, patch + +import pytest + + +class _RecordingClientWebSocket: + scope: dict[str, list[tuple[bytes, bytes]]] = {"headers": []} + + def __init__(self) -> None: + self.sent: list[str] = [] + self.closed: list[tuple[int, str | None]] = [] + + async def send_text(self, data: str) -> None: + self.sent.append(data) + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + self.closed.append((code, reason)) + + +@pytest.mark.asyncio +async def test_async_realtime_upstream_handshake_refusal_sends_error_event_then_policy_close(): + import websockets + + from litellm.llms.azure.realtime.handler import AzureOpenAIRealtime + from litellm.types.realtime import RealtimeErrorEvent + + handler = AzureOpenAIRealtime() + model = "gpt-realtime" + + dummy_websocket = _RecordingClientWebSocket() + dummy_logging_obj = MagicMock() + + refused = websockets.exceptions.InvalidStatus( + websockets.http11.Response(401, "Unauthorized", websockets.datastructures.Headers()) + ) + + with patch("websockets.connect", side_effect=refused): + await handler.async_realtime( # pyright: ignore[reportUnknownMemberType] # handler's websocket param is a Protocol here but the mock connect type is incomplete + model=model, + websocket=dummy_websocket, + logging_obj=dummy_logging_obj, + api_base="https://example.openai.azure.com", + api_key="bad-key", + api_version="2025-08-28", + query_params={"model": model}, + ) + + assert len(dummy_websocket.sent) == 1 + event = cast(RealtimeErrorEvent, json.loads(dummy_websocket.sent[0])) + assert event["type"] == "error" + assert event["error"]["type"] == "server_error" + assert "401" in event["error"]["message"] + assert dummy_websocket.closed and dummy_websocket.closed[0][0] == 1008 + + +@pytest.mark.asyncio +async def test_async_realtime_unexpected_error_sends_error_event_then_internal_close(): + from litellm.llms.azure.realtime.handler import AzureOpenAIRealtime + from litellm.types.realtime import RealtimeErrorEvent + + handler = AzureOpenAIRealtime() + model = "gpt-realtime" + + dummy_websocket = _RecordingClientWebSocket() + dummy_logging_obj = MagicMock() + + with patch("websockets.connect", side_effect=OSError("connection reset")): + await handler.async_realtime( # pyright: ignore[reportUnknownMemberType] # same as above + model=model, + websocket=dummy_websocket, + logging_obj=dummy_logging_obj, + api_base="https://example.openai.azure.com", + api_key="bad-key", + api_version="2025-08-28", + query_params={"model": model}, + ) + + assert len(dummy_websocket.sent) == 1 + event = cast(RealtimeErrorEvent, json.loads(dummy_websocket.sent[0])) + assert event["type"] == "error" + assert event["error"]["type"] == "server_error" + assert event["error"]["message"] == "Internal server error" + assert "connection reset" not in dummy_websocket.sent[0] + assert dummy_websocket.closed and dummy_websocket.closed[0] == (1011, "Internal server error") diff --git a/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py b/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py index 4221954d787..7cd2b9e259c 100644 --- a/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py +++ b/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py @@ -416,3 +416,52 @@ async def test_async_realtime_ws_url_has_no_ssl(): # Verify ssl is None for ws:// URLs (the fix for issue #19222) assert called_kwargs["ssl"] is None + + +@pytest.mark.asyncio +async def test_async_realtime_upstream_handshake_refusal_sends_error_event_then_policy_close(): + from typing import cast + + import websockets + + from litellm.llms.openai.realtime.handler import OpenAIRealtime + from litellm.types.realtime import RealtimeErrorEvent + + handler = OpenAIRealtime() + model = "gpt-realtime" + + sent: list[str] = [] + closed: list[tuple[int, str | None]] = [] + + class RecordingClientWebSocket: + scope: dict[str, list[tuple[bytes, bytes]]] = {"headers": []} + + async def send_text(self, data: str) -> None: + sent.append(data) + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + closed.append((code, reason)) + + dummy_websocket = RecordingClientWebSocket() + dummy_logging_obj = MagicMock() + + refused = websockets.exceptions.InvalidStatus( + websockets.http11.Response(401, "Unauthorized", websockets.datastructures.Headers()) + ) + + with patch("websockets.connect", side_effect=refused): + await handler.async_realtime( # pyright: ignore[reportUnknownMemberType] # handler's websocket param is Any + model=model, + websocket=dummy_websocket, + logging_obj=dummy_logging_obj, + api_base="https://api.openai.com/", + api_key="bad-key", + query_params={"model": model}, + ) + + assert len(sent) == 1 + event = cast(RealtimeErrorEvent, json.loads(sent[0])) + assert event["type"] == "error" + assert event["error"]["type"] == "server_error" + assert "401" in event["error"]["message"] + assert closed and closed[0][0] == 1008 From e0c2cbff21fb27f23e6d7dfc3534c1f30d3cc189 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 11:34:01 -0700 Subject: [PATCH 06/48] feat(openai): add GPT-6 Sol and GPT-6 Luna (#42515) Add gpt-6-sol and gpt-6-luna to the model cost map with pricing from the OpenAI pricing page and reasoning effort levels none through max. Extend the long-context priority pricing and reasoning effort capability tests to cover both models Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 152 ++++++++++++++++++ model_prices_and_context_window.json | 152 ++++++++++++++++++ .../test_reasoning_effort_capability.py | 19 +++ ...penai_service_tier_long_context_pricing.py | 14 ++ 4 files changed, 337 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 589c4099bd7..8f9958e0031 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -34269,6 +34269,158 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-6-sol": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, + "cache_creation_input_token_cost_flex": 1.25e-06, + "cache_creation_input_token_cost_priority": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "cache_read_input_token_cost_batches": 1e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2e-07, + "cache_creation_input_token_cost_batches": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, + "cache_read_input_token_cost_flex": 1e-07, + "cache_read_input_token_cost_priority": 4e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "input_cost_per_token_above_272k_tokens_priority": 8e-06, + "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_above_272k_tokens_batches": 2e-06, + "input_cost_per_token_flex": 1e-06, + "input_cost_per_token_priority": 4e-06, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_272k_tokens": 1.5e-05, + "output_cost_per_token_above_272k_tokens_flex": 7.5e-06, + "output_cost_per_token_above_272k_tokens_priority": 3e-05, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_above_272k_tokens_batches": 7.5e-06, + "output_cost_per_token_flex": 5e-06, + "output_cost_per_token_priority": 2e-05, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, + "gpt-6-luna": { + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07, + "cache_creation_input_token_cost_flex": 6.25e-08, + "cache_creation_input_token_cost_priority": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 4e-08, + "cache_read_input_token_cost_batches": 5e-09, + "cache_read_input_token_cost_above_272k_tokens_batches": 1e-08, + "cache_creation_input_token_cost_batches": 6.25e-08, + "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-07, + "cache_read_input_token_cost_flex": 5e-09, + "cache_read_input_token_cost_priority": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, + "input_cost_per_token_above_272k_tokens_flex": 1e-07, + "input_cost_per_token_above_272k_tokens_priority": 4e-07, + "input_cost_per_token_batches": 5e-08, + "input_cost_per_token_above_272k_tokens_batches": 1e-07, + "input_cost_per_token_flex": 5e-08, + "input_cost_per_token_priority": 2e-07, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_above_272k_tokens": 7.5e-07, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-07, + "output_cost_per_token_above_272k_tokens_priority": 1.5e-06, + "output_cost_per_token_batches": 2.5e-07, + "output_cost_per_token_above_272k_tokens_batches": 3.75e-07, + "output_cost_per_token_flex": 2.5e-07, + "output_cost_per_token_priority": 1e-06, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 589c4099bd7..8f9958e0031 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -34269,6 +34269,158 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-6-sol": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, + "cache_creation_input_token_cost_flex": 1.25e-06, + "cache_creation_input_token_cost_priority": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "cache_read_input_token_cost_batches": 1e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2e-07, + "cache_creation_input_token_cost_batches": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, + "cache_read_input_token_cost_flex": 1e-07, + "cache_read_input_token_cost_priority": 4e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "input_cost_per_token_above_272k_tokens_priority": 8e-06, + "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_above_272k_tokens_batches": 2e-06, + "input_cost_per_token_flex": 1e-06, + "input_cost_per_token_priority": 4e-06, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_272k_tokens": 1.5e-05, + "output_cost_per_token_above_272k_tokens_flex": 7.5e-06, + "output_cost_per_token_above_272k_tokens_priority": 3e-05, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_above_272k_tokens_batches": 7.5e-06, + "output_cost_per_token_flex": 5e-06, + "output_cost_per_token_priority": 2e-05, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, + "gpt-6-luna": { + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07, + "cache_creation_input_token_cost_flex": 6.25e-08, + "cache_creation_input_token_cost_priority": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 4e-08, + "cache_read_input_token_cost_batches": 5e-09, + "cache_read_input_token_cost_above_272k_tokens_batches": 1e-08, + "cache_creation_input_token_cost_batches": 6.25e-08, + "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-07, + "cache_read_input_token_cost_flex": 5e-09, + "cache_read_input_token_cost_priority": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, + "input_cost_per_token_above_272k_tokens_flex": 1e-07, + "input_cost_per_token_above_272k_tokens_priority": 4e-07, + "input_cost_per_token_batches": 5e-08, + "input_cost_per_token_above_272k_tokens_batches": 1e-07, + "input_cost_per_token_flex": 5e-08, + "input_cost_per_token_priority": 2e-07, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_above_272k_tokens": 7.5e-07, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-07, + "output_cost_per_token_above_272k_tokens_priority": 1.5e-06, + "output_cost_per_token_batches": 2.5e-07, + "output_cost_per_token_above_272k_tokens_batches": 3.75e-07, + "output_cost_per_token_flex": 2.5e-07, + "output_cost_per_token_priority": 1e-06, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, diff --git a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py index adee44aa8a3..4617839c5e3 100644 --- a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py +++ b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py @@ -418,6 +418,25 @@ class TestGpt6AstraAdvertisesItsDocumentedLevels: ) +class TestGpt6SolAndLunaAdvertiseNoneThroughMax: + @pytest.mark.parametrize("model", ["gpt-6-sol", "gpt-6-luna"]) + def test_the_entry_advertises_none_through_max(self, local_model_cost_map, model): + """OpenAI documents none, low, medium (default), high, xhigh and max for both. Unlike + gpt-6-astra they take none.""" + from litellm.utils import _get_model_info_helper + + model_info = dict(_get_model_info_helper(model=model, custom_llm_provider="openai")) + + assert resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) == ( + "none", + "low", + "medium", + "high", + "xhigh", + "max", + ) + + class TestNearestDeclaredReasoningEffort: def test_a_declared_level_is_kept(self): assert nearest_declared_reasoning_effort("high", ("none", "high")) == "high" diff --git a/tests/test_litellm/test_openai_service_tier_long_context_pricing.py b/tests/test_litellm/test_openai_service_tier_long_context_pricing.py index c766370230c..9b3a1e57169 100644 --- a/tests/test_litellm/test_openai_service_tier_long_context_pricing.py +++ b/tests/test_litellm/test_openai_service_tier_long_context_pricing.py @@ -58,6 +58,18 @@ PRIORITY_LONG_CONTEXT = { "cache_read_input_token_cost_above_272k_tokens_priority": 4e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05, }, + "gpt-6-sol": { + "input_cost_per_token_above_272k_tokens_priority": 8e-06, + "output_cost_per_token_above_272k_tokens_priority": 3e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, + }, + "gpt-6-luna": { + "input_cost_per_token_above_272k_tokens_priority": 4e-07, + "output_cost_per_token_above_272k_tokens_priority": 1.5e-06, + "cache_read_input_token_cost_above_272k_tokens_priority": 4e-08, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07, + }, } EXPECTED = {**FLEX_LONG_CONTEXT, **PRIORITY_LONG_CONTEXT} @@ -90,4 +102,6 @@ TIERED_COST_CASES = [ ("gpt-5.6-terra", "priority", 8e-06, 3.6e-05), ("gpt-5.6-luna", "priority", 8e-07, 3.6e-06), ("gpt-6-astra", "priority", 4e-05, 0.00015), + ("gpt-6-sol", "priority", 8e-06, 3e-05), + ("gpt-6-luna", "priority", 4e-07, 1.5e-06), ] From a788c4ab2b3c797c2d73b5e27d9ea21bb8200798 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 18:37:42 +0000 Subject: [PATCH 07/48] fix(router): explain fallback outcome in plain words in the raised error (#42509) * fix(router): explain fallback outcome in plain words in the raised error Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): keep the fallback outcome trailer on the outermost hop only Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): explain failed context-window and content-policy fallbacks too Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): name both Router and proxy fallback config in the no-fallback hint Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: ryan Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 29 +++++++--- litellm/router_utils/common_utils.py | 33 +++++++++++- .../test_router_exception_redaction.py | 54 +++++++++++++++---- 3 files changed, 97 insertions(+), 19 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 80e8eb0c090..54b4d04a0cb 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -176,6 +176,8 @@ from litellm.router_utils.common_utils import ( _is_proxy_admin_request, filter_team_based_models, filter_web_search_deployments, + format_fallback_outcome_message, + format_no_fallback_group_message, get_request_team_id, provider_for_generic_call, resolve_model_group_alias, @@ -7162,6 +7164,9 @@ class Router: # behind the router name, and fallbacks are configured per tier, not per router. lookup_groups: Final[tuple[str, ...]] = fallback_lookup_groups(kwargs, model_group) fallback_failure_exception_str = "" + no_fallback_group_explained = False + hop_depth: Final = kwargs.get("fallback_depth") + nested_fallback_hop: Final = isinstance(hop_depth, int) and hop_depth > 0 if disable_fallbacks is True or original_model_group is None: raise e @@ -7353,8 +7358,13 @@ class Router: " -> ".join(lookup_groups), masked_fallbacks, ) - if hasattr(original_exception, "message") and litellm.expose_router_debug_in_errors: - original_exception.message += f"No fallback model group found for lookup_groups={' -> '.join(lookup_groups)}. Fallbacks={masked_fallbacks}" + if ( + hasattr(original_exception, "message") + and litellm.expose_router_debug_in_errors + and not nested_fallback_hop + ): + original_exception.message += format_no_fallback_group_message(lookup_groups, fallbacks) + no_fallback_group_explained = True raise original_exception input_kwargs.update( @@ -7385,11 +7395,16 @@ class Router: cooldown_info, ) - if hasattr(original_exception, "message") and litellm.expose_router_debug_in_errors: - # add the available fallbacks to the exception - original_exception.message += f". Received Model Group={model_group}\nAvailable Model Group Fallbacks={mask_sensitive_structure(fallback_model_group)}" - if len(fallback_failure_exception_str) > 0: - original_exception.message += f"\nError doing the fallback: {fallback_failure_exception_str}" + attempted_fallback_group: Final = input_kwargs.get("fallback_model_group") + if ( + hasattr(original_exception, "message") + and litellm.expose_router_debug_in_errors + and not no_fallback_group_explained + and not nested_fallback_hop + ): + original_exception.message += format_fallback_outcome_message( + model_group, attempted_fallback_group, fallback_failure_exception_str + ) raise original_exception diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 49cca8ee99e..7d111a80264 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -1,6 +1,6 @@ import hashlib import json -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final @@ -12,6 +12,7 @@ from litellm._logging import verbose_logger, verbose_router_logger from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure from litellm.types.router import CredentialLiteLLMParams from litellm.types.utils import LlmProviders @@ -76,6 +77,36 @@ def truncate_fallback_error_detail(detail: str) -> str: return f"{detail[:ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS]}... [truncated {dropped} characters]" +def format_no_fallback_group_message(lookup_groups: Sequence[str], fallbacks: Sequence[Mapping[str, object]]) -> str: + """User-facing explanation appended when a request fails and no fallback chain matches its model group.""" + requested: Final = " -> ".join(lookup_groups) + configured: Final = tuple(dict.fromkeys(key for entry in fallbacks for key in entry)) + configured_text: Final = ( + f" Fallbacks are configured for: {', '.join(configured)}." if configured else " No fallbacks are configured." + ) + return ( + f"\n\nLiteLLM: model group '{requested}' failed with the error above and no fallback model group was found " + f"for it, so the request was not retried on another model.{configured_text}" + " Add a fallbacks entry for that model group (Router fallbacks or proxy router_settings.fallbacks)" + " to retry on another model." + ) + + +def format_fallback_outcome_message( + model_group: str | None, + fallback_model_group: Sequence[object] | None, + fallback_failure_detail: str, +) -> str: + """User-facing explanation appended when the fallback orchestrator gives up and re-raises the primary error.""" + lead: Final = f"\n\nLiteLLM: model group '{model_group}' failed with the error above." + if not fallback_model_group: + return f"{lead} No fallback was attempted." + targets: Final = ", ".join(str(mask_sensitive_structure(target)) for target in fallback_model_group) + if not fallback_failure_detail: + return f"{lead} Fallback model group(s) configured: {targets}." + return f"{lead} Fallback to {targets} also failed: {fallback_failure_detail}" + + def get_litellm_params_sensitive_credential_hash(litellm_params: dict) -> str: """ Hash of the credential params, used for mapping the file id to the right model diff --git a/tests/test_litellm/test_router_exception_redaction.py b/tests/test_litellm/test_router_exception_redaction.py index 6754775db22..5b52496fc1c 100644 --- a/tests/test_litellm/test_router_exception_redaction.py +++ b/tests/test_litellm/test_router_exception_redaction.py @@ -26,8 +26,8 @@ Five leak sites are gated in `litellm/router.py`: 1. Deployment timeout debug after `litellm.Timeout` 2. ContextWindowExceededError fallback hint 3. ContentPolicyViolationError fallback hint -4. "No fallback model group found for..." when fallbacks dict misses -5. "Received Model Group=...\\nAvailable Model Group Fallbacks=..." +4. "no fallback model group was found" when fallbacks dict misses +5. "model group '...' failed with the error above" plus the fallback outcome (always fires on terminal raise from the fallback orchestrator) Site 5 is the broadest — it fires for every failing call that goes @@ -42,8 +42,9 @@ import pytest import litellm from litellm import Router -_RECEIVED_MODEL_GROUP_PHRASE = "Received Model Group=" -_AVAILABLE_FALLBACKS_PHRASE = "Available Model Group Fallbacks=" +_RECEIVED_MODEL_GROUP_PHRASE = "failed with the error above" +_AVAILABLE_FALLBACKS_PHRASE = "No fallback was attempted" +_NO_FALLBACK_GROUP_PHRASE = "no fallback model group was found" _CONTEXT_WINDOW_HINT_PHRASE = "context_window_fallbacks=" _INTERNAL_MODEL_GROUP_NAME = "all-anthropic/claude-secret-internal" _FALLBACK_CREDENTIAL = "sk-INLINEFALLBACKSECRET1234567890" @@ -124,7 +125,7 @@ def test_flag_defaults_on(): assert litellm.expose_router_debug_in_errors is True -# --- Site 5: "Received Model Group=..." on terminal raise -------------------- +# --- Site 5: fallback outcome on terminal raise -------------------- @pytest.mark.asyncio @@ -192,7 +193,7 @@ async def test_flag_on_shows_context_window_fallback_hint(monkeypatch: pytest.Mo assert _INTERNAL_MODEL_GROUP_NAME in msg, msg -# --- Site 4: "No fallback model group found..." when fallbacks miss --------- +# --- Site 4: "no fallback model group was found" when fallbacks miss --------- @pytest.mark.asyncio @@ -221,7 +222,7 @@ async def test_flag_off_does_not_leak_when_no_fallback_group_found(monkeypatch: messages=[{"role": "user", "content": "hi"}], ) msg = excinfo.value.message - assert "No fallback model group found" not in msg, msg + assert _NO_FALLBACK_GROUP_PHRASE not in msg, msg assert "some-other-group" not in msg, msg assert _INTERNAL_MODEL_GROUP_NAME not in msg, msg @@ -250,8 +251,12 @@ async def test_flag_on_shows_when_no_fallback_group_found(monkeypatch: pytest.Mo messages=[{"role": "user", "content": "hi"}], ) msg = excinfo.value.message - assert "No fallback model group found" in msg, msg - assert _INTERNAL_MODEL_GROUP_NAME in msg, msg + assert _NO_FALLBACK_GROUP_PHRASE in msg, msg + assert f"model group '{_INTERNAL_MODEL_GROUP_NAME}' failed with the error above" in msg, msg + assert "Fallbacks are configured for: some-other-group" in msg, msg + assert "not retried on another model" in msg, msg + assert _AVAILABLE_FALLBACKS_PHRASE not in msg, msg + assert msg.count("failed with the error above") == 1, msg # --- Site 1: Deployment timeout debug on litellm.Timeout -------------------- @@ -349,6 +354,30 @@ async def test_flag_on_shows_content_policy_fallback_hint(monkeypatch: pytest.Mo assert _INTERNAL_MODEL_GROUP_NAME in msg, msg +@pytest.mark.asyncio +async def test_flag_on_explains_failed_content_policy_fallback(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", True) + router = Router( + model_list=[ + {"model_name": _INTERNAL_MODEL_GROUP_NAME, "litellm_params": {"model": "gpt-4o", "api_key": "key"}}, + {"model_name": "policy-safe-group", "litellm_params": {"model": "gpt-4o", "api_key": "key"}}, + ], + content_policy_fallbacks=[{_INTERNAL_MODEL_GROUP_NAME: ["policy-safe-group"]}], + num_retries=0, + ) + with pytest.raises(litellm.ContentPolicyViolationError) as excinfo: + await router.acompletion( + model=_INTERNAL_MODEL_GROUP_NAME, + messages=[{"role": "user", "content": "hi"}], + mock_response=_content_policy_error(), + ) + msg = excinfo.value.message + assert f"model group '{_INTERNAL_MODEL_GROUP_NAME}' failed with the error above" in msg, msg + assert "Fallback to policy-safe-group also failed: " in msg, msg + assert _AVAILABLE_FALLBACKS_PHRASE not in msg, msg + assert msg.count("failed with the error above") == 1, msg + + # --- Credential masking: raw provider keys never leak, either flag state ---- @@ -387,7 +416,7 @@ async def test_flag_on_masks_fallback_credentials(monkeypatch: pytest.MonkeyPatc async def test_flag_on_scrubs_credential_from_inner_fallback_exception_string(monkeypatch: pytest.MonkeyPatch): """If the fallback attempt itself raises an exception whose message embeds a raw provider credential (e.g. a provider SDK echoing back the api_key it was - called with), that string is re-embedded via `Error doing the fallback: ...` + called with), that string is re-embedded via `Fallback to ... also failed: ...` on the terminal raise. The router must scrub known secret patterns from it. The primary fails with a benign rate-limit; the fallback deployment fails with an exception whose text contains the secret.""" @@ -423,6 +452,9 @@ async def test_flag_on_scrubs_credential_from_inner_fallback_exception_string(mo messages=[{"role": "user", "content": "hi"}], ) msg = excinfo.value.message - assert "Error doing the fallback:" in msg, msg + assert f"model group '{_INTERNAL_MODEL_GROUP_NAME}' failed with the error above" in msg, msg + assert "Fallback to fallback-group also failed: " in msg, msg + assert "content_filter_policy" in msg, msg + assert msg.count("failed with the error above") == 1, msg assert inner_secret not in msg, msg assert "REDACTED" in msg, msg From 4bcdaf3d4bcb54d47052f2e72c0c687c927c429e Mon Sep 17 00:00:00 2001 From: Meet Patel <93856438+patel-26meet@users.noreply.github.com> Date: Wed, 23 Sep 2026 00:10:39 +0530 Subject: [PATCH 08/48] fix(cost): honor deployment pricing for image generation (#39311) * fix image cost: honor deployment pricing * fix types: coerce fal deployment price, drop private import * fix: forward every custom pricing field through get_litellm_params * test: assert optional keys are absent, not merely None, in get_litellm_params * test: type the deployment image pricing test parameters * fix: bill deployment per-image and per-pixel prices on unlisted image models * test: type the remaining image cost test parameters --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/cost_calculator.py | 55 +++---- .../litellm_core_utils/get_litellm_params.py | 2 + .../litellm_core_utils/llm_cost_calc/utils.py | 81 +++++++++++ .../aiml/image_generation/cost_calculator.py | 7 +- .../image_generation/cost_calculator.py | 27 +++- .../image_generation/cost_calculator.py | 7 +- litellm/llms/fal_ai/cost_calculator.py | 19 ++- .../llms/gemini/image_edit/cost_calculator.py | 3 + .../image_generation/cost_calculator.py | 9 +- .../image_generation/cost_calculator.py | 28 ++-- litellm/llms/recraft/cost_calculator.py | 7 +- litellm/llms/runwayml/cost_calculator.py | 7 +- .../image_generation/cost_calculator.py | 9 +- .../llm_cost_calc/test_llm_cost_calc_utils.py | 135 +++++++++++++++++- .../test_get_litellm_params.py | 27 +++- .../test_azure_ai_flux2_image_generation.py | 30 ++++ tests/test_litellm/test_cost_calculator.py | 92 +++++++++++- .../test_gpt_image_cost_calculator.py | 34 +++++ 18 files changed, 518 insertions(+), 61 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index ae54b91c5a0..d66d80a564b 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1532,6 +1532,7 @@ def completion_cost( size=size, optional_params=optional_params, call_type=call_type, + model_info=_deployment_model_info(litellm_logging_obj, custom_pricing, router_model_id), ) elif call_type in _VIDEO_CALL_TYPES: ### VIDEO GENERATION COST CALCULATION ### @@ -2011,13 +2012,9 @@ def _deployment_model_info( ) -> ModelInfo | None: if not custom_pricing: return None - registered_deployment_info: Final = ( - _cost_map_model_info(router_model_id, None) - if router_model_id is not None and router_model_id in litellm.model_cost - else None - ) + registered_deployment_info: Final = _raw_cost_map_entry(router_model_id) if router_model_id is not None else None if registered_deployment_info is not None: - return registered_deployment_info + return cast(ModelInfo, registered_deployment_info) # cast-ok: router registers deployment prices under its id if litellm_logging_obj is None: return None litellm_params: Final = getattr(litellm_logging_obj, "litellm_params", None) @@ -2085,8 +2082,7 @@ def pricing_entry_for_cost_calc( deployment_entry: Final = _deployment_model_info(litellm_logging_obj, custom_pricing, router_model_id) deployment_key: Final = router_model_id or model if deployment_entry is not None and deployment_key is not None: - registered_entry: Final = _raw_cost_map_entry(router_model_id) if router_model_id is not None else None - return deployment_key, registered_entry or deployment_entry + return deployment_key, deployment_entry selected_model: Final = _select_model_name_for_cost_calc( model=model, completion_response=completion_response, @@ -2346,6 +2342,7 @@ def default_image_cost_calculator( n: int | None = 1, # Default to 1 image size: str | None = "1024-x-1024", # OpenAI default optional_params: dict | None = None, + model_info: ModelInfo | None = None, ) -> float: """ Default image cost calculator for image generation @@ -2356,6 +2353,7 @@ def default_image_cost_calculator( quality (Optional[str]): Image quality setting n (Optional[int]): Number of images generated size (Optional[str]): Image size (e.g. "1024x1024" or "1024-x-1024") + model_info (Optional[ModelInfo]): The deployment's own prices, consulted before the cost map Returns: float: Cost in USD for the image generation @@ -2386,9 +2384,7 @@ def default_image_cost_calculator( model_without_provider: Final = f"{size_str}/{model.split('/')[-1]}" model_with_quality_without_provider = f"{quality}/{model_without_provider}" if quality else model_without_provider - # Try model with quality first, fall back to base model name - cost_info: dict | None = None - models_to_check: Final[list[str | None]] = [ + models_to_check: Final = ( model_name_with_quality, base_model_name, model_name_with_v2_quality, @@ -2396,22 +2392,33 @@ def default_image_cost_calculator( model_without_provider, model, model_name_without_custom_llm_provider, - ] - for _model in models_to_check: - if _model is not None and _model in litellm.model_cost: - cost_info = litellm.model_cost[_model] - break - if cost_info is None: + ) + matched_model: Final = next( + (_model for _model in models_to_check if _model is not None and _model in litellm.model_cost), None + ) + if matched_model is None and model_info is None: raise Exception(f"Model not found in cost map. Tried checking {models_to_check}") - # Priority 1: Use per-image pricing if available (for gpt-image-1 and similar models) - if "input_cost_per_image" in cost_info and cost_info["input_cost_per_image"] is not None: - return cost_info["input_cost_per_image"] * n - # Priority 2: Fall back to per-pixel pricing for backward compatibility - elif "input_cost_per_pixel" in cost_info and cost_info["input_cost_per_pixel"] is not None: - return cost_info["input_cost_per_pixel"] * height * width * n - else: + shared_cost_info: Final = litellm.model_cost[matched_model] if matched_model is not None else None + price_tables: Final = tuple(table for table in (model_info, shared_cost_info) if table is not None) + image_count: Final = n if n is not None else 1 + unit_counts: Final = ( + ("input_cost_per_image", image_count), + ("output_cost_per_image", image_count), + ("input_cost_per_pixel", height * width * image_count), + ) + cost: Final = next( + ( + price * units + for price_table in price_tables + for cost_key, units in unit_counts + if (price := price_table.get(cost_key)) is not None + ), + None, + ) + if cost is None: raise Exception(f"No pricing information found for model {model}. Tried checking {models_to_check}") + return cost def default_video_cost_calculator( diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 36fd7fa4e61..7f373569b21 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -4,6 +4,7 @@ from typing import Final from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.llms.openai.data_residency import infer_openai_data_residency +from litellm.types.router import CustomPricingLiteLLMParams AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset( { @@ -65,6 +66,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( } ) | AWS_CREDENTIAL_KWARGS_KEYS + | frozenset(CustomPricingLiteLLMParams.model_fields) ) # Backward-compatible alias for existing imports/tests. diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 3845f50af79..46bf2ec2960 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -24,6 +24,7 @@ from litellm.types.utils import ( CallTypes, CompletionTokensDetailsWrapper, CostPerToken, + CustomPricingLiteLLMParams, DataResidency, ImageResponse, ModelInfo, @@ -49,6 +50,15 @@ _IMAGE_RESPONSE_CALL_TYPES: Final = frozenset( # Pre-resolved DataResidency enum values for fast membership checks _VALID_DATA_RESIDENCIES: Final = frozenset(r.value for r in DataResidency) +_DEPLOYMENT_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields) + +_IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = ( + "input_cost_per_token", + "output_cost_per_token", + "input_cost_per_image_token", + "output_cost_per_image_token", +) + # Pre-resolved service-tier cost-key suffixes (e.g. "_priority"). Used per # request in the cost-calc path, so the f-strings are built once here instead # of being rebuilt for every model_info key on every call. Longest-first so a @@ -826,6 +836,53 @@ def _get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: floa return default_value +def deployment_pricing(model_info: ModelInfo | None) -> ModelInfo | None: + """The prices a deployment sets itself, as floats; None when it sets none that parse.""" + if model_info is None: + return None + priced_keys: Final = tuple(key for key in _DEPLOYMENT_PRICING_KEYS if model_info.get(key) is not None) + pricing: Final = MappingProxyType( + { + key: price + for key in priced_keys + if (price := _get_cost_per_unit(model_info, key, default_value=None)) is not None + } + ) + if not pricing: + return None + return cast(ModelInfo, pricing) # cast-ok: a read-only subset of ModelInfo pricing keys, values validated above + + +def prices_tokens(model_info: ModelInfo) -> bool: + """Whether the price table carries any token rate, so a token-priced calculator can bill from usage.""" + return any(model_info.get(key) is not None for key in _IMAGE_TOKEN_RATE_KEYS) + + +def flat_image_cost(model_info: ModelInfo | None, image_response: ImageResponse) -> float: + """The per-image price times the images returned; 0.0 when the table sets no per-image price.""" + if model_info is None: + return 0.0 + output_cost_per_image: Final = _get_cost_per_unit(model_info, "output_cost_per_image", default_value=None) or 0.0 + num_images: Final = len(image_response.data) if image_response.data else 0 + return output_cost_per_image * num_images + + +def resolve_image_model_info(model: str, custom_llm_provider: str, model_info: ModelInfo | None) -> ModelInfo: + """The price table an image cost calculator consults for ``model``. + + ``shared_backend_model_info`` keeps deployment prices off the shared ``{provider}/{model}`` key, so + a name lookup alone reads the public rate, and a model only the deployment prices has no entry at all. + """ + if model_info is None: + return get_model_info(model=model, custom_llm_provider=custom_llm_provider) + try: + shared_model_info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model + return model_info + resolved: Final[ModelInfo] = {**shared_model_info, **model_info} + return resolved + + def calculate_cache_writing_cost( cache_creation_tokens: int, cache_creation_token_details: CacheCreationTokenDetails | None, @@ -1711,6 +1768,7 @@ def calculate_image_response_cost_from_usage( model: str, image_response: ImageResponse, custom_llm_provider: str, + model_info: ModelInfo | None = None, ) -> float | None: """ Calculate image generation cost from usage metadata when available. @@ -1735,6 +1793,9 @@ def calculate_image_response_cost_from_usage( if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0: return None + if model_info is not None and not prices_tokens(model_info): + return None + input_tokens_details: Final[object] = getattr(usage, "input_tokens_details", None) prompt_tokens_details: PromptTokensDetailsWrapper | None = None if input_tokens_details is not None: @@ -1790,6 +1851,7 @@ def calculate_image_response_cost_from_usage( model=model, usage=normalized_usage, custom_llm_provider=custom_llm_provider, + model_info=model_info, ) return prompt_cost + completion_cost @@ -1850,9 +1912,15 @@ class CostCalculatorUtils: size: str | None = None, optional_params: dict | None = None, call_type: str | None = None, + model_info: ModelInfo | None = None, ) -> float: """ Route the image generation cost calculator based on the custom_llm_provider + + ``model_info`` is the deployment's own price table. Its valid prices are laid over the shared + cost-map entry and handed to the provider calculator, so per-image, per-pixel and per-token + deployment prices all apply while provider logic (token-first billing, grounding surcharges, + image counting) stays in one place. An unparseable price is logged and ignored. """ from litellm.cost_calculator import default_image_cost_calculator from litellm.llms.azure_ai.image_generation.cost_calculator import ( @@ -1878,12 +1946,14 @@ class CostCalculatorUtils: quality or completion_response.quality or _requested_image_param(optional_params, "quality") or "standard" ) resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0) + pricing: Final = deployment_pricing(model_info) if custom_llm_provider == litellm.LlmProviders.VERTEX_AI.value: if isinstance(completion_response, ImageResponse): return vertex_ai_image_cost_calculator( model=model, image_response=completion_response, + model_info=pricing, ) elif custom_llm_provider == litellm.LlmProviders.BEDROCK.value: if isinstance(completion_response, ImageResponse): @@ -1902,6 +1972,7 @@ class CostCalculatorUtils: return recraft_image_cost_calculator( model=model, image_response=completion_response, + model_info=pricing, ) elif custom_llm_provider == litellm.LlmProviders.AIML.value: from litellm.llms.aiml.image_generation.cost_calculator import ( @@ -1911,6 +1982,7 @@ class CostCalculatorUtils: return aiml_image_cost_calculator( model=model, image_response=completion_response, + model_info=pricing, ) elif custom_llm_provider == litellm.LlmProviders.COMETAPI.value: from litellm.llms.cometapi.image_generation.cost_calculator import ( @@ -1920,6 +1992,7 @@ class CostCalculatorUtils: return cometapi_image_cost_calculator( model=model, image_response=completion_response, + model_info=pricing, ) elif custom_llm_provider == litellm.LlmProviders.GEMINI.value: if call_type in ( @@ -1933,6 +2006,7 @@ class CostCalculatorUtils: return gemini_image_edit_cost_calculator( model=model, image_response=completion_response, + model_info=pricing, ) from litellm.llms.gemini.image_generation.cost_calculator import ( cost_calculator as gemini_image_cost_calculator, @@ -1941,6 +2015,7 @@ class CostCalculatorUtils: return gemini_image_cost_calculator( model=model, image_response=completion_response, + model_info=pricing, ) elif custom_llm_provider == litellm.LlmProviders.AZURE_AI.value: return azure_ai_image_cost_calculator( @@ -1949,6 +2024,7 @@ class CostCalculatorUtils: size=resolved_size, n=resolved_n, optional_params=optional_params, + model_info=pricing, ) elif custom_llm_provider == litellm.LlmProviders.FAL_AI.value: from litellm.llms.fal_ai.cost_calculator import ( @@ -1959,6 +2035,7 @@ class CostCalculatorUtils: model=model, image_response=completion_response, optional_params=optional_params, + model_info=pricing, ) elif custom_llm_provider == litellm.LlmProviders.RUNWAYML.value: from litellm.llms.runwayml.cost_calculator import ( @@ -1968,6 +2045,7 @@ class CostCalculatorUtils: return runwayml_image_cost_calculator( model=model, image_response=completion_response, + model_info=pricing, ) elif ( custom_llm_provider == litellm.LlmProviders.OPENAI.value @@ -1984,6 +2062,7 @@ class CostCalculatorUtils: model=model, image_response=completion_response, custom_llm_provider=custom_llm_provider, + model_info=pricing, ) # Fall through to default for DALL-E models return default_image_cost_calculator( @@ -1993,6 +2072,7 @@ class CostCalculatorUtils: n=resolved_n, size=resolved_size, optional_params=optional_params, + model_info=pricing, ) else: return default_image_cost_calculator( @@ -2002,5 +2082,6 @@ class CostCalculatorUtils: n=resolved_n, size=resolved_size, optional_params=optional_params, + model_info=pricing, ) return 0.0 diff --git a/litellm/llms/aiml/image_generation/cost_calculator.py b/litellm/llms/aiml/image_generation/cost_calculator.py index 13427dcafc2..abf4216807a 100644 --- a/litellm/llms/aiml/image_generation/cost_calculator.py +++ b/litellm/llms/aiml/image_generation/cost_calculator.py @@ -1,19 +1,22 @@ from typing import Any, Final import litellm -from litellm.types.utils import ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.types.utils import ImageResponse, ModelInfo def cost_calculator( model: str, image_response: Any, + model_info: ModelInfo | None = None, ) -> float: """ AI/ML flux image generation cost calculator """ - _model_info: Final = litellm.get_model_info( + _model_info: Final = resolve_image_model_info( model=model, custom_llm_provider=litellm.LlmProviders.AIML.value, + model_info=model_info, ) output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 diff --git a/litellm/llms/azure_ai/image_generation/cost_calculator.py b/litellm/llms/azure_ai/image_generation/cost_calculator.py index 35d0f4fb6c3..08b732197f3 100644 --- a/litellm/llms/azure_ai/image_generation/cost_calculator.py +++ b/litellm/llms/azure_ai/image_generation/cost_calculator.py @@ -3,9 +3,22 @@ from typing import Any, Final import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import ( + _get_cost_per_unit, calculate_image_response_cost_from_usage, + resolve_image_model_info, ) -from litellm.types.utils import ImageResponse +from litellm.types.utils import ImageResponse, ModelInfo + + +def _input_cost_per_pixel(resolved: ModelInfo) -> float: + deployment_price: Final = _get_cost_per_unit(resolved, "input_cost_per_pixel", default_value=None) + if deployment_price is not None: + return deployment_price + model_cost_key: Final = resolved.get("key") + shared_entry: Final = litellm.model_cost.get(model_cost_key) if model_cost_key is not None else None + if shared_entry is None: + return 0.0 + return shared_entry.get("input_cost_per_pixel") or 0.0 def cost_calculator( @@ -14,13 +27,15 @@ def cost_calculator( size: str | None = None, n: int | None = None, optional_params: Mapping[str, object] | None = None, + model_info: ModelInfo | None = None, ) -> float: """ Azure AI image generation cost calculator """ - _model_info: Final = litellm.get_model_info( + _model_info: Final = resolve_image_model_info( model=model, custom_llm_provider=litellm.LlmProviders.AZURE_AI.value, + model_info=model_info, ) if isinstance(image_response, ImageResponse): @@ -28,6 +43,7 @@ def cost_calculator( model=model, image_response=image_response, custom_llm_provider=litellm.LlmProviders.AZURE_AI.value, + model_info=_model_info, ) if token_based_cost is not None: return token_based_cost @@ -37,9 +53,7 @@ def cost_calculator( if output_cost_per_image: return output_cost_per_image * num_images - model_cost: Final = litellm.model_cost[_model_info["key"]] - input_cost_per_pixel: Final[float] = model_cost.get("input_cost_per_pixel") or 0.0 - if input_cost_per_pixel: + if _input_cost_per_pixel(_model_info): from litellm.cost_calculator import default_image_cost_calculator width: Final = optional_params.get("width") if optional_params else None @@ -50,10 +64,11 @@ def cost_calculator( else size or image_response.size ) return default_image_cost_calculator( - model=_model_info["key"], + model=_model_info.get("key", model), custom_llm_provider=litellm.LlmProviders.AZURE_AI.value, size=pixel_size, n=num_images, + model_info=model_info, ) return 0.0 diff --git a/litellm/llms/cometapi/image_generation/cost_calculator.py b/litellm/llms/cometapi/image_generation/cost_calculator.py index 0ad9f75c45f..8f767cf311e 100644 --- a/litellm/llms/cometapi/image_generation/cost_calculator.py +++ b/litellm/llms/cometapi/image_generation/cost_calculator.py @@ -1,19 +1,22 @@ from typing import Any, Final import litellm -from litellm.types.utils import ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.types.utils import ImageResponse, ModelInfo def cost_calculator( model: str, image_response: Any, + model_info: ModelInfo | None = None, ) -> float: """ CometAPI image generation cost calculator """ - _model_info: Final = litellm.get_model_info( + _model_info: Final = resolve_image_model_info( model=model, custom_llm_provider=litellm.LlmProviders.COMETAPI.value, + model_info=model_info, ) output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 diff --git a/litellm/llms/fal_ai/cost_calculator.py b/litellm/llms/fal_ai/cost_calculator.py index 31f0995bf9f..3036590e58d 100644 --- a/litellm/llms/fal_ai/cost_calculator.py +++ b/litellm/llms/fal_ai/cost_calculator.py @@ -7,7 +7,8 @@ from typing import Final from pydantic import TypeAdapter import litellm -from litellm.types.utils import ImageObject, ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import deployment_pricing, resolve_image_model_info +from litellm.types.utils import ImageObject, ImageResponse, ModelInfo FAL_KEYED_PRICING_DEFAULT_QUALITY: Final[str] = "high" _DEFAULT_KEYED_DIMENSIONS: Final[tuple[int, int]] = (1024, 768) @@ -149,6 +150,7 @@ def cost_calculator( model: str, image_response: object, optional_params: Mapping[str, object] | None = None, + model_info: ModelInfo | None = None, ) -> float: """ fal.ai image generation cost calculator @@ -156,8 +158,14 @@ def cost_calculator( if not isinstance(image_response, ImageResponse): raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}") normalized_model: Final = model.removeprefix(f"{litellm.LlmProviders.FAL_AI.value}/") - params: Final[Mapping[str, object]] = optional_params or MappingProxyType({}) images: Final = tuple(image_response.data or ()) + deployment_prices: Final = deployment_pricing(model_info) + deployment_cost_per_image: Final = ( + None if deployment_prices is None else deployment_prices.get("output_cost_per_image") + ) + if deployment_cost_per_image is not None: + return deployment_cost_per_image * len(images) + params: Final[Mapping[str, object]] = optional_params or MappingProxyType({}) keyed_costs: Final = tuple( _keyed_cost_per_image( model=normalized_model, @@ -168,15 +176,16 @@ def cost_calculator( ) if not any(cost is None for cost in keyed_costs): return sum(cost for cost in keyed_costs if cost is not None) - model_info: Final = litellm.get_model_info( + resolved_model_info: Final = resolve_image_model_info( model=normalized_model, custom_llm_provider=litellm.LlmProviders.FAL_AI.value, + model_info=deployment_prices, ) - raw_output_cost_per_image: Final = model_info.get("output_cost_per_image") + raw_output_cost_per_image: Final = resolved_model_info.get("output_cost_per_image") output_cost_per_image: Final = ( float(raw_output_cost_per_image) if isinstance(raw_output_cost_per_image, (int, float)) else 0.0 ) - raw_output_cost_per_pixel: Final = model_info.get("output_cost_per_pixel") + raw_output_cost_per_pixel: Final = resolved_model_info.get("output_cost_per_pixel") output_cost_per_pixel: Final = ( float(raw_output_cost_per_pixel) if isinstance(raw_output_cost_per_pixel, (int, float)) else None ) diff --git a/litellm/llms/gemini/image_edit/cost_calculator.py b/litellm/llms/gemini/image_edit/cost_calculator.py index 956edb849a0..321fbbaeb37 100644 --- a/litellm/llms/gemini/image_edit/cost_calculator.py +++ b/litellm/llms/gemini/image_edit/cost_calculator.py @@ -7,11 +7,13 @@ from typing import Any from litellm.llms.gemini.image_generation.cost_calculator import ( cost_calculator as image_generation_cost_calculator, ) +from litellm.types.utils import ModelInfo def cost_calculator( model: str, image_response: Any, + model_info: ModelInfo | None = None, ) -> float: """ Gemini image edit cost calculator. @@ -22,4 +24,5 @@ def cost_calculator( return image_generation_cost_calculator( model=model, image_response=image_response, + model_info=model_info, ) diff --git a/litellm/llms/gemini/image_generation/cost_calculator.py b/litellm/llms/gemini/image_generation/cost_calculator.py index ea0e77e1b81..e3232f8fb7b 100644 --- a/litellm/llms/gemini/image_generation/cost_calculator.py +++ b/litellm/llms/gemini/image_generation/cost_calculator.py @@ -4,24 +4,26 @@ Google AI Image Generation Cost Calculator from typing import Any, Final -import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_image_response_cost_from_usage, calculate_image_response_web_search_cost, + resolve_image_model_info, ) -from litellm.types.utils import ImageResponse +from litellm.types.utils import ImageResponse, ModelInfo def cost_calculator( model: str, image_response: Any, + model_info: ModelInfo | None = None, ) -> float: """ Google AI Image Generation Cost Calculator """ - _model_info: Final = litellm.get_model_info( + _model_info: Final = resolve_image_model_info( model=model, custom_llm_provider="gemini", + model_info=model_info, ) if not isinstance(image_response, ImageResponse): @@ -37,6 +39,7 @@ def cost_calculator( model=model, image_response=image_response, custom_llm_provider="gemini", + model_info=_model_info, ) if token_based_cost is not None: return token_based_cost + web_search_cost diff --git a/litellm/llms/openai/image_generation/cost_calculator.py b/litellm/llms/openai/image_generation/cost_calculator.py index 938a0a57f2a..350fdbaee2d 100644 --- a/litellm/llms/openai/image_generation/cost_calculator.py +++ b/litellm/llms/openai/image_generation/cost_calculator.py @@ -9,27 +9,37 @@ from typing import Final from litellm import verbose_logger from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_image_response_cost_from_usage, + flat_image_cost, generic_cost_per_token, + resolve_image_model_info, ) -from litellm.types.utils import ImageResponse, Usage +from litellm.types.utils import ImageResponse, ModelInfo, Usage def cost_calculator( model: str, image_response: ImageResponse, custom_llm_provider: str | None = None, + model_info: ModelInfo | None = None, ) -> float: """Calculate cost for OpenAI gpt-image models (token-based pricing).""" + provider: Final = custom_llm_provider or "openai" + price_table: Final = ( + None + if model_info is None + else resolve_image_model_info(model=model, custom_llm_provider=provider, model_info=model_info) + ) + usage: Final = getattr(image_response, "usage", None) if usage is None: verbose_logger.debug("No usage data available for %s, cannot calculate token-based cost", model) - return 0.0 - - provider: Final = custom_llm_provider or "openai" + return flat_image_cost(price_table, image_response) # A chat Usage with an explicit output breakdown: cost via generic_cost_per_token. if isinstance(usage, Usage) and usage.completion_tokens_details is not None: - prompt_cost, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider=provider) + prompt_cost, completion_cost = generic_cost_per_token( + model=model, usage=usage, custom_llm_provider=provider, model_info=price_table + ) return prompt_cost + completion_cost # ImageUsage / ResponseAPIUsage: reuse the shared helper (same path as @@ -38,7 +48,7 @@ def cost_calculator( # does not itemize output and splitting text/image when it does. if getattr(usage, "input_tokens", None) is not None: token_based_cost: Final = calculate_image_response_cost_from_usage( - model=model, image_response=image_response, custom_llm_provider=provider + model=model, image_response=image_response, custom_llm_provider=provider, model_info=price_table ) if token_based_cost is not None: return token_based_cost @@ -46,7 +56,9 @@ def cost_calculator( # Fallback: a Usage with no output breakdown that the image helper can't read — # cost via generic_cost_per_token (text rate) instead of returning 0.0. if isinstance(usage, Usage): - prompt_cost, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider=provider) + prompt_cost, completion_cost = generic_cost_per_token( + model=model, usage=usage, custom_llm_provider=provider, model_info=price_table + ) return prompt_cost + completion_cost - return 0.0 + return flat_image_cost(price_table, image_response) diff --git a/litellm/llms/recraft/cost_calculator.py b/litellm/llms/recraft/cost_calculator.py index 2866d8e1d96..221d4a35913 100644 --- a/litellm/llms/recraft/cost_calculator.py +++ b/litellm/llms/recraft/cost_calculator.py @@ -1,19 +1,22 @@ from typing import Any, Final import litellm -from litellm.types.utils import ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.types.utils import ImageResponse, ModelInfo def cost_calculator( model: str, image_response: Any, + model_info: ModelInfo | None = None, ) -> float: """ Recraft image generation cost calculator """ - _model_info: Final = litellm.get_model_info( + _model_info: Final = resolve_image_model_info( model=model, custom_llm_provider=litellm.LlmProviders.RECRAFT.value, + model_info=model_info, ) output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 diff --git a/litellm/llms/runwayml/cost_calculator.py b/litellm/llms/runwayml/cost_calculator.py index fdd4b904b60..07f7d564ac2 100644 --- a/litellm/llms/runwayml/cost_calculator.py +++ b/litellm/llms/runwayml/cost_calculator.py @@ -1,12 +1,14 @@ from typing import Any, Final import litellm -from litellm.types.utils import ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.types.utils import ImageResponse, ModelInfo def cost_calculator( model: str, image_response: Any, + model_info: ModelInfo | None = None, ) -> float: """ RunwayML image generation cost calculator. @@ -14,9 +16,10 @@ def cost_calculator( RunwayML charges per image generated, not per pixel. Pricing is stored in model_prices_and_context_window.json with output_cost_per_image. """ - _model_info: Final = litellm.get_model_info( + _model_info: Final = resolve_image_model_info( model=model, custom_llm_provider=litellm.LlmProviders.RUNWAYML.value, + model_info=model_info, ) output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 diff --git a/litellm/llms/vertex_ai/image_generation/cost_calculator.py b/litellm/llms/vertex_ai/image_generation/cost_calculator.py index f3117e65681..6db4f02ee5a 100644 --- a/litellm/llms/vertex_ai/image_generation/cost_calculator.py +++ b/litellm/llms/vertex_ai/image_generation/cost_calculator.py @@ -4,24 +4,26 @@ Vertex AI Image Generation Cost Calculator from typing import Final -import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_image_response_cost_from_usage, calculate_image_response_web_search_cost, + resolve_image_model_info, ) -from litellm.types.utils import ImageResponse +from litellm.types.utils import ImageResponse, ModelInfo def cost_calculator( model: str, image_response: ImageResponse, + model_info: ModelInfo | None = None, ) -> float: """ Vertex AI Image Generation Cost Calculator """ - _model_info: Final = litellm.get_model_info( + _model_info: Final = resolve_image_model_info( model=model, custom_llm_provider="vertex_ai", + model_info=model_info, ) web_search_cost: Final = calculate_image_response_web_search_cost( @@ -34,6 +36,7 @@ def cost_calculator( model=model, image_response=image_response, custom_llm_provider="vertex_ai", + model_info=_model_info, ) if token_based_cost is not None: return token_based_cost + web_search_cost diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 4b25c87d70f..5b21684ca9c 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1,6 +1,7 @@ +import json from collections.abc import Mapping from datetime import datetime, timezone -from typing import cast +from typing import Final, cast import pytest @@ -3637,6 +3638,138 @@ def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_ assert creation_1h == pytest.approx(expected_creation_1h) +def _image_response(num_images: int = 1, usage: ImageUsage | None = None) -> ImageResponse: + return ImageResponse( + data=[ImageObject(url="https://example.com/img.png") for _ in range(num_images)], + usage=usage, + ) + + +_GPT_IMAGE_2_HIGH_1024: Final = {"quality": "high", "image_size": {"width": 1024, "height": 1024}} + + +@pytest.mark.parametrize( + ("model", "optional_params", "model_info", "num_images", "expected_cost"), + [ + ("fal-ai/unlisted-image-model", None, {"output_cost_per_image": 0.08}, 1, 0.08), + ("fal-ai/unlisted-image-model", None, {"output_cost_per_image": 0.08}, 2, 0.16), + ("fal-ai/unlisted-image-model", None, {"output_cost_per_image": "0.08"}, 1, 0.08), + ("openai/gpt-image-2", _GPT_IMAGE_2_HIGH_1024, {"output_cost_per_image": 0.5}, 1, 0.5), + ("openai/gpt-image-2", _GPT_IMAGE_2_HIGH_1024, {"mode": "image_generation"}, 1, 0.211), + ("openai/gpt-image-2", _GPT_IMAGE_2_HIGH_1024, {"output_cost_per_image": "0.08 USD"}, 1, 0.211), + ], +) +def test_route_image_generation_cost_honors_deployment_model_info( + _local_model_cost_map: None, + model: str, + optional_params: dict[str, object] | None, + model_info: ModelInfo, + num_images: int, + expected_cost: float, +) -> None: + cost = CostCalculatorUtils.route_image_generation_cost_calculator( + model=model, + completion_response=_image_response(num_images), + custom_llm_provider="fal_ai", + optional_params=optional_params, + call_type="image_generation", + model_info=model_info, + ) + + assert cost == pytest.approx(expected_cost) + + +def test_route_image_generation_cost_openai_honors_deployment_input_cost_per_image( + _local_model_cost_map: None, +) -> None: + cost = CostCalculatorUtils.route_image_generation_cost_calculator( + model="dall-e-3", + completion_response=_image_response(), + custom_llm_provider="openai", + quality="standard", + size="1024-x-1024", + call_type="image_generation", + model_info={"input_cost_per_image": 0.07}, + ) + + assert cost == pytest.approx(0.07) + + +def test_route_image_generation_cost_gemini_adds_grounding_to_deployment_image_price( + _local_model_cost_map: None, +) -> None: + usage = ImageUsage( + input_tokens=0, + input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=0), + output_tokens=0, + total_tokens=0, + web_search_requests=3, + ) + + cost = CostCalculatorUtils.route_image_generation_cost_calculator( + model="gemini/gemini-3.1-flash-image-preview", + completion_response=_image_response(usage=usage), + custom_llm_provider="gemini", + call_type="image_generation", + model_info={"output_cost_per_image": 0.1}, + ) + + assert cost == pytest.approx(0.1 + 3 * 0.014) + + +def test_route_image_generation_cost_gemini_bills_tokens_when_no_image_returned( + _local_model_cost_map: None, +) -> None: + usage = ImageUsage( + input_tokens=10, + input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=10), + output_tokens=1290, + total_tokens=1300, + ) + + cost = CostCalculatorUtils.route_image_generation_cost_calculator( + model="gemini/gemini-3.1-flash-image-preview", + completion_response=ImageResponse(data=[], usage=usage), + custom_llm_provider="gemini", + call_type="image_generation", + model_info={"output_cost_per_image": 0.08}, + ) + + assert cost == pytest.approx(10 * 5e-07 + 1290 * 6e-05) + + +@pytest.mark.parametrize( + ("custom_llm_provider", "model"), + [ + ("gemini", "gemini/unlisted-image-model"), + ("vertex_ai", "vertex_ai/unlisted-image-model"), + ("azure_ai", "unlisted-image-model"), + ("openai", "gpt-image-unlisted"), + ], +) +def test_route_image_generation_cost_bills_deployment_image_price_when_unlisted_model_reports_tokens( + _local_model_cost_map: None, + custom_llm_provider: str, + model: str, +) -> None: + usage = ImageUsage( + input_tokens=10, + input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=10), + output_tokens=1290, + total_tokens=1300, + ) + + cost = CostCalculatorUtils.route_image_generation_cost_calculator( + model=model, + completion_response=_image_response(num_images=2, usage=usage), + custom_llm_provider=custom_llm_provider, + call_type="image_generation", + model_info={"output_cost_per_image": 0.05}, + ) + + assert cost == pytest.approx(0.10) + + def _batch_rates_model_info(**rates: object) -> ModelInfo: return cast(ModelInfo, dict(rates)) diff --git a/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py b/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py index 4c963d14ada..7fb45e1b092 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py +++ b/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py @@ -4,6 +4,8 @@ Tests for get_litellm_params and related helpers. Ensures backward compatibility after sparse kwargs extraction optimization. """ +from typing import Final + import pytest from litellm.litellm_core_utils.get_litellm_params import ( @@ -12,6 +14,10 @@ from litellm.litellm_core_utils.get_litellm_params import ( get_litellm_params, ) +NAMED_PRICE_PARAMS: Final = frozenset( + {"input_cost_per_token", "output_cost_per_token", "input_cost_per_second", "output_cost_per_second"} +) + class TestGetBaseModelFromLitellmCallMetadata: def test_none_metadata_returns_none(self): @@ -40,10 +46,27 @@ class TestGetLitellmParamsKwargsExtraction: """Verify that optional kwargs are correctly extracted via sparse extraction.""" def test_no_kwargs_omits_optional_keys(self): - """When no kwargs passed, optional keys should not be in result.""" + """When no kwargs passed, optional keys are absent; the named price params are present as None.""" result = get_litellm_params(api_key="test-key") - for key in _OPTIONAL_KWARGS_KEYS: + for key in _OPTIONAL_KWARGS_KEYS - NAMED_PRICE_PARAMS: assert key not in result + for key in NAMED_PRICE_PARAMS: + assert result[key] is None + + def test_custom_pricing_kwargs_are_extracted(self) -> None: + from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model + from litellm.types.router import CustomPricingLiteLLMParams + + assert set(CustomPricingLiteLLMParams.model_fields) <= _OPTIONAL_KWARGS_KEYS + + result = get_litellm_params(output_cost_per_image=0.08, input_cost_per_audio_token=1e-6) + assert result["output_cost_per_image"] == 0.08 + assert result["input_cost_per_audio_token"] == 1e-6 + assert use_custom_pricing_for_model(result) is True + + result_without_prices = get_litellm_params() + assert "output_cost_per_image" not in result_without_prices + assert use_custom_pricing_for_model(result_without_prices) is False def test_present_kwargs_are_extracted(self): result = get_litellm_params( diff --git a/tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py b/tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py index 512e98b4151..fdd21c87732 100644 --- a/tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py +++ b/tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py @@ -205,6 +205,36 @@ def test_flux2_flex_cost_accepts_lowercase_model_spelling(): assert cost == pytest.approx(5e-08 * 1536 * 1024 * 2) +def test_flux2_flex_cost_prefers_deployment_input_cost_per_pixel() -> None: + response: Final = ImageResponse(data=[ImageObject(b64_json="aW1n"), ImageObject(b64_json="aW1n")]) + + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model="FLUX.2-flex", + completion_response=response, + custom_llm_provider="azure_ai", + size="2048x1024", + call_type="image_generation", + model_info={"input_cost_per_pixel": 2e-07}, + ) + + assert cost == pytest.approx(2e-07 * 2048 * 1024 * 2) + + +def test_unlisted_azure_ai_model_bills_deployment_input_cost_per_pixel() -> None: + response: Final = ImageResponse(data=[ImageObject(b64_json="aW1n"), ImageObject(b64_json="aW1n")]) + + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model="unlisted-flux-deployment", + completion_response=response, + custom_llm_provider="azure_ai", + size="1024x1024", + call_type="image_generation", + model_info={"input_cost_per_pixel": 1e-07}, + ) + + assert cost == pytest.approx(1e-07 * 1024 * 1024 * 2) + + def test_flux2_response_preserves_mapped_dimensions(): config = AzureFoundryFluxImageGenerationConfig() params = config.map_openai_params( diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index a33701cd1bf..7d2a04c500f 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1,6 +1,7 @@ import datetime import time -from types import MappingProxyType +from pathlib import Path +from types import MappingProxyType, SimpleNamespace from typing import Final, cast import pytest @@ -22,8 +23,13 @@ from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import OpenAIRealtimeStreamList, ResponseAPIUsage, ResponsesAPIResponse from litellm.types.rerank import RerankResponse from litellm.types.utils import ( + CacheCreationTokenDetails, CallTypes, Choices, + ImageObject, + ImageResponse, + ImageUsage, + ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, @@ -686,6 +692,90 @@ def test_tiered_pricing_only_deployment_selects_router_model_id(): assert router_model_id in selected +@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) +def test_completion_cost_image_generation_reads_deployment_model_info_price_from_logging_metadata( + _local_model_cost_map: None, metadata_key: str +) -> None: + cost = completion_cost( + completion_response=ImageResponse(data=[ImageObject(url="https://example.com/img.png")]), + model="fal_ai/fal-ai/unlisted-image-model", + call_type="image_generation", + custom_pricing=True, + litellm_logging_obj=SimpleNamespace( + litellm_params={metadata_key: {"model_info": {"output_cost_per_image": 0.08}}} + ), + ) + + assert cost == pytest.approx(0.08) + + +def test_completion_cost_image_generation_registered_deployment_price_keeps_map_token_rates( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + deployment_id: Final = "gemini-image-deployment-priced-per-image" + monkeypatch.setitem( + litellm.model_cost, + deployment_id, + {"mode": "image_generation", "litellm_provider": "gemini", "output_cost_per_image": 0.1}, + ) + usage: Final = ImageUsage( + input_tokens=10, + input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=10), + output_tokens=1290, + total_tokens=1300, + ) + + cost = completion_cost( + completion_response=ImageResponse(data=[ImageObject(url="https://example.com/img.png")], usage=usage), + model="gemini/gemini-3.1-flash-image-preview", + custom_llm_provider="gemini", + call_type="image_generation", + custom_pricing=True, + router_model_id=deployment_id, + litellm_logging_obj=SimpleNamespace(litellm_params={"metadata": {"model_info": {"id": deployment_id}}}), + ) + + assert cost == pytest.approx(10 * 5e-07 + 1290 * 6e-05) + + +def test_completion_cost_image_generation_ignores_deployment_model_info_without_custom_pricing( + _local_model_cost_map: None, +) -> None: + cost = completion_cost( + completion_response=ImageResponse(data=[ImageObject(url="https://example.com/img.png")]), + model="fal_ai/openai/gpt-image-2", + call_type="image_generation", + custom_pricing=False, + optional_params={"quality": "high", "image_size": {"width": 1024, "height": 1024}}, + litellm_logging_obj=SimpleNamespace( + litellm_params={"litellm_metadata": {"model_info": {"output_cost_per_image": 0.5}}} + ), + ) + + assert cost == pytest.approx(0.211) + + +async def test_router_image_generation_bills_litellm_params_output_cost_per_image() -> None: + from litellm import Router + + router = Router( + model_list=[ + { + "model_name": "img", + "litellm_params": { + "model": "fal_ai/fal-ai/unlisted-image-model", + "api_key": "sk-fake", + "output_cost_per_image": 0.08, + }, + } + ] + ) + + response = await router.aimage_generation(model="img", prompt="x", mock_response="https://example.com/img.png") + + assert response._hidden_params["response_cost"] == pytest.approx(0.08) + + def test_tiered_pricing_only_deployment_completion_cost_is_nonzero(): """End-to-end: a tier-only deployment must produce the tiered cost, not $0. Mirrors the reported dashscope/qwen3.7-plus trace (12 prompt + 377 diff --git a/tests/test_litellm/test_gpt_image_cost_calculator.py b/tests/test_litellm/test_gpt_image_cost_calculator.py index 42d4c699200..0f471430daf 100644 --- a/tests/test_litellm/test_gpt_image_cost_calculator.py +++ b/tests/test_litellm/test_gpt_image_cost_calculator.py @@ -16,6 +16,8 @@ import litellm from litellm.types.utils import ( ImageObject, ImageResponse, + ImageUsage, + ImageUsageInputTokensDetails, ) @@ -52,6 +54,38 @@ class TestGPTImageCostCalculator: assert cost == 0.0 + @pytest.mark.parametrize( + "usage", + [ + None, + ImageUsage( + input_tokens=0, + input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=0), + output_tokens=0, + total_tokens=0, + ), + ], + ) + def test_gpt_image_1_bills_deployment_output_cost_per_image_without_usage_tokens( + self, usage: ImageUsage | None + ) -> None: + from litellm.llms.openai.image_generation.cost_calculator import cost_calculator + + image_response = ImageResponse( + created=1234567890, + data=[ImageObject(url="http://example.com/one.jpg"), ImageObject(url="http://example.com/two.jpg")], + usage=usage, + ) + + cost = cost_calculator( + model="gpt-image-1", + image_response=image_response, + custom_llm_provider="openai", + model_info={"output_cost_per_image": 0.05}, + ) + + assert cost == pytest.approx(0.10) + class TestGPTImageCostRouting: """Test that gpt-image models are properly routed to the token-based calculator""" From ee3f5a8bdfb52486ca954fe4feaaad590d904108 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 11:50:19 -0700 Subject: [PATCH 09/48] fix(streaming): let a later usage event zero out stale cache counts (#40736) (#42330) * fix(streaming): let a later usage event zero out stale cache counts (#40736) The usage merger only replaced cache_creation_input_tokens and cache_read_input_tokens with a positive value, so when Anthropic's message_delta restated the cache block with a 0 write, the 58k write from message_start survived next to the 58k read. prompt_tokens minus both cache counts then went negative and the write was billed twice A usage event that reports any prompt-side count is now authoritative for both cache fields, zeros included. An event with no prompt-side counts, such as an output-only message_delta, still leaves the earlier values alone * test(streaming): cover an input-only message_delta and the derived uncached input in the cache count merger test --------- Co-authored-by: muhammadwaqar12 Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: shrey kharbanda --- .../streaming_chunk_builder_utils.py | 12 +++- .../test_streaming_chunk_builder_utils.py | 67 +++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 025db65a7ce..aa4e0cf5495 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -165,6 +165,14 @@ class _UsageSummary(TypedDict): cost: float | None +def _reports_prompt_side_usage(usage_summary: "_UsageSummary") -> bool: + return ( + (usage_summary["prompt_tokens"] or 0) > 0 + or (usage_summary["cache_creation_input_tokens"] or 0) > 0 + or (usage_summary["cache_read_input_tokens"] or 0) > 0 + ) + + def capture_cache_creation_token_details( prompt_tokens_details: PromptTokensDetailsWrapper | None, current: CacheCreationTokenDetails | None, @@ -888,11 +896,11 @@ class ChunkProcessor: if usage_chunk_dict["completion_tokens"] is not None and usage_chunk_dict["completion_tokens"] > 0: completion_usage_updates += 1 if usage_chunk_dict["cache_creation_input_tokens"] is not None and ( - usage_chunk_dict["cache_creation_input_tokens"] > 0 or cache_creation_input_tokens is None + _reports_prompt_side_usage(usage_chunk_dict) or cache_creation_input_tokens is None ): cache_creation_input_tokens = usage_chunk_dict["cache_creation_input_tokens"] if usage_chunk_dict["cache_read_input_tokens"] is not None and ( - usage_chunk_dict["cache_read_input_tokens"] > 0 or cache_read_input_tokens is None + _reports_prompt_side_usage(usage_chunk_dict) or cache_read_input_tokens is None ): cache_read_input_tokens = usage_chunk_dict["cache_read_input_tokens"] if usage_chunk_dict["completion_tokens_details"] is not None: diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index 5d75c6699cf..af763da2d87 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1,5 +1,6 @@ from collections.abc import Mapping, Sequence from typing import Final +from unittest.mock import MagicMock import pytest @@ -7,6 +8,7 @@ import pytest from litellm import ChatCompletionUsageBlock, stream_chunk_builder from litellm.types.utils import GenericStreamingChunk from litellm.litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor +from litellm.llms.anthropic.chat.handler import ModelResponseIterator from litellm.types.utils import ( ChatCompletionDeltaToolCall, ChatCompletionMessageToolCall, @@ -1650,6 +1652,71 @@ def test_calculate_usage_falls_back_to_prompt_counter_when_mock_stream_has_no_ad assert usage.prompt_tokens == 77 +@pytest.mark.parametrize( + ("message_delta_usage", "expected_cache_creation", "expected_cache_read"), + [ + ( + { + "input_tokens": 2, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 58352, + "output_tokens": 408, + }, + 0, + 58352, + ), + ({"output_tokens": 408}, 58352, 0), + ({"input_tokens": 2, "output_tokens": 408}, 58352, 0), + ], + ids=["delta_restates_cache_counts", "delta_reports_output_only", "delta_reports_input_and_output_only"], +) +def test_anthropic_stream_usage_takes_cache_counts_from_last_event_that_reports_them( + message_delta_usage: Mapping[str, int], expected_cache_creation: int, expected_cache_read: int +) -> None: + iterator: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=True, json_mode=False) + events: Final = ( + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [], + "stop_reason": None, + "usage": { + "input_tokens": 2, + "cache_creation_input_tokens": 58352, + "cache_read_input_tokens": 0, + "output_tokens": 1, + }, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hi"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": dict(message_delta_usage)}, + {"type": "message_stop"}, + ) + + response: Final = stream_chunk_builder( + chunks=[iterator.chunk_parser(event) for event in events], + messages=[{"role": "user", "content": "hi"}], + ) + + assert response.usage.cache_creation_input_tokens == expected_cache_creation + assert response.usage.cache_read_input_tokens == expected_cache_read + assert response.usage.prompt_tokens == 58354 + assert response.usage.prompt_tokens_details.cache_creation_tokens == expected_cache_creation + assert response.usage.prompt_tokens_details.cached_tokens == expected_cache_read + assert ( + response.usage.prompt_tokens + - response.usage.cache_read_input_tokens + - response.usage.cache_creation_input_tokens + == 2 + ) + + _ZERO_USAGE_TEXT_CHUNKS: Final = ( _openai_chunk(choices=[{"index": 0, "delta": {"role": "assistant", "content": "Hi"}, "finish_reason": None}]), _openai_chunk(choices=[{"index": 0, "delta": {"content": " there"}, "finish_reason": None}]), From 5af9b0136c98a14b6c68f8301e8dce71c96f437f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 18:51:30 +0000 Subject: [PATCH 10/48] fix(fal_ai): handle seconds=auto and oversized sizes for minimax h3 videos (#42504) * test(fal_ai): e2e for minimax h3 auto duration and oversized size Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(fal_ai): keep auto duration literal and make h3 tier lookup total Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(fal_ai): omit duration for h3 when seconds is auto Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(fal_ai): move the minimax h3 auto duration and oversized size repro to tests/integration Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/fal_ai/videos/transformation.py | 7 ++- tests/integration/contracts.json | 6 ++ .../providers/test_fal_ai_video_wire.py | 63 +++++++++++++++++++ .../test_fal_ai_video_transformation.py | 12 ++++ 4 files changed, 87 insertions(+), 1 deletion(-) diff --git a/litellm/llms/fal_ai/videos/transformation.py b/litellm/llms/fal_ai/videos/transformation.py index 51082a6773b..44b33a8f58f 100644 --- a/litellm/llms/fal_ai/videos/transformation.py +++ b/litellm/llms/fal_ai/videos/transformation.py @@ -104,7 +104,10 @@ def _profile_for_model(model: str) -> _ModelProfile: def _resolution_for_short_side(short_side: int, profile: _ModelProfile) -> str: - return next(resolution for threshold, resolution in profile.resolution_tiers if short_side <= threshold) + return next( + (resolution for threshold, resolution in profile.resolution_tiers if short_side <= threshold), + profile.resolution_tiers[-1][1], + ) def _model_path_from_request_url(raw_response: httpx.Response) -> str | None: @@ -351,6 +354,8 @@ class FalAIVideoConfig(BaseVideoConfig): duration: Final[str | None] = _duration_value(seconds) if duration is None: raise ValueError("fal.ai seconds must be a numeric value") + if duration == "auto": + return MappingProxyType({}) if profile.integer_duration else MappingProxyType({"duration": duration}) return MappingProxyType({"duration": int(duration) if profile.integer_duration else duration}) def validate_environment( diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index e1b5940935f..8c9f33094da 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -169,6 +169,12 @@ "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_video_failed_result_reports_failed_status_and_fal_error": [ "other.provider_wire.fal_ai.video_failed_result_surfaces_fal_error" ], + "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_auto_duration_omits_duration_and_queues": [ + "other.provider_wire.fal_ai.h3_auto_duration_omits_duration_and_queues" + ], + "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_oversized_size_uses_top_resolution_tier_and_queues": [ + "other.provider_wire.fal_ai.h3_oversized_size_uses_top_resolution_tier" + ], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_generation_sends_quality_and_size_and_charges_keyed_row": [ "other.provider_wire.fal_ai.gpt_image_generation_quality_size_wire_and_keyed_pricing" ], diff --git a/tests/integration/providers/test_fal_ai_video_wire.py b/tests/integration/providers/test_fal_ai_video_wire.py index 276ea2868a7..1f3113bb840 100644 --- a/tests/integration/providers/test_fal_ai_video_wire.py +++ b/tests/integration/providers/test_fal_ai_video_wire.py @@ -9,6 +9,11 @@ from integration._support.wire import Reply, Request, wire_server _MODEL: Final = "bytedance/seedance-2.5/text-to-video" _H3_MODEL: Final = "minimax/h3/text-to-video" _MP4: Final = b"\x00\x00\x00\x18ftypmp42" + uuid.uuid4().bytes * 4 +_OVERSIZED_SIDE: Final = "9" * 30 + + +def _h3_queue_reply(request_id: str) -> Reply: + return Reply(body=json.dumps({"status": "IN_QUEUE", "request_id": request_id, "queue_position": 0}).encode()) @pytest.mark.covers("other.provider_wire.fal_ai.video_queue_create_status_and_content_download") @@ -177,3 +182,61 @@ def test_fal_video_failed_result_reports_failed_status_and_fal_error(gateway: Ga content: Final = gateway.request("GET", f"/v1/videos/{video_id}/content") assert content.status_code == 422, content.text assert "Failed to download the file" in content.text + + +@pytest.mark.covers("other.provider_wire.fal_ai.h3_auto_duration_omits_duration_and_queues") +def test_fal_h3_auto_duration_omits_duration_and_queues(gateway: Gateway) -> None: + request_id: Final = "fal-h3-auto-" + uuid.uuid4().hex + + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == f"/{_H3_MODEL}" + assert json.loads(request.body) == { + "prompt": "a cat playing volleyball on a beach", + "resolution": "768P", + "aspect_ratio": "16:9", + } + return _h3_queue_reply(request_id) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"fal_ai/{_H3_MODEL}", api_base=wire.url, api_key="synthetic-fal-key") + response: Final = gateway.request( + "POST", + "/v1/videos", + {"model": model, "prompt": "a cat playing volleyball on a beach", "seconds": "auto", "size": "1280x720"}, + ) + assert response.status_code == 200, response.text + assert response.json()["status"] == "queued" + assert [(request.method, request.target) for request in wire.drain()] == [("POST", f"/{_H3_MODEL}")] + + +@pytest.mark.covers("other.provider_wire.fal_ai.h3_oversized_size_uses_top_resolution_tier") +def test_fal_h3_oversized_size_uses_top_resolution_tier_and_queues(gateway: Gateway) -> None: + request_id: Final = "fal-h3-oversized-" + uuid.uuid4().hex + + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == f"/{_H3_MODEL}" + assert json.loads(request.body) == { + "prompt": "a cat playing volleyball on a beach", + "duration": 5, + "resolution": "4K", + "aspect_ratio": "1:1", + } + return _h3_queue_reply(request_id) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"fal_ai/{_H3_MODEL}", api_base=wire.url, api_key="synthetic-fal-key") + response: Final = gateway.request( + "POST", + "/v1/videos", + { + "model": model, + "prompt": "a cat playing volleyball on a beach", + "seconds": "5", + "size": f"{_OVERSIZED_SIDE}x{_OVERSIZED_SIDE}", + }, + ) + assert response.status_code == 200, response.text + assert response.json()["status"] == "queued" + assert [(request.method, request.target) for request in wire.drain()] == [("POST", f"/{_H3_MODEL}")] diff --git a/tests/test_litellm/llms/fal_ai/videos/test_fal_ai_video_transformation.py b/tests/test_litellm/llms/fal_ai/videos/test_fal_ai_video_transformation.py index 86ecbf6701b..4b8bfd264a6 100644 --- a/tests/test_litellm/llms/fal_ai/videos/test_fal_ai_video_transformation.py +++ b/tests/test_litellm/llms/fal_ai/videos/test_fal_ai_video_transformation.py @@ -1,3 +1,4 @@ +import sys from typing import Final from unittest.mock import AsyncMock, Mock @@ -86,6 +87,17 @@ class TestFalAIVideoTransformation: assert mapped["reference_image_urls"] == [url] assert "image_url" not in mapped + def test_map_openai_params_h3_omits_auto_duration(self): + assert self.config.map_openai_params({"seconds": "auto"}, H3_TEXT_MODEL, False) == {} + assert self.config.map_openai_params({"seconds": "auto"}, MODEL, False) == {"duration": "auto"} + + def test_map_openai_params_h3_size_beyond_tiers_uses_top_resolution(self): + side = str(sys.maxsize + 1) + assert self.config.map_openai_params({"size": f"{side}x{side}"}, H3_TEXT_MODEL, False) == { + "resolution": "4K", + "aspect_ratio": "1:1", + } + def test_transform_video_create_request(self): body, files, url = self.config.transform_video_create_request( model=MODEL, From f3a0bb7249dae8bbf0aa4936a211e36f1f30b2cb Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 11:52:08 -0700 Subject: [PATCH 11/48] fix(proxy): write key deleted audit logs for cascade and alias key deletions (#42446) * fix(proxy): write key deleted audit logs for cascade and alias key deletions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): assert persisted key deleted audit rows for cascade paths Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): route /audit and /v2/login to the control plane in split transport Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/hooks/key_management_event_hooks.py | 69 +++++--- .../internal_user_endpoints.py | 7 + .../management_endpoints/team_endpoints.py | 15 ++ .../management_helpers/bulk_user_deletion.py | 35 +++- tests/e2e/coverage_registry/mgmt.yaml | 4 + tests/e2e/management/management_client.py | 28 +++ tests/e2e/management/test_management_e2e.py | 132 +++++++++++++- tests/e2e/models.py | 26 +++ tests/e2e/transport.py | 2 + .../hooks/test_key_management_event_hooks.py | 46 +++++ .../test_internal_user_endpoints.py | 60 +++++++ .../test_team_endpoints.py | 162 ++++++++++++++++++ .../test_bulk_user_deletion.py | 96 ++++++++++- 13 files changed, 645 insertions(+), 37 deletions(-) diff --git a/litellm/proxy/hooks/key_management_event_hooks.py b/litellm/proxy/hooks/key_management_event_hooks.py index f75197532b4..5dc0f659a8d 100644 --- a/litellm/proxy/hooks/key_management_event_hooks.py +++ b/litellm/proxy/hooks/key_management_event_hooks.py @@ -1,7 +1,8 @@ import asyncio import json +from collections.abc import Sequence from datetime import datetime, timezone -from typing import Final +from typing import TYPE_CHECKING, Final from pydantic import TypeAdapter @@ -23,6 +24,9 @@ from litellm.proxy._types import ( from litellm.proxy.utils import _hash_token_if_needed from litellm.secret_managers.base_secret_manager import BaseSecretManager +if TYPE_CHECKING: + from prisma import models as prisma_models + # NOTE: This is the prefix for all virtual keys stored in AWS Secrets Manager LITELLM_PREFIX_STORED_VIRTUAL_KEYS: Final = "litellm/" @@ -233,6 +237,19 @@ class KeyManagementEventHooks: Handles the following: - Storing Audit Logs for key deletion """ + KeyManagementEventHooks.create_key_deleted_audit_logs( + keys_being_deleted=keys_being_deleted, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) + await KeyManagementEventHooks._delete_virtual_keys_from_secret_manager(keys_being_deleted=keys_being_deleted) + + @staticmethod + def create_key_deleted_audit_logs( + keys_being_deleted: Sequence["LiteLLM_VerificationToken | prisma_models.LiteLLM_VerificationToken"], + user_api_key_dict: UserAPIKeyAuth, + litellm_changed_by: str | None = None, + ) -> None: from litellm.proxy.management_helpers.audit_logs import ( create_audit_log_for_update, get_audit_log_changed_by, @@ -240,35 +257,33 @@ class KeyManagementEventHooks: ) from litellm.proxy.proxy_server import litellm_proxy_admin_name - # we do this after the first for loop, since first for loop is for validation. we only want this inserted after validation passes - if is_audit_logging_enabled() and data.keys is not None: - # make an audit log for each key deleted - for key in keys_being_deleted: - if key.token is None: - continue - _key_row = key.model_dump_json(exclude_none=True) + if not is_audit_logging_enabled(): + return + for key in keys_being_deleted: + key_row = LiteLLM_VerificationToken.model_validate(key, from_attributes=True) + if key_row.token is None: + continue + _key_row = key_row.model_dump_json(exclude_none=True) - asyncio.create_task( - create_audit_log_for_update( - request_data=LiteLLM_AuditLogs( - id=str(uuid.uuid4()), - updated_at=datetime.now(timezone.utc), - changed_by=get_audit_log_changed_by( - litellm_changed_by=litellm_changed_by, - user_api_key_dict=user_api_key_dict, - litellm_proxy_admin_name=litellm_proxy_admin_name, - ), - changed_by_api_key=user_api_key_dict.token, - table_name=LitellmTableNames.KEY_TABLE_NAME, - object_id=key.token, - action="deleted", - updated_values="{}", - before_value=_key_row, - ) + asyncio.create_task( + create_audit_log_for_update( + request_data=LiteLLM_AuditLogs( + id=str(uuid.uuid4()), + updated_at=datetime.now(timezone.utc), + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), + changed_by_api_key=user_api_key_dict.token, + table_name=LitellmTableNames.KEY_TABLE_NAME, + object_id=key_row.token, + action="deleted", + updated_values="{}", + before_value=_key_row, ) ) - # delete the keys from the secret manager - await KeyManagementEventHooks._delete_virtual_keys_from_secret_manager(keys_being_deleted=keys_being_deleted) + ) @staticmethod async def _store_virtual_key_in_secret_manager(secret_name: str, secret_token: str, team_id: str | None = None): diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 6509977f7ff..133181e9203 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -48,6 +48,7 @@ from litellm.proxy.common_utils.user_api_key_cache import ( user_object_permission_id_cache_key, ) from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler +from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks from litellm.proxy.hooks.model_max_budget_limiter import build_model_max_budget_usage from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks from litellm.proxy.management_endpoints.common_daily_activity import ( @@ -2539,6 +2540,12 @@ async def delete_user( prisma_client=prisma_client, ) await _verification_token_table(prisma_client).delete_many(where=key_filter) + if keys_to_delete: + KeyManagementEventHooks.create_key_deleted_audit_logs( + keys_being_deleted=keys_to_delete, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) await delete_cache_key_objects( hashed_tokens=hashed_tokens_to_delete, user_api_key_cache=user_api_key_cache, diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 0a142166bc5..d092fc2fbb7 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -118,6 +118,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_ from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars from litellm.proxy.common_utils.json_merge_patch import apply_json_merge_patch from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks from litellm.proxy.hooks.model_max_budget_limiter import ( build_model_max_budget_usage, resolve_model_budget, @@ -3751,6 +3752,13 @@ async def _team_member_delete( } ) + if keys_to_delete: + KeyManagementEventHooks.create_key_deleted_audit_logs( + keys_being_deleted=keys_to_delete, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + await delete_cache_team_object( team_id=data.team_id, team_alias=existing_team_row.team_alias, @@ -4469,6 +4477,13 @@ async def delete_team( await prisma_client.delete_data(team_id_list=data.team_ids, table_name="key") + if keys_to_delete: + KeyManagementEventHooks.create_key_deleted_audit_logs( + keys_being_deleted=keys_to_delete, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) + await _invalidate_deleted_key_cache( keys=keys_to_delete, user_api_key_cache=user_api_key_cache, diff --git a/litellm/proxy/management_helpers/bulk_user_deletion.py b/litellm/proxy/management_helpers/bulk_user_deletion.py index af51a194413..8b5b601fe8a 100644 --- a/litellm/proxy/management_helpers/bulk_user_deletion.py +++ b/litellm/proxy/management_helpers/bulk_user_deletion.py @@ -31,6 +31,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.auth_checks import delete_cache_key_objects, get_jwt_key_mapping_cache_keys_for_tokens from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks from litellm.proxy.list_api.common import PROBLEM_TYPE_BASE, ManagementProblem from litellm.proxy.management_endpoints.common_utils import ( @@ -93,20 +94,28 @@ class _TeamRemoval: team: LiteLLM_TeamTable removed: frozenset[str] matched: frozenset[int] - deleted_key_tokens: tuple[str, ...] + deleted_keys: tuple["prisma_models.LiteLLM_VerificationToken", ...] jwt_mapping_cache_keys: tuple[str, ...] + @property + def deleted_key_tokens(self) -> tuple[str, ...]: + return tuple(k.token for k in self.deleted_keys) + @dataclass(frozen=True, slots=True) class _UserBatchDeletion: removals: Mapping[str, _TeamRemoval] - deleted_key_tokens: tuple[str, ...] + deleted_keys: tuple["prisma_models.LiteLLM_VerificationToken", ...] jwt_mapping_cache_keys: tuple[str, ...] + @property + def deleted_key_tokens(self) -> tuple[str, ...]: + return tuple(k.token for k in self.deleted_keys) + @dataclass(frozen=True, slots=True) class _DeletedKeys: - tokens: tuple[str, ...] + keys: tuple["prisma_models.LiteLLM_VerificationToken", ...] jwt_mapping_cache_keys: tuple[str, ...] @@ -276,7 +285,7 @@ async def _remove_members_from_team( ), removed=cleanup_ids, matched=matched, - deleted_key_tokens=tuple(k.token for k in keys), + deleted_keys=tuple(keys), jwt_mapping_cache_keys=jwt_mapping_cache_keys, ) @@ -330,6 +339,12 @@ async def bulk_remove_team_members( members: Final = tuple(data.members[i] for i in kept_indexes) async with prisma_client.tx(timeout=_BATCH_TX_TIMEOUT) as tx: removal: Final = await _remove_members_from_team(prisma_client, tx, team_id, members, user_api_key_dict) + if removal.deleted_keys: + KeyManagementEventHooks.create_key_deleted_audit_logs( + keys_being_deleted=removal.deleted_keys, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) await delete_cache_key_objects( hashed_tokens=removal.deleted_key_tokens, user_api_key_cache=user_api_key_cache, @@ -407,7 +422,7 @@ async def _delete_user_rows( await _org_membership_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) await _membership_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) await _user_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) - return _DeletedKeys(tokens=tuple(k.token for k in keys), jwt_mapping_cache_keys=jwt_mapping_cache_keys) + return _DeletedKeys(keys=tuple(keys), jwt_mapping_cache_keys=jwt_mapping_cache_keys) async def _delete_users_tx( @@ -446,7 +461,7 @@ async def _delete_users_tx( ) return _UserBatchDeletion( removals=removals, - deleted_key_tokens=deleted_keys.tokens + tuple(t for r in removals.values() for t in r.deleted_key_tokens), + deleted_keys=deleted_keys.keys + tuple(k for r in removals.values() for k in r.deleted_keys), jwt_mapping_cache_keys=deleted_keys.jwt_mapping_cache_keys + tuple(k for r in removals.values() for k in r.jwt_mapping_cache_keys), ) @@ -469,6 +484,12 @@ async def _delete_users( except Exception as e: # noqa: BLE001 # the rolled-back batch is reported per row, not as a request failure verbose_proxy_logger.error("users/bulk_delete: failed to delete users %s: %s", sorted(user_ids), e) return _error_message(e) + if deletion.deleted_keys: + KeyManagementEventHooks.create_key_deleted_audit_logs( + keys_being_deleted=deletion.deleted_keys, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) await delete_cache_key_objects( hashed_tokens=deletion.deleted_key_tokens, user_api_key_cache=user_api_key_cache, @@ -555,7 +576,7 @@ async def bulk_delete_users( litellm_changed_by, ) if candidates - else _UserBatchDeletion(removals=MappingProxyType({}), deleted_key_tokens=(), jwt_mapping_cache_keys=()) + else _UserBatchDeletion(removals=MappingProxyType({}), deleted_keys=(), jwt_mapping_cache_keys=()) ) def result(index: int, user_id: str) -> UserDeleteResult: diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index 85fbd0acd91..e1a840b1239 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -94,6 +94,10 @@ - {id: mgmt.mcp_toolset.update.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:3098", rationale: "Narrowing the tools to one entry reads back exactly that entry"} - {id: mgmt.mcp_toolset.update.clear_persists, module: mgmt, tier: P0, surface: api, assertions: [clear_persists], source: "mcp_management_endpoints.py:3098", fail_before_fix: proven, rationale: "An explicit null clears the stored description; the update used to drop null and keep the old value"} - {id: mgmt.mcp_toolset.delete.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:3149", rationale: "A deleted toolset is gone by id and from the list on every replica"} +- {id: mgmt.key.delete.audit_logged, module: mgmt, tier: P0, surface: api, assertions: [audit_logged], source: "key_management_endpoints.py:3981", rationale: "Every hard key deletion writes one LiteLLM_VerificationToken deleted audit row, whether the key is addressed by key or by alias"} +- {id: mgmt.team.member_delete.audit_logs_keys, module: mgmt, tier: P0, surface: api, assertions: [audit_logs_keys], source: "team_endpoints.py:3563", rationale: "Removing a team member hard-deletes their keys and each deleted key writes a deleted audit row"} +- {id: mgmt.team.delete.audit_logs_keys, module: mgmt, tier: P0, surface: api, assertions: [audit_logs_keys], source: "team_endpoints.py:4344", rationale: "Deleting a team hard-deletes its keys and each deleted key writes a deleted audit row"} +- {id: mgmt.user.delete.audit_logs_keys, module: mgmt, tier: P0, surface: api, assertions: [audit_logs_keys], source: "internal_user_endpoints.py:2369", rationale: "Deleting a user hard-deletes their keys and each deleted key writes a deleted audit row"} - {id: mgmt.user.jwt.database_roles, module: mgmt, tier: P0, surface: api, assertions: [database_roles], source: "auth/handle_jwt.py", rationale: "User-only JWT subjects retain their seeded database roles and memberships"} - {id: mgmt.key.jwt.viewer_denied, module: mgmt, tier: P0, surface: api, assertions: [viewer_denied], source: "auth/route_checks.py", rationale: "An admin viewer can read a key but cannot update it or change stored state"} diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py index 8470d318db8..7366695c0d1 100644 --- a/tests/e2e/management/management_client.py +++ b/tests/e2e/management/management_client.py @@ -25,6 +25,8 @@ from e2e_http import ( unwrap, ) from models import ( + AuditLogPage, + AuditLogParams, ChatBody, ChatMessage, ConnectionTestBody, @@ -35,6 +37,7 @@ from models import ( CustomerResponse, KeyBlockBody, KeyDeleteBody, + KeyDeleteByAliasBody, KeyGenerateBody, KeyGenerateResponse, KeyInfoParams, @@ -159,6 +162,31 @@ class ManagementClient: def update_key_models(self, key: str, models: list[str]) -> None: _ = unwrap(self.update_key(KeyUpdateBody(key=key, models=models))) + def delete_key_by_alias(self, key_alias: str) -> None: + _ = unwrap( + self.proxy.transport.post( + "/key/delete", + headers=self.proxy.management_headers(), + json=KeyDeleteByAliasBody(key_aliases=[key_alias]), + response_type=NoBody, + ) + ) + + def key_deleted_audit_logs(self, token_hash: str) -> AuditLogPage: + return unwrap( + self.proxy.transport.get( + "/audit", + headers=self.proxy.management_headers(), + params=AuditLogParams( + object_id=token_hash, + action="deleted", + table_name="LiteLLM_VerificationToken", + page_size=100, + ), + response_type=AuditLogPage, + ) + ) + def key_info_as(self, key: str, *, caller_key: str | None = None) -> Result[KeyInfoResponse]: return self.proxy.transport.get( "/key/info", diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index 476165b715d..da0fc37aff8 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -16,8 +16,8 @@ from typing import Final import pytest -from e2e_config import UI_PASSWORD, UI_USERNAME, unique_marker -from e2e_http import StreamingResponse, Success +from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, UI_PASSWORD, UI_USERNAME, unique_marker +from e2e_http import StreamingResponse, Success, unwrap from lifecycle import ResourceManager from management_client import ( DASHBOARD_SESSION_TEAM_ID, @@ -26,7 +26,9 @@ from management_client import ( ManagementClient, ) from models import ( + AuditLogPage, KeyGenerateBody, + KeyGenerateResponse, KeyUpdateBody, LiteLLMParamsBody, ModelInfoEntry, @@ -40,6 +42,7 @@ from models import ( UserNewBody, UserUpdateBody, ) +from proxy_client import Converged, await_converged pytestmark = pytest.mark.e2e @@ -801,3 +804,128 @@ class TestCustomer: assert info.user_id == customer, ( f"/customer/info did not report the created end-user; got {info.user_id!r}" ) + + +def _await_deleted_audit_rows(client: ManagementClient, token_hash: str) -> AuditLogPage: + outcome = await_converged( + lambda: client.key_deleted_audit_logs(token_hash), + converged=lambda page: page.total >= 1, + timeout=POLL_TIMEOUT, + interval=POLL_INTERVAL, + now=time.monotonic, + sleep=time.sleep, + ) + return outcome.result if isinstance(outcome, Converged) else outcome.last_result + + +def _assert_single_deleted_row(page: AuditLogPage, token_hash: str) -> None: + assert page.total == 1, page + row = page.audit_logs[0] + assert row.action == "deleted", row + assert row.table_name == "LiteLLM_VerificationToken", row + assert row.object_id == token_hash, row + assert row.changed_by, row + + +def _assert_key_deleted(client: ManagementClient, key: str) -> None: + def gone() -> bool | None: + match client.key_info_as(key): + case Success(data=response): + return True if response.info.status == "deleted" else None + case _: + return True + + _ = _poll( + client, + gone, + "/key/info never reported status 'deleted' for a key whose deletion returned", + ) + + +def _token_of(created: KeyGenerateResponse) -> str: + assert created.token is not None, created + return created.token + + +def _generate_response( + client: ManagementClient, resources: ResourceManager, body: KeyGenerateBody +) -> KeyGenerateResponse: + created = unwrap(client.generate_key(body)) + resources.defer(lambda: client.delete_key_strict(created.key, missing_ok=True)) + return created + + +class TestKeyDeletionAuditLog: + @pytest.mark.covers("mgmt.key.delete.audit_logged") + def test_key_delete_by_key_writes_audit_row( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + created = _generate_response(client, resources, KeyGenerateBody(key_alias=f"e2e-audit-{unique_marker()}")) + token = _token_of(created) + + client.delete_key_strict(created.key) + + _assert_key_deleted(client, created.key) + _assert_single_deleted_row(_await_deleted_audit_rows(client, token), token) + + @pytest.mark.covers("mgmt.key.delete.audit_logged") + def test_key_delete_by_alias_writes_audit_row( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + alias = f"e2e-audit-{unique_marker()}" + created = _generate_response(client, resources, KeyGenerateBody(key_alias=alias)) + token = _token_of(created) + + client.delete_key_by_alias(alias) + + _assert_key_deleted(client, created.key) + _assert_single_deleted_row(_await_deleted_audit_rows(client, token), token) + + @pytest.mark.covers("mgmt.team.member_delete.audit_logs_keys") + def test_team_member_delete_writes_audit_row_for_member_keys( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + team_id = _create_team(client, resources, f"e2e-audit-team-{unique_marker()}", []) + user_id = _create_user( + client, + resources, + UserNewBody(user_email=f"e2e-audit-{unique_marker()}@example.com", user_role="internal_user"), + ) + client.add_team_member(team_id, user_id) + created = _generate_response(client, resources, KeyGenerateBody(user_id=user_id, team_id=team_id)) + token = _token_of(created) + + client.delete_team_member(team_id, user_id) + + _assert_key_deleted(client, created.key) + _assert_single_deleted_row(_await_deleted_audit_rows(client, token), token) + + @pytest.mark.covers("mgmt.team.delete.audit_logs_keys") + def test_team_delete_writes_audit_row_for_team_keys( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + team_id = _create_team(client, resources, f"e2e-audit-team-{unique_marker()}", []) + created = _generate_response(client, resources, KeyGenerateBody(team_id=team_id)) + token = _token_of(created) + + client.delete_team(team_id) + + _assert_key_deleted(client, created.key) + _assert_single_deleted_row(_await_deleted_audit_rows(client, token), token) + + @pytest.mark.covers("mgmt.user.delete.audit_logs_keys") + def test_user_delete_writes_audit_row_for_user_keys( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + user_id = _create_user( + client, + resources, + UserNewBody(user_email=f"e2e-audit-{unique_marker()}@example.com", user_role="internal_user"), + ) + created = _generate_response(client, resources, KeyGenerateBody(user_id=user_id)) + token = _token_of(created) + + client.delete_user_strict(user_id) + + _assert_key_deleted(client, created.key) + _assert_single_deleted_row(_await_deleted_audit_rows(client, token), token) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 5278dbd287a..f9c17314477 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -124,6 +124,32 @@ class KeyDeleteBody(BaseModel): keys: list[str] +class KeyDeleteByAliasBody(BaseModel): + key_aliases: list[str] + + +class AuditLogParams(BaseModel): + object_id: str + action: str + table_name: str + page_size: int + + +class AuditLogEntry(BaseModel): + id: str + changed_by: str | None = None + changed_by_api_key: str | None = None + action: str + table_name: str + object_id: str + before_value: object | None = None + + +class AuditLogPage(BaseModel): + audit_logs: list[AuditLogEntry] + total: int + + class KeyInfoParams(BaseModel): key: str diff --git a/tests/e2e/transport.py b/tests/e2e/transport.py index a3eec815441..4f3953f68b2 100644 --- a/tests/e2e/transport.py +++ b/tests/e2e/transport.py @@ -313,6 +313,8 @@ CONTROL_PLANE_PREFIXES: tuple[str, ...] = ( "/config", "/guardrails", "/router/settings", + "/audit", + "/v2/login", "/openapi.json", ) diff --git a/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py b/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py index 76027d6b7e2..7a9d155cd67 100644 --- a/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py +++ b/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py @@ -701,3 +701,49 @@ class TestKeyUpdatedAuditLogObjectId: assert updated_values["project_id"] is None assert json.loads(audit_row.before_value)["project_id"] == "project-orbit" assert updated_values["max_budget"] == 2000.0 + + +@pytest.mark.asyncio +async def test_key_deleted_hook_writes_audit_log_for_alias_deletion(): + from litellm.proxy._types import ( + KeyRequest, + LiteLLM_AuditLogs, + LiteLLM_VerificationToken, + LitellmTableNames, + UserAPIKeyAuth, + ) + + captured: Final[list[LiteLLM_AuditLogs]] = [] + + async def capture_audit_log(request_data: LiteLLM_AuditLogs) -> None: + captured.append(request_data) + + with ( + patch("litellm.store_audit_logs", True), + patch( + "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + new=capture_audit_log, + ), + patch.object( + KeyManagementEventHooks, + "_delete_virtual_keys_from_secret_manager", + new_callable=AsyncMock, + ), + ): + await KeyManagementEventHooks.async_key_deleted_hook( + data=KeyRequest(key_aliases=["a"]), + keys_being_deleted=[LiteLLM_VerificationToken(token="hashed", key_alias="a")], + response={}, + user_api_key_dict=UserAPIKeyAuth(user_id="admin", token="callertok"), + ) + for _ in range(100): + if captured: + break + await asyncio.sleep(0.01) + + assert len(captured) == 1 + audit_row = captured[0] + assert audit_row.action == "deleted" + assert audit_row.object_id == "hashed" + assert audit_row.table_name == LitellmTableNames.KEY_TABLE_NAME + assert audit_row.changed_by == "admin" diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index f684e2040bd..2d1049b143e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -1,3 +1,4 @@ +import asyncio import hashlib import json import logging @@ -4779,3 +4780,62 @@ def test_user_routes_answer_503_no_db_connection_when_the_callers_user_read_hits assert response.status_code == 503, response.text assert response.json() == _DB_OUTAGE_503_BODY + + +@pytest.mark.asyncio +async def test_delete_user_writes_deleted_audit_log_for_user_keys(mocker): + from litellm.proxy._types import ( + DeleteUserRequest, + LiteLLM_VerificationToken, + LitellmTableNames, + UserAPIKeyAuth, + ) + from litellm.proxy.management_endpoints.internal_user_endpoints import delete_user + + mock_prisma_client = mocker.MagicMock() + + mock_user_row = mocker.MagicMock() + mock_user_row.user_id = "doomed-user" + mock_user_row.user_email = "doomed@example.com" + mock_user_row.teams = [] + mock_user_row.model_dump_json.return_value = "{}" + mock_user_row.model_dump.return_value = {"user_id": "doomed-user", "user_email": "doomed@example.com", "teams": []} + + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(return_value=mock_user_row) + mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock(return_value=[]) + + user_key = LiteLLM_VerificationToken(token="hashed-user-key", user_id="doomed-user") + mock_prisma_client.db.litellm_verificationtoken.find_many = mocker.AsyncMock(return_value=[user_key]) + mock_prisma_client.db.litellm_verificationtoken.delete_many = mocker.AsyncMock(return_value=1) + mock_prisma_client.db.litellm_invitationlink.delete_many = mocker.AsyncMock(return_value=0) + mock_prisma_client.db.litellm_organizationmembership.delete_many = mocker.AsyncMock(return_value=0) + mock_prisma_client.db.litellm_teammembership.delete_many = mocker.AsyncMock(return_value=0) + mock_prisma_client.db.litellm_jwtkeymapping.find_many = mocker.AsyncMock(return_value=[]) + mock_prisma_client.db.litellm_usertable.delete_many = mocker.AsyncMock(return_value=1) + + mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mocker.patch("litellm.store_audit_logs", True) + captured: Final[list] = [] + + async def _capture(request_data): + captured.append(request_data) + + mocker.patch( + "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + new=_capture, + ) + + caller = UserAPIKeyAuth(user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN) + await delete_user(data=DeleteUserRequest(user_ids=["doomed-user"]), user_api_key_dict=caller) + for _ in range(100): + if captured: + break + await asyncio.sleep(0.01) + + key_rows: Final = [r for r in captured if r.table_name == LitellmTableNames.KEY_TABLE_NAME] + assert len(key_rows) == 1 + audit_row: Final = key_rows[0] + assert audit_row.action == "deleted" + assert audit_row.object_id == user_key.token + assert audit_row.changed_by + assert json.loads(audit_row.before_value)["token"] == user_key.token diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 8bf9c598b1a..7a9b6b66946 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -5129,6 +5129,168 @@ async def test_team_member_delete_cleans_verification_tokens( ) +@pytest.mark.asyncio +async def test_team_member_delete_writes_deleted_audit_log_for_member_keys( + mock_db_client, mock_admin_auth +): + from litellm.proxy._types import ( + LiteLLM_VerificationToken, + LitellmTableNames, + TeamMemberDeleteRequest, + ) + from litellm.proxy.management_endpoints.team_endpoints import team_member_delete + + test_team_id = "team-del-audit-123" + test_user_id = "user-audit@example.com" + member_key = LiteLLM_VerificationToken(token="hashed-member-key", team_id=test_team_id, user_id=test_user_id) + + mock_team_row = MagicMock() + mock_team_row.model_dump.return_value = { + "team_id": test_team_id, + "members_with_roles": [ + {"user_id": test_user_id, "user_email": None, "role": "user"} + ], + "team_member_permissions": [], + "metadata": {}, + "models": [], + "spend": 0.0, + } + + mock_db_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=mock_team_row + ) + mock_db_client.db.litellm_teamtable.update = AsyncMock(return_value=mock_team_row) + + mock_user_row = MagicMock() + mock_user_row.user_id = test_user_id + mock_user_row.teams = [test_team_id] + mock_db_client.db.litellm_usertable.find_many = AsyncMock( + return_value=[mock_user_row] + ) + mock_db_client.db.litellm_usertable.update = AsyncMock(return_value=MagicMock()) + + mock_db_client.db.litellm_teammembership = MagicMock() + mock_db_client.db.litellm_teammembership.delete_many = AsyncMock( + return_value=MagicMock() + ) + + mock_db_client.db.litellm_verificationtoken = MagicMock() + mock_db_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[member_key]) + mock_db_client.db.litellm_verificationtoken.delete_many = AsyncMock( + return_value=MagicMock() + ) + mock_db_client.db.litellm_jwtkeymapping = MagicMock() + mock_db_client.db.litellm_jwtkeymapping.find_many = AsyncMock(return_value=[]) + mock_db_client.db.litellm_deletedverificationtoken = MagicMock() + mock_db_client.db.litellm_deletedverificationtoken.create_many = AsyncMock(return_value=MagicMock()) + + _wire_member_delete_tx(mock_db_client) + + captured: Final[list] = [] + + async def _capture(request_data): + captured.append(request_data) + + with ( + patch("litellm.store_audit_logs", True), + patch( + "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + new=_capture, + ), + ): + await team_member_delete( + data=TeamMemberDeleteRequest(team_id=test_team_id, user_id=test_user_id), + user_api_key_dict=mock_admin_auth, + ) + for _ in range(100): + if captured: + break + await asyncio.sleep(0.01) + + key_rows: Final = [r for r in captured if r.table_name == LitellmTableNames.KEY_TABLE_NAME] + assert len(key_rows) == 1 + audit_row: Final = key_rows[0] + assert audit_row.action == "deleted" + assert audit_row.object_id == member_key.token + assert audit_row.changed_by + assert json.loads(audit_row.before_value)["token"] == member_key.token + + +@pytest.mark.asyncio +async def test_delete_team_writes_deleted_audit_log_for_team_keys( + monkeypatch, +): + from litellm.proxy._types import DeleteTeamRequest, LiteLLM_VerificationToken, LitellmTableNames + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + team = LiteLLM_TeamTable( + team_id="team-doomed", + team_alias="doomed-team", + members_with_roles=[], + metadata={}, + model_max_budget={}, + model_spend={}, + ) + team_key = LiteLLM_VerificationToken(token="hashed-doomed-key", team_id="team-doomed") + + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=team) + mock_prisma_client.delete_data = AsyncMock(return_value={"deleted_teams": ["team-doomed"]}) + mock_prisma_client.db.litellm_deletedteamtable.create_many = AsyncMock() + mock_prisma_client.db.litellm_deletedverificationtoken.create_many = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[team_key]) + mock_prisma_client.db.execute_raw = AsyncMock() + mock_prisma_client.db.litellm_teammembership.delete_many = AsyncMock() + mock_prisma_client.get_data = AsyncMock(return_value=None) + + mock_tx = AsyncMock() + mock_tx.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + mock_tx_cm = MagicMock() + mock_tx_cm.__aenter__ = AsyncMock(return_value=mock_tx) + mock_tx_cm.__aexit__ = AsyncMock(return_value=False) + mock_prisma_client.db.tx = MagicMock(return_value=mock_tx_cm) + _wire_team_delete_tx(mock_prisma_client) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", UserApiKeyCache()) + monkeypatch.setattr("litellm.proxy.proxy_server.create_audit_log_for_update", AsyncMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") + + caller = UserAPIKeyAuth( + user_id="admin-user", + api_key="sk-admin", + user_role=LitellmUserRoles.PROXY_ADMIN.value, + ) + captured: Final[list] = [] + + async def _capture(request_data): + captured.append(request_data) + + monkeypatch.setattr("litellm.store_audit_logs", True) + monkeypatch.setattr( + "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + _capture, + ) + await delete_team( + data=DeleteTeamRequest(team_ids=["team-doomed"]), + http_request=MagicMock(), + user_api_key_dict=caller, + litellm_changed_by="admin-user", + ) + for _ in range(100): + if captured: + break + await asyncio.sleep(0.01) + + key_rows: Final = [r for r in captured if r.table_name == LitellmTableNames.KEY_TABLE_NAME] + assert len(key_rows) == 1 + audit_row: Final = key_rows[0] + assert audit_row.action == "deleted" + assert audit_row.object_id == team_key.token + assert audit_row.changed_by + assert json.loads(audit_row.before_value)["token"] == team_key.token + + @pytest.mark.asyncio async def test_team_member_delete_reads_on_the_lock_holding_transaction( mock_db_client, mock_admin_auth diff --git a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py index 3c05068c4b0..32e5bea613c 100644 --- a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py +++ b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py @@ -1,3 +1,4 @@ +import asyncio import copy import json from collections.abc import Callable, Mapping, Sequence @@ -7,7 +8,7 @@ from typing import Final import pytest from pydantic import BaseModel, ConfigDict, ValidationError -from litellm.proxy._types import LiteLLM_TeamTable, LitellmUserRoles, Member, UserAPIKeyAuth +from litellm.proxy._types import LiteLLM_TeamTable, LitellmTableNames, LitellmUserRoles, Member, UserAPIKeyAuth from litellm.proxy.auth.auth_checks import jwt_key_mapping_cache_key from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.list_api.common import ManagementProblem @@ -748,3 +749,96 @@ def test_request_models_reject_unknown_fields(): BulkTeamMemberDeleteRequest.model_validate({"members": [{"user_id": "u1", "role": "admin"}]}) with pytest.raises(ValidationError, match="dry_run"): BulkDeleteUserRequest.model_validate({"user_ids": ["u1"], "dry_run": True}) + + +@pytest.mark.asyncio +async def test_bulk_delete_writes_deleted_audit_log_for_deleted_keys(mocker): + prisma = _FakePrisma( + users=[_user("u1", "t1"), _user("keep", "t1")], + teams=[_team("t1", "u1", "keep")], + tokens=[ + {"token": "team-key", "user_id": "u1", "team_id": "t1"}, + {"token": "personal-key", "user_id": "u1"}, + {"token": "keep-key", "user_id": "keep", "team_id": "t1"}, + ], + ) + + captured: Final[list] = [] + + async def _capture(request_data): + captured.append(request_data) + + mocker.patch("litellm.store_audit_logs", True) + mocker.patch( + "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + new=_capture, + ) + await _delete(prisma, ["u1"]) + for _ in range(100): + if len([r for r in captured if r.table_name == LitellmTableNames.KEY_TABLE_NAME]) >= 2: + break + await asyncio.sleep(0.01) + + key_rows: Final = [r for r in captured if r.table_name == LitellmTableNames.KEY_TABLE_NAME] + assert {r.object_id for r in key_rows} == {"team-key", "personal-key"} + assert {r.action for r in key_rows} == {"deleted"} + assert all(r.changed_by for r in key_rows) + assert {json.loads(r.before_value)["token"] for r in key_rows} == {"team-key", "personal-key"} + + +@pytest.mark.asyncio +async def test_bulk_member_delete_writes_deleted_audit_log_for_removed_team_keys(mocker): + prisma = _FakePrisma( + users=[_user("u1", "t1"), _user("keep", "t1")], + teams=[_team("t1", "u1", "keep")], + tokens=[ + {"token": "team-key", "user_id": "u1", "team_id": "t1"}, + {"token": "keep-key", "user_id": "keep", "team_id": "t1"}, + ], + ) + + captured: Final[list] = [] + + async def _capture(request_data): + captured.append(request_data) + + mocker.patch("litellm.store_audit_logs", True) + mocker.patch( + "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + new=_capture, + ) + await _remove(prisma, "t1", [{"user_id": "u1"}]) + for _ in range(100): + if captured: + break + await asyncio.sleep(0.01) + + key_rows: Final = [r for r in captured if r.table_name == LitellmTableNames.KEY_TABLE_NAME] + assert len(key_rows) == 1 + audit_row: Final = key_rows[0] + assert audit_row.action == "deleted" + assert audit_row.object_id == "team-key" + assert audit_row.changed_by + assert json.loads(audit_row.before_value)["token"] == "team-key" + + +@pytest.mark.asyncio +async def test_bulk_delete_skips_the_key_audit_log_when_the_tx_rolls_back(mocker): + prisma = _FakePrisma( + users=[_user("u1", "a-good", "z-bad"), _user("u2", "a-good")], + teams=[_team("a-good", "u1", "u2"), _team("z-bad", "u1")], + tokens=[{"token": "k1", "user_id": "u1", "team_id": "a-good"}], + fail_locks=frozenset({"z-bad"}), + ) + cache = _cache_with("k1") + + mocker.patch("litellm.store_audit_logs", True) + mock_audit_write = mocker.patch( + "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + new=mocker.AsyncMock(), + ) + results = await _delete(prisma, ["u1", "u2"], cache=cache) + + assert [(r.user_id, r.success) for r in results] == [("u1", False), ("u2", False)] + assert [t["token"] for t in prisma.db.litellm_verificationtoken.rows] == ["k1"] + mock_audit_write.assert_not_called() From 85ed18e8f2e5b5e6b51be33c9e6cab2b5e323024 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 19:02:43 +0000 Subject: [PATCH 12/48] fix(rust_bridge): keep the Messages route on Python until the Rust path is ready (#42517) * fix(rust_bridge): keep the Messages route on Python until the Rust path is ready Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(rust_bridge): keep token counter and tokenizer routes on Python Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: ryan Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/rust_bridge/catalog.py | 6 +++--- tests/test_litellm/rust_bridge/test_catalog.py | 4 ---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index d7479631e04..d6cbe17d790 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -93,9 +93,9 @@ Rules: TypeAlias = tuple[Rule, ...] RULES: Final[Rules] = ( RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})), RouteRule(Route.OCR, Rollout.RUST_OPT_OUT), - RouteRule(Route.MESSAGES, Rollout.RUST_OPT_IN), - RouteRule(Route.TOKEN_COUNTER, Rollout.RUST_OPT_IN), - RouteRule(Route.TOKENIZER, Rollout.RUST_OPT_IN), + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY), + RouteRule(Route.TOKEN_COUNTER, Rollout.PYTHON_ONLY), + RouteRule(Route.TOKENIZER, Rollout.PYTHON_ONLY), RouteRule(Route.TRANSCRIPTION, Rollout.RUST_REQUIRED, providers=frozenset({"bedrock"})), CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.LOCAL})), CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.REDIS})), diff --git a/tests/test_litellm/rust_bridge/test_catalog.py b/tests/test_litellm/rust_bridge/test_catalog.py index 82e3766e8a2..b6b486e73f0 100644 --- a/tests/test_litellm/rust_bridge/test_catalog.py +++ b/tests/test_litellm/rust_bridge/test_catalog.py @@ -53,10 +53,6 @@ def test_shipped_decisions( enabled: Final = environment == "1" if environment is not None else process is not False assert catalog.rollout(context) is Rollout.RUST_OPT_OUT assert catalog.decision(context) is (Decision.RUST_WITH_FALLBACK if enabled else Decision.PYTHON) - elif route in (Route.MESSAGES, Route.TOKEN_COUNTER, Route.TOKENIZER): - enabled: Final = environment == "1" if environment is not None else process is True - assert catalog.rollout(context) is Rollout.RUST_OPT_IN - assert catalog.decision(context) is (Decision.RUST_WITH_FALLBACK if enabled else Decision.PYTHON) elif route is Route.TRANSCRIPTION and provider == "bedrock": assert catalog.rollout(context) is Rollout.RUST_REQUIRED assert catalog.decision(context) is Decision.RUST_REQUIRED From 9d299d016a6044cf22474764faa7288c4eab54fa Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 19:03:40 +0000 Subject: [PATCH 13/48] chore(pricing): remove retired models flagged by the provider sync (#42521) Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 441 ------------------ model_prices_and_context_window.json | 441 ------------------ 2 files changed, 882 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8f9958e0031..0e80d4732c8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -578,17 +578,6 @@ "supports_vision": true, "supports_tool_choice": true }, - "amazon.nova-sonic-v1:0": { - "deprecation_date": "2026-09-14", - "input_cost_per_audio_token": 3.4e-06, - "input_cost_per_token": 6e-08, - "litellm_provider": "bedrock", - "mode": "realtime", - "output_cost_per_audio_token": 1.36e-05, - "output_cost_per_token": 2.4e-07, - "supports_audio_input": true, - "supports_audio_output": true - }, "amazon.nova-2-sonic-v1:0": { "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 3.3e-07, @@ -4685,21 +4674,6 @@ "supports_prompt_caching": true, "supports_vision": false }, - "azure/eu/o1-preview-2024-09-12": { - "cache_read_input_token_cost": 8.25e-06, - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_vision": false - }, "azure/eu/o3-mini-2025-01-31": { "cache_read_input_token_cost": 6.05e-07, "deprecation_date": "2026-11-19", @@ -9819,39 +9793,6 @@ "supports_reasoning": true, "supports_vision": false }, - "azure/o1-preview": { - "cache_read_input_token_cost": 7.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_vision": false - }, - "azure/o1-preview-2024-09-12": { - "cache_read_input_token_cost": 7.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_vision": false - }, "azure/o3": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5e-07, @@ -10720,21 +10661,6 @@ "supports_prompt_caching": true, "supports_vision": false }, - "azure/us/o1-preview-2024-09-12": { - "cache_read_input_token_cost": 8.25e-06, - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_vision": false - }, "azure/us/o3-2025-04-16": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, @@ -15609,28 +15535,6 @@ "output_cost_per_token": 6e-07, "supports_tool_choice": true }, - "cohere.command-r-plus-v1:0": { - "deprecation_date": "2026-08-19", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_tool_choice": true - }, - "cohere.command-r-v1:0": { - "deprecation_date": "2026-08-19", - "input_cost_per_token": 5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_tool_choice": true - }, "cohere.command-text-v14": { "input_cost_per_token": 1.5e-06, "litellm_provider": "bedrock", @@ -42975,32 +42879,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/anthropic/claude-opus-4": { - "input_cost_per_image": 0.0048, - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_pdf_input": true, - "supports_response_schema": false, - "supports_web_search": true - }, "openrouter/anthropic/claude-opus-4.1": { "input_cost_per_image": 0.0048, "cache_creation_input_token_cost": 1.875e-05, @@ -43928,27 +43806,6 @@ "supports_vision": true, "supports_web_search": false }, - "openrouter/mistralai/mistral-large-2512": { - "cache_read_input_token_cost": 5.5e-08, - "input_cost_per_image": 0, - "input_cost_per_token": 5.5e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 209715, - "max_tokens": 209715, - "mode": "chat", - "output_cost_per_token": 1.65e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": false, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, "openrouter/mistralai/mistral-7b-instruct": { "input_cost_per_token": 1.3e-07, "litellm_provider": "openrouter", @@ -48806,22 +48663,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "us.amazon.nova-premier-v1:0": { - "deprecation_date": "2026-09-14", - "input_cost_per_token": 2.5e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 1000000, - "max_output_tokens": 10000, - "max_tokens": 10000, - "mode": "chat", - "output_cost_per_token": 1.25e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_vision": true, - "cache_read_input_token_cost": 6.25e-07 - }, "us.amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 8e-07, @@ -71275,15 +71116,6 @@ "output_cost_per_token": 1.6e-06, "source": "https://api.together.ai/v1/models" }, - "vertex_ai/gemini-2.5-flash-native-audio": { - "input_cost_per_audio_token": 3e-06, - "input_cost_per_token": 5e-07, - "litellm_provider": "vertex_ai", - "mode": "realtime", - "output_cost_per_audio_token": 1.2e-05, - "output_cost_per_token": 2e-06, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" - }, "vertex_ai/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, @@ -71293,16 +71125,6 @@ "output_cost_per_token": 1e-05, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, - "vertex_ai/gemini-3.1-flash-live-preview": { - "input_cost_per_audio_token": 3e-06, - "input_cost_per_second": 8.33333333333e-05, - "input_cost_per_token": 7.5e-07, - "litellm_provider": "vertex_ai", - "mode": "realtime", - "output_cost_per_audio_token": 1.2e-05, - "output_cost_per_token": 4.5e-06, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" - }, "vertex_ai/gemini-3.1-flash-tts-preview": { "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, @@ -71335,16 +71157,6 @@ "output_cost_per_token": 9e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, - "vertex_ai/gemini-robotics-er-2": { - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, - "litellm_provider": "vertex_ai", - "mode": "chat", - "output_cost_per_token": 5e-06, - "output_cost_per_token_batches": 2.5e-06, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" - }, "vertex_ai/gemma-4-26b-a4b-it": { "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, @@ -71754,14 +71566,6 @@ "output_cost_per_token_batches": 2.42e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/eu/o1-preview": { - "cache_read_input_token_cost": 8.25e-06, - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/eu/o3-2025-04-16": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, @@ -72103,14 +71907,6 @@ "output_cost_per_token_batches": 2.42e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/us/o1-preview": { - "cache_read_input_token_cost": 8.25e-06, - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/us/o3-deep-research": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 2.75e-06, @@ -74456,85 +74252,6 @@ "supports_vision": false, "supports_web_search": false }, - "openrouter/deepseek/deepseek-v4-flash-0731:batch": { - "cache_read_input_token_cost": 3.5e-09, - "input_cost_per_token": 1.1e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, - "mode": "chat", - "output_cost_per_token": 3.3e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, - "openrouter/deepseek/deepseek-v4-flash-0731:free": { - "input_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 393216, - "max_tokens": 393216, - "mode": "chat", - "output_cost_per_token": 0.0, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": false, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, - "openrouter/deepseek/deepseek-v4-flash-vision-exp:batch": { - "cache_read_input_token_cost": 3.5e-09, - "input_cost_per_token": 1.1e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, - "mode": "chat", - "output_cost_per_token": 3.3e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, - "openrouter/deepseek/deepseek-v4-pro-0813:batch": { - "cache_read_input_token_cost": 2.2e-08, - "input_cost_per_token": 6.6e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, - "mode": "chat", - "output_cost_per_token": 1.98e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/dots-studio/dots-3-note-preview:free": { "deprecation_date": "2026-12-31", "input_cost_per_token": 0.0, @@ -75050,26 +74767,6 @@ "supports_vision": false, "supports_web_search": false }, - "openrouter/kwaipilot/kat-coder-pro-v2": { - "cache_read_input_token_cost": 6e-08, - "input_cost_per_token": 3e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 144000, - "max_tokens": 144000, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": false, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/kwaipilot/kat-coder-pro-v2.5": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 7.4e-07, @@ -75149,26 +74846,6 @@ "supports_vision": true, "supports_web_search": false }, - "openrouter/meta/muse-glimmer-30b:batch": { - "cache_read_input_token_cost": 2e-08, - "input_cost_per_token": 1.75e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 131072, - "max_output_tokens": 117964, - "max_tokens": 117964, - "mode": "chat", - "output_cost_per_token": 7.5e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, "openrouter/meta/muse-spark-1.1": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.25e-06, @@ -75307,26 +74984,6 @@ "supports_vision": false, "supports_web_search": false }, - "openrouter/minimax/minimax-m3:batch": { - "cache_read_input_token_cost": 6e-08, - "input_cost_per_token": 3e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 524288, - "max_output_tokens": 471859, - "max_tokens": 471859, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, "openrouter/mistralai/codestral-2508:batch": { "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, @@ -76276,25 +75933,6 @@ "supports_vision": true, "supports_web_search": true }, - "openrouter/openai/gpt-oss-120b:batch": { - "input_cost_per_token": 1.5e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 131072, - "max_output_tokens": 117964, - "max_tokens": 117964, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": false, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/openai/o3-mini:batch": { "cache_read_input_token_cost": 2.75e-07, "input_cost_per_token": 5.5e-07, @@ -76469,45 +76107,6 @@ "supports_vision": true, "supports_web_search": true }, - "openrouter/qwen/qwen3.5-9b:batch": { - "input_cost_per_token": 1.7e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 235929, - "max_tokens": 235929, - "mode": "chat", - "output_cost_per_token": 2.5e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": false, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, - "openrouter/qwen/qwen3.8-2.4t-a95b:batch": { - "cache_read_input_token_cost": 2.5e-07, - "input_cost_per_token": 2e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 1010000, - "max_output_tokens": 909000, - "max_tokens": 909000, - "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/qwen/qwen3.8-27b:free": { "input_cost_per_token": 0.0, "litellm_provider": "openrouter", @@ -77040,26 +76639,6 @@ "supports_vision": true, "supports_web_search": false }, - "openrouter/thinkingmachines/inkling:batch": { - "cache_read_input_token_cost": 1.7e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 524288, - "max_output_tokens": 471859, - "max_tokens": 471859, - "mode": "chat", - "output_cost_per_token": 4.05e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, "openrouter/thinkingmachines/inkling:free": { "input_cost_per_token": 0.0, "litellm_provider": "openrouter", @@ -77181,26 +76760,6 @@ "supports_vision": true, "supports_web_search": true }, - "openrouter/z-ai/glm-5.2:batch": { - "cache_read_input_token_cost": 7e-08, - "input_cost_per_token": 7e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, - "mode": "chat", - "output_cost_per_token": 2.2e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/z-ai/glm-5.3-flash:batch": { "cache_read_input_token_cost": 1.2e-08, "input_cost_per_token": 6e-08, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8f9958e0031..0e80d4732c8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -578,17 +578,6 @@ "supports_vision": true, "supports_tool_choice": true }, - "amazon.nova-sonic-v1:0": { - "deprecation_date": "2026-09-14", - "input_cost_per_audio_token": 3.4e-06, - "input_cost_per_token": 6e-08, - "litellm_provider": "bedrock", - "mode": "realtime", - "output_cost_per_audio_token": 1.36e-05, - "output_cost_per_token": 2.4e-07, - "supports_audio_input": true, - "supports_audio_output": true - }, "amazon.nova-2-sonic-v1:0": { "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 3.3e-07, @@ -4685,21 +4674,6 @@ "supports_prompt_caching": true, "supports_vision": false }, - "azure/eu/o1-preview-2024-09-12": { - "cache_read_input_token_cost": 8.25e-06, - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_vision": false - }, "azure/eu/o3-mini-2025-01-31": { "cache_read_input_token_cost": 6.05e-07, "deprecation_date": "2026-11-19", @@ -9819,39 +9793,6 @@ "supports_reasoning": true, "supports_vision": false }, - "azure/o1-preview": { - "cache_read_input_token_cost": 7.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_vision": false - }, - "azure/o1-preview-2024-09-12": { - "cache_read_input_token_cost": 7.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_vision": false - }, "azure/o3": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5e-07, @@ -10720,21 +10661,6 @@ "supports_prompt_caching": true, "supports_vision": false }, - "azure/us/o1-preview-2024-09-12": { - "cache_read_input_token_cost": 8.25e-06, - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_vision": false - }, "azure/us/o3-2025-04-16": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, @@ -15609,28 +15535,6 @@ "output_cost_per_token": 6e-07, "supports_tool_choice": true }, - "cohere.command-r-plus-v1:0": { - "deprecation_date": "2026-08-19", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_tool_choice": true - }, - "cohere.command-r-v1:0": { - "deprecation_date": "2026-08-19", - "input_cost_per_token": 5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_tool_choice": true - }, "cohere.command-text-v14": { "input_cost_per_token": 1.5e-06, "litellm_provider": "bedrock", @@ -42975,32 +42879,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/anthropic/claude-opus-4": { - "input_cost_per_image": 0.0048, - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_pdf_input": true, - "supports_response_schema": false, - "supports_web_search": true - }, "openrouter/anthropic/claude-opus-4.1": { "input_cost_per_image": 0.0048, "cache_creation_input_token_cost": 1.875e-05, @@ -43928,27 +43806,6 @@ "supports_vision": true, "supports_web_search": false }, - "openrouter/mistralai/mistral-large-2512": { - "cache_read_input_token_cost": 5.5e-08, - "input_cost_per_image": 0, - "input_cost_per_token": 5.5e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 209715, - "max_tokens": 209715, - "mode": "chat", - "output_cost_per_token": 1.65e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": false, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, "openrouter/mistralai/mistral-7b-instruct": { "input_cost_per_token": 1.3e-07, "litellm_provider": "openrouter", @@ -48806,22 +48663,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "us.amazon.nova-premier-v1:0": { - "deprecation_date": "2026-09-14", - "input_cost_per_token": 2.5e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 1000000, - "max_output_tokens": 10000, - "max_tokens": 10000, - "mode": "chat", - "output_cost_per_token": 1.25e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_vision": true, - "cache_read_input_token_cost": 6.25e-07 - }, "us.amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 8e-07, @@ -71275,15 +71116,6 @@ "output_cost_per_token": 1.6e-06, "source": "https://api.together.ai/v1/models" }, - "vertex_ai/gemini-2.5-flash-native-audio": { - "input_cost_per_audio_token": 3e-06, - "input_cost_per_token": 5e-07, - "litellm_provider": "vertex_ai", - "mode": "realtime", - "output_cost_per_audio_token": 1.2e-05, - "output_cost_per_token": 2e-06, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" - }, "vertex_ai/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, @@ -71293,16 +71125,6 @@ "output_cost_per_token": 1e-05, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, - "vertex_ai/gemini-3.1-flash-live-preview": { - "input_cost_per_audio_token": 3e-06, - "input_cost_per_second": 8.33333333333e-05, - "input_cost_per_token": 7.5e-07, - "litellm_provider": "vertex_ai", - "mode": "realtime", - "output_cost_per_audio_token": 1.2e-05, - "output_cost_per_token": 4.5e-06, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" - }, "vertex_ai/gemini-3.1-flash-tts-preview": { "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, @@ -71335,16 +71157,6 @@ "output_cost_per_token": 9e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, - "vertex_ai/gemini-robotics-er-2": { - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, - "litellm_provider": "vertex_ai", - "mode": "chat", - "output_cost_per_token": 5e-06, - "output_cost_per_token_batches": 2.5e-06, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" - }, "vertex_ai/gemma-4-26b-a4b-it": { "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, @@ -71754,14 +71566,6 @@ "output_cost_per_token_batches": 2.42e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/eu/o1-preview": { - "cache_read_input_token_cost": 8.25e-06, - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/eu/o3-2025-04-16": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, @@ -72103,14 +71907,6 @@ "output_cost_per_token_batches": 2.42e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/us/o1-preview": { - "cache_read_input_token_cost": 8.25e-06, - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/us/o3-deep-research": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 2.75e-06, @@ -74456,85 +74252,6 @@ "supports_vision": false, "supports_web_search": false }, - "openrouter/deepseek/deepseek-v4-flash-0731:batch": { - "cache_read_input_token_cost": 3.5e-09, - "input_cost_per_token": 1.1e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, - "mode": "chat", - "output_cost_per_token": 3.3e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, - "openrouter/deepseek/deepseek-v4-flash-0731:free": { - "input_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 393216, - "max_tokens": 393216, - "mode": "chat", - "output_cost_per_token": 0.0, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": false, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, - "openrouter/deepseek/deepseek-v4-flash-vision-exp:batch": { - "cache_read_input_token_cost": 3.5e-09, - "input_cost_per_token": 1.1e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, - "mode": "chat", - "output_cost_per_token": 3.3e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, - "openrouter/deepseek/deepseek-v4-pro-0813:batch": { - "cache_read_input_token_cost": 2.2e-08, - "input_cost_per_token": 6.6e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, - "mode": "chat", - "output_cost_per_token": 1.98e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/dots-studio/dots-3-note-preview:free": { "deprecation_date": "2026-12-31", "input_cost_per_token": 0.0, @@ -75050,26 +74767,6 @@ "supports_vision": false, "supports_web_search": false }, - "openrouter/kwaipilot/kat-coder-pro-v2": { - "cache_read_input_token_cost": 6e-08, - "input_cost_per_token": 3e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 144000, - "max_tokens": 144000, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": false, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/kwaipilot/kat-coder-pro-v2.5": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 7.4e-07, @@ -75149,26 +74846,6 @@ "supports_vision": true, "supports_web_search": false }, - "openrouter/meta/muse-glimmer-30b:batch": { - "cache_read_input_token_cost": 2e-08, - "input_cost_per_token": 1.75e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 131072, - "max_output_tokens": 117964, - "max_tokens": 117964, - "mode": "chat", - "output_cost_per_token": 7.5e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, "openrouter/meta/muse-spark-1.1": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.25e-06, @@ -75307,26 +74984,6 @@ "supports_vision": false, "supports_web_search": false }, - "openrouter/minimax/minimax-m3:batch": { - "cache_read_input_token_cost": 6e-08, - "input_cost_per_token": 3e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 524288, - "max_output_tokens": 471859, - "max_tokens": 471859, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, "openrouter/mistralai/codestral-2508:batch": { "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, @@ -76276,25 +75933,6 @@ "supports_vision": true, "supports_web_search": true }, - "openrouter/openai/gpt-oss-120b:batch": { - "input_cost_per_token": 1.5e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 131072, - "max_output_tokens": 117964, - "max_tokens": 117964, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": false, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/openai/o3-mini:batch": { "cache_read_input_token_cost": 2.75e-07, "input_cost_per_token": 5.5e-07, @@ -76469,45 +76107,6 @@ "supports_vision": true, "supports_web_search": true }, - "openrouter/qwen/qwen3.5-9b:batch": { - "input_cost_per_token": 1.7e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 235929, - "max_tokens": 235929, - "mode": "chat", - "output_cost_per_token": 2.5e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": false, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, - "openrouter/qwen/qwen3.8-2.4t-a95b:batch": { - "cache_read_input_token_cost": 2.5e-07, - "input_cost_per_token": 2e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 1010000, - "max_output_tokens": 909000, - "max_tokens": 909000, - "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/qwen/qwen3.8-27b:free": { "input_cost_per_token": 0.0, "litellm_provider": "openrouter", @@ -77040,26 +76639,6 @@ "supports_vision": true, "supports_web_search": false }, - "openrouter/thinkingmachines/inkling:batch": { - "cache_read_input_token_cost": 1.7e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 524288, - "max_output_tokens": 471859, - "max_tokens": 471859, - "mode": "chat", - "output_cost_per_token": 4.05e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, "openrouter/thinkingmachines/inkling:free": { "input_cost_per_token": 0.0, "litellm_provider": "openrouter", @@ -77181,26 +76760,6 @@ "supports_vision": true, "supports_web_search": true }, - "openrouter/z-ai/glm-5.2:batch": { - "cache_read_input_token_cost": 7e-08, - "input_cost_per_token": 7e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, - "mode": "chat", - "output_cost_per_token": 2.2e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/z-ai/glm-5.3-flash:batch": { "cache_read_input_token_cost": 1.2e-08, "input_cost_per_token": 6e-08, From 5035c458fbd25720118386dfb21bbfade4f8ff3c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:05:23 -0700 Subject: [PATCH 14/48] feat(proxy): admin-only /debug/report sharing the bug report environment (#42440) * feat(proxy): add admin-only /debug/report sharing the bug report environment fields Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(proxy): add verbose=true to /debug/report listing every config key with typed values Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): inject auth into /debug/report through Annotated to keep the B008 budget flat Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): bound the verbose config walk, drop nested-list recursion from the safe renderer, regenerate schema.d.ts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): count pass-through and mcp header maps plus operator-named budget maps in verbose /debug/report, single-exit scalar renderers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(ui): regenerate schema.d.ts after dropping the verbose query from /debug/report Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: ryan Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/bug_report.py | 53 +++++++----- litellm/proxy/bug_report_config.py | 29 +++++-- litellm/proxy/common_utils/debug_utils.py | 22 ++++- .../litellm_core_utils/test_bug_report.py | 26 ++++++ .../proxy/common_utils/test_debug_utils.py | 81 ++++++++++++++++++- .../proxy/test_bug_report_config.py | 14 +++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 61 ++++++++++++++ 7 files changed, 255 insertions(+), 31 deletions(-) diff --git a/litellm/litellm_core_utils/bug_report.py b/litellm/litellm_core_utils/bug_report.py index 4a8f4bc65ad..25fddd6a10e 100644 --- a/litellm/litellm_core_utils/bug_report.py +++ b/litellm/litellm_core_utils/bug_report.py @@ -23,16 +23,22 @@ Surface = Literal["sdk", "proxy"] @dataclass(frozen=True, slots=True) -class BugReport: +class EnvironmentReport: surface: Surface - exception_type: str - litellm_frames: tuple[str, ...] litellm_version: str python_version: str + deployment: str | None + config_lines: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class BugReport: + environment: EnvironmentReport + exception_type: str + litellm_frames: tuple[str, ...] call_type: str | None custom_llm_provider: str | None stream: bool | None - config_lines: tuple[str, ...] def bug_report_enabled() -> bool: @@ -69,6 +75,22 @@ def allowlisted(value: object, allowed: frozenset[str]) -> str | None: return value if isinstance(value, str) and value in allowed else None +def _deployment(surface: Surface) -> str | None: + if surface == "sdk": + return "pip / Python SDK" + return "Docker" if os.path.exists("/.dockerenv") else None + + +def build_environment_report(*, surface: Surface, config_lines: tuple[str, ...] = ()) -> EnvironmentReport: + return EnvironmentReport( + surface=surface, + litellm_version=litellm_version, + python_version=platform.python_version(), + deployment=_deployment(surface), + config_lines=config_lines, + ) + + def build_bug_report( exc: BaseException, *, @@ -79,20 +101,17 @@ def build_bug_report( config_lines: tuple[str, ...] = (), ) -> BugReport: return BugReport( - surface=surface, + environment=build_environment_report(surface=surface, config_lines=config_lines), exception_type=type(exc).__name__, litellm_frames=_get_litellm_frames(exc), - litellm_version=litellm_version, - python_version=platform.python_version(), call_type=call_type, custom_llm_provider=allowlisted(custom_llm_provider, KNOWN_PROVIDERS), stream=stream if isinstance(stream, bool) else None, - config_lines=config_lines, ) def _domain(report: BugReport) -> str: - if report.surface == "sdk": + if report.environment.surface == "sdk": return "Python SDK: the litellm package itself" if report.custom_llm_provider is not None: return "LLM translation: a specific provider's request or response" @@ -119,11 +138,11 @@ def _description(report: BugReport, frames: tuple[str, ...], config_lines: tuple "```\n\n```\n\n" f"Exception: `{report.exception_type}`\n\n" f"{frame_block}" - f"Surface: {report.surface}\n" + f"Surface: {report.environment.surface}\n" f"Endpoint / call: {report.call_type or 'unknown'}\n" f"Provider: {report.custom_llm_provider or 'unknown'}\n" - f"LiteLLM: {report.litellm_version}\n" - f"Python: {report.python_version}\n" + f"LiteLLM: {report.environment.litellm_version}\n" + f"Python: {report.environment.python_version}\n" f"{stream_line}" f"{config_block}" ) @@ -131,17 +150,13 @@ def _description(report: BugReport, frames: tuple[str, ...], config_lines: tuple def _issue_url(report: BugReport, frames: tuple[str, ...], config_lines: tuple[str, ...]) -> str: deployment: Final[tuple[tuple[str, str], ...]] = ( - (("deployment", "pip / Python SDK"),) - if report.surface == "sdk" - else (("deployment", "Docker"),) - if os.path.exists("/.dockerenv") - else () + () if report.environment.deployment is None else (("deployment", report.environment.deployment),) ) fields: Final = ( ("template", "bug_report.yml"), ("labels", "bug"), ("title", _title(report, frames)), - ("version", report.litellm_version), + ("version", report.environment.litellm_version), ("domain", _domain(report)), ("description", _description(report, frames, config_lines)), ) + deployment @@ -150,7 +165,7 @@ def _issue_url(report: BugReport, frames: tuple[str, ...], config_lines: tuple[s def bug_report_issue_url(report: BugReport) -> str: frames: Final = report.litellm_frames - config_lines: Final = report.config_lines + config_lines: Final = report.environment.config_lines candidates: Final = ( *((frames, config_lines[:count]) for count in range(len(config_lines), -1, -1)), *((frames[index:], ()) for index in range(1, len(frames) + 1)), diff --git a/litellm/proxy/bug_report_config.py b/litellm/proxy/bug_report_config.py index 527be65c9a7..d7b920de4d5 100644 --- a/litellm/proxy/bug_report_config.py +++ b/litellm/proxy/bug_report_config.py @@ -11,7 +11,14 @@ from typing import Final from pydantic import JsonValue, TypeAdapter, ValidationError import litellm -from litellm.litellm_core_utils.bug_report import KNOWN_PROVIDERS, BugReport, allowlisted, build_bug_report +from litellm.litellm_core_utils.bug_report import ( + KNOWN_PROVIDERS, + BugReport, + EnvironmentReport, + allowlisted, + build_bug_report, + build_environment_report, +) from litellm.proxy._types import ConfigGeneralSettings from litellm.router_utils.routing_groups import VALID_ROUTING_STRATEGIES from litellm.types.caching import LiteLLMCacheType @@ -191,6 +198,19 @@ def safe_config_lines(config: Mapping[str, object], general_settings: Mapping[st ) +def _proxy_config_lines() -> tuple[str, ...]: + from litellm.proxy import proxy_server + + return safe_config_lines( + proxy_server.proxy_config.config, + _object_map(proxy_server.general_settings), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # bare dict global, validated by _object_map + ) + + +def build_proxy_environment_report() -> EnvironmentReport: + return build_environment_report(surface="proxy", config_lines=_proxy_config_lines()) + + def build_proxy_bug_report( exc: BaseException, *, @@ -198,16 +218,11 @@ def build_proxy_bug_report( custom_llm_provider: object = None, stream: object = None, ) -> BugReport: - from litellm.proxy import proxy_server - return build_bug_report( exc, surface="proxy", call_type=call_type, custom_llm_provider=custom_llm_provider, stream=stream, - config_lines=safe_config_lines( - proxy_server.proxy_config.config, - _object_map(proxy_server.general_settings), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # bare dict global, validated by _object_map - ), + config_lines=_proxy_config_lines(), ) diff --git a/litellm/proxy/common_utils/debug_utils.py b/litellm/proxy/common_utils/debug_utils.py index dc329e55e31..2544321a1b6 100644 --- a/litellm/proxy/common_utils/debug_utils.py +++ b/litellm/proxy/common_utils/debug_utils.py @@ -8,7 +8,7 @@ import sys import tracemalloc from collections import Counter from collections.abc import Mapping, Sequence -from typing import Any, Final, NamedTuple, Protocol, TypedDict +from typing import Annotated, Any, Final, NamedTuple, Protocol, TypedDict from fastapi import APIRouter, Depends, HTTPException, Query from typing_extensions import ReadOnly @@ -16,8 +16,11 @@ from typing_extensions import ReadOnly from litellm import get_secret_str from litellm._logging import verbose_proxy_logger from litellm.constants import PYTHON_GC_THRESHOLD +from litellm.litellm_core_utils.bug_report import EnvironmentReport from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.bug_report_config import build_proxy_environment_report +from litellm.proxy.common_utils.resource_ownership import is_proxy_admin router: Final = APIRouter() @@ -783,6 +786,23 @@ async def configure_gc_thresholds_endpoint( } +@router.get("/debug/report", include_in_schema=False) +async def get_debug_report( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> EnvironmentReport: + """ + The same LiteLLM-owned environment facts the bug report link puts in a GitHub issue: + versions, deployment kind, and config flags whose keys and values LiteLLM defines. + Nothing from the operator's config values, request data, or errors + + Example usage: + curl http://localhost:4000/debug/report -H "Authorization: Bearer sk-1234" + """ + if not is_proxy_admin(user_api_key_dict): + raise HTTPException(status_code=403, detail="Only proxy admins can read /debug/report") + return build_proxy_environment_report() + + @router.get( "/otel-spans", dependencies=[Depends(user_api_key_auth)], diff --git a/tests/test_litellm/litellm_core_utils/test_bug_report.py b/tests/test_litellm/litellm_core_utils/test_bug_report.py index f47361dda2e..62d7090b960 100644 --- a/tests/test_litellm/litellm_core_utils/test_bug_report.py +++ b/tests/test_litellm/litellm_core_utils/test_bug_report.py @@ -21,6 +21,7 @@ from litellm.litellm_core_utils.bug_report import ( bug_report_issue_url, bug_report_notice, build_bug_report, + build_environment_report, should_report_bug, strip_bug_report_notice, ) @@ -202,3 +203,28 @@ def test_oversized_config_is_trimmed_from_the_end_before_any_frame(): assert all(frame in description for frame in report.litellm_frames) assert "general_settings.flag_0000 = true" in description assert "general_settings.flag_0399 = true" not in description + + +def test_issue_url_carries_exactly_the_environment_report_fields(): + report = build_bug_report( + RuntimeError("boom"), + surface="proxy", + config_lines=("litellm_settings.drop_params = true",), + ) + environment = report.environment + query = parse_qs(urlparse(bug_report_issue_url(report)).query) + description = query["description"][0] + + assert environment == build_environment_report( + surface="proxy", config_lines=("litellm_settings.drop_params = true",) + ) + assert query["version"] == [environment.litellm_version] + assert f"Surface: {environment.surface}\n" in description + assert f"LiteLLM: {environment.litellm_version}\n" in description + assert f"Python: {environment.python_version}\n" in description + assert "\nlitellm_settings.drop_params = true\n" in description + assert query.get("deployment") == (None if environment.deployment is None else [environment.deployment]) + + +def test_sdk_environment_reports_the_pip_deployment(): + assert build_environment_report(surface="sdk").deployment == "pip / Python SDK" diff --git a/tests/test_litellm/proxy/common_utils/test_debug_utils.py b/tests/test_litellm/proxy/common_utils/test_debug_utils.py index 163ea530be9..a64c94c2991 100644 --- a/tests/test_litellm/proxy/common_utils/test_debug_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_debug_utils.py @@ -1,16 +1,25 @@ +import json import os import socket +from collections.abc import Iterator, Mapping +from dataclasses import asdict from pathlib import Path import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy import proxy_server +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.bug_report_config import build_proxy_bug_report from litellm.proxy.common_utils.debug_utils import ( PSUTIL_MISSING_ERROR, _ProcFilesystemProcess, _summary_process_memory, get_memory_summary, ) +from litellm.proxy.common_utils.debug_utils import router as debug_router PAGE_SIZE = 4096 STATM_SIZE_PAGES = 100_000 @@ -67,3 +76,73 @@ async def test_memory_summary_names_the_host_and_worker_that_answered() -> None: assert summary["hostname"] == socket.gethostname() assert summary["worker_pid"] == os.getpid() assert summary["memory"]["ram_usage_mb"] > 0 + + +HOSTILE_CONFIG: Mapping[str, object] = { + "model_list": [ + { + "model_name": "acme-prod-gpt4", + "litellm_params": { + "model": "azure/acme-gpt4o-deployment", + "api_base": "https://acme-eastus.openai.azure.com", + "api_key": "sk-live-secret-1", + }, + } + ], + "litellm_settings": {"drop_params": True, "callbacks": ["langfuse", "acme_hooks.audit_logger"]}, +} + +HOSTILE_GENERAL_SETTINGS: Mapping[str, object] = { + "master_key": "sk-live-secret-master", + "database_url": "postgres://user:hunter2@10.0.0.7/litellm", + "store_model_in_db": True, +} + +HOSTILE_STRINGS = ("acme", "sk-live-secret", "hunter2", "10.0.0.7", "azure.com") + + +@pytest.fixture +def hostile_proxy_config(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + previous_config = proxy_server.proxy_config.get_config_state() + proxy_server.proxy_config.update_config_state(config=HOSTILE_CONFIG) + monkeypatch.setattr(proxy_server, "general_settings", dict(HOSTILE_GENERAL_SETTINGS)) + yield + proxy_server.proxy_config.update_config_state(config=previous_config) + + +def _debug_client(caller: UserAPIKeyAuth) -> TestClient: + app = FastAPI() + app.include_router(debug_router) + app.dependency_overrides[user_api_key_auth] = lambda: caller + return TestClient(app) + + +@pytest.mark.parametrize( + "caller", + [ + UserAPIKeyAuth(), + UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER), + UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY), + ], +) +@pytest.mark.usefixtures("hostile_proxy_config") +def test_debug_report_refuses_everyone_but_proxy_admins(caller: UserAPIKeyAuth) -> None: + response = _debug_client(caller).get("/debug/report") + + assert response.status_code == 403, response.text + assert "litellm_version" not in response.text + + +@pytest.mark.usefixtures("hostile_proxy_config") +def test_debug_report_returns_what_the_bug_report_link_carries_and_nothing_from_the_operator() -> None: + response = _debug_client(UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)).get("/debug/report") + + assert response.status_code == 200, response.text + assert response.json() == json.loads(json.dumps(asdict(build_proxy_bug_report(RuntimeError("boom")).environment))) + assert response.json()["config_lines"] == [ + "general_settings.store_model_in_db = true", + "litellm_settings.drop_params = true", + "litellm_settings.callbacks = [langfuse]", + "model_list[*].provider = [azure]", + ] + assert not any(hostile in response.text for hostile in HOSTILE_STRINGS), response.text diff --git a/tests/test_litellm/proxy/test_bug_report_config.py b/tests/test_litellm/proxy/test_bug_report_config.py index 06bca8c66fb..6cffa55781e 100644 --- a/tests/test_litellm/proxy/test_bug_report_config.py +++ b/tests/test_litellm/proxy/test_bug_report_config.py @@ -5,7 +5,7 @@ from collections.abc import Iterator, Mapping import pytest from litellm.proxy import proxy_server -from litellm.proxy.bug_report_config import build_proxy_bug_report, safe_config_lines +from litellm.proxy.bug_report_config import build_proxy_bug_report, build_proxy_environment_report, safe_config_lines CUSTOMER_STRINGS = ( "acme", @@ -193,6 +193,14 @@ def loaded_proxy_config(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: def test_build_proxy_bug_report_reads_the_loaded_proxy_config(): report = build_proxy_bug_report(RuntimeError("boom"), stream=False) - assert report.surface == "proxy" + assert report.environment.surface == "proxy" assert report.stream is False - assert report.config_lines == safe_config_lines(CUSTOMER_CONFIG, CUSTOMER_GENERAL_SETTINGS) + assert report.environment.config_lines == safe_config_lines(CUSTOMER_CONFIG, CUSTOMER_GENERAL_SETTINGS) + + +@pytest.mark.usefixtures("loaded_proxy_config") +def test_proxy_environment_report_matches_the_bug_report_environment(): + environment = build_proxy_environment_report() + + assert environment == build_proxy_bug_report(RuntimeError("boom")).environment + assert environment.config_lines == safe_config_lines(CUSTOMER_CONFIG, CUSTOMER_GENERAL_SETTINGS) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 68d3ab364d6..a0fec542acf 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -4356,6 +4356,31 @@ export interface paths { patch?: never; trace?: never; }; + "/debug/report": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Debug Report + * @description The same LiteLLM-owned environment facts the bug report link puts in a GitHub issue: + * versions, deployment kind, and config flags whose keys and values LiteLLM defines. + * Nothing from the operator's config values, request data, or errors + * + * Example usage: + * curl http://localhost:4000/debug/report -H "Authorization: Bearer sk-1234" + */ + get: operations["get_debug_report_debug_report_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/delete/allowed_ip": { parameters: { query?: never; @@ -28779,6 +28804,22 @@ export interface components { /** Template Id */ template_id: string; }; + /** EnvironmentReport */ + EnvironmentReport: { + /** Config Lines */ + config_lines: string[]; + /** Deployment */ + deployment: string | null; + /** Litellm Version */ + litellm_version: string; + /** Python Version */ + python_version: string; + /** + * Surface + * @enum {string} + */ + surface: "sdk" | "proxy"; + }; /** ErrorResponse */ ErrorResponse: { /** @@ -48592,6 +48633,26 @@ export interface operations { }; }; }; + get_debug_report_debug_report_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EnvironmentReport"]; + }; + }; + }; + }; delete_allowed_ip_delete_allowed_ip_post: { parameters: { query?: never; From 64456ce10311d52f32c88efa06e8bba65141d3a3 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:12:20 -0500 Subject: [PATCH 15/48] fix(aws_secret_manager_v2): restore secret scheduled for deletion instead of failing CreateSecret (#42454) * fix(aws_secret_manager_v2): restore secret scheduled for deletion instead of failing CreateSecret Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(aws_secret_manager_v2): reapply CreateSecret metadata and reschedule deletion when in-place restore fails Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../secret_managers/aws_secret_manager_v2.py | 131 ++++++++++--- .../test_aws_secret_manager_rotation.py | 178 +++++++++++++++++- 2 files changed, 286 insertions(+), 23 deletions(-) diff --git a/litellm/secret_managers/aws_secret_manager_v2.py b/litellm/secret_managers/aws_secret_manager_v2.py index 0fb59105b5f..3e9f4e259d5 100644 --- a/litellm/secret_managers/aws_secret_manager_v2.py +++ b/litellm/secret_managers/aws_secret_manager_v2.py @@ -16,6 +16,8 @@ Requires: import json import os +from collections.abc import Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Final import httpx @@ -294,32 +296,13 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): raise ValueError("Tags must be a dict or list of {Key, Value} pairs") data["Tags"] = tags_list - endpoint_url, headers, body = self._prepare_request( - action="CreateSecret", + create_response: Final = await self._async_create_or_restore_secret( secret_name=secret_name, - secret_value=secret_value, - optional_params=optional_params, request_data=data, + optional_params=optional_params, + timeout=timeout, ) - async_client: Final = get_async_httpx_client( - llm_provider=httpxSpecialProvider.SecretManager, - params={"timeout": timeout}, - ) - - try: - response: Final = await async_client.post( - url=endpoint_url, - headers=headers, - data=body.decode("utf-8"), - ) - response.raise_for_status() - create_response: Final = response.json() - except httpx.HTTPStatusError as err: - raise ValueError(f"HTTP error occurred: {err.response.text}") - except httpx.TimeoutException: - raise ValueError("Timeout error occurred") - if self.replica_regions: try: await self.async_replicate_secret( @@ -343,6 +326,110 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): return create_response + async def _async_create_or_restore_secret( + self, + secret_name: str, + request_data: Mapping[str, object], + optional_params: dict | None, + timeout: float | httpx.Timeout | None, + ) -> dict[str, object]: + try: + return await self._async_post_action( + action="CreateSecret", + secret_name=secret_name, + request_data=request_data, + optional_params=optional_params, + timeout=timeout, + ) + except ValueError: + if not await self._async_is_scheduled_for_deletion( + secret_name=secret_name, + optional_params=optional_params, + timeout=timeout, + ): + raise + + verbose_logger.info( + "Secret %s is scheduled for deletion, restoring and updating in place (RestoreSecret + UpdateSecret)", + secret_name, + ) + await self._async_post_action( + action="RestoreSecret", + secret_name=secret_name, + request_data=None, + optional_params=optional_params, + timeout=timeout, + ) + update_data: Final = MappingProxyType( + {("SecretId" if key == "Name" else key): value for key, value in request_data.items() if key != "Tags"} + ) + tags: Final = request_data.get("Tags") + try: + updated: Final = await self._async_post_action( + action="UpdateSecret", + secret_name=secret_name, + request_data=update_data, + optional_params=optional_params, + timeout=timeout, + ) + if tags is not None: + await self._async_post_action( + action="TagResource", + secret_name=secret_name, + request_data=MappingProxyType({"SecretId": secret_name, "Tags": tags}), + optional_params=optional_params, + timeout=timeout, + ) + except ValueError: + await self.async_delete_secret(secret_name=secret_name, optional_params=optional_params, timeout=timeout) + raise + return updated + + async def _async_is_scheduled_for_deletion( + self, + secret_name: str, + optional_params: dict | None, + timeout: float | httpx.Timeout | None, + ) -> bool: + try: + described: Final = await self._async_post_action( + action="DescribeSecret", + secret_name=secret_name, + request_data=None, + optional_params=optional_params, + timeout=timeout, + ) + except ValueError: + return False + return described.get("DeletedDate") is not None + + async def _async_post_action( + self, + action: str, + secret_name: str, + request_data: Mapping[str, object] | None, + optional_params: dict | None, + timeout: float | httpx.Timeout | None, + ) -> dict[str, object]: + endpoint_url, headers, body = self._prepare_request( + action=action, + secret_name=secret_name, + optional_params=optional_params, + request_data=dict(request_data) if request_data is not None else None, + ) + async_client: Final = get_async_httpx_client( + llm_provider=httpxSpecialProvider.SecretManager, + params={"timeout": timeout}, + ) + try: + response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8")) + response.raise_for_status() + return response.json() + except httpx.HTTPStatusError as err: + raise ValueError(f"HTTP error occurred: {err.response.text}") + except httpx.TimeoutException: + raise ValueError("Timeout error occurred") + async def async_replicate_secret( self, secret_name: str, diff --git a/tests/test_litellm/secret_managers/test_aws_secret_manager_rotation.py b/tests/test_litellm/secret_managers/test_aws_secret_manager_rotation.py index 4a4cec6bf77..af1380f6fd1 100644 --- a/tests/test_litellm/secret_managers/test_aws_secret_manager_rotation.py +++ b/tests/test_litellm/secret_managers/test_aws_secret_manager_rotation.py @@ -1,11 +1,17 @@ -from collections.abc import Mapping +import json +from collections.abc import Iterator, Mapping +from contextlib import contextmanager from dataclasses import dataclass, replace from types import MappingProxyType from typing import Final, TypeAlias +import httpx import pytest +import litellm +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.secret_managers.aws_secret_manager_v2 import AWSSecretsManagerV2 +from litellm.types.llms.custom_http import httpxSpecialProvider OptionalParams: TypeAlias = Mapping[str, object] | None @@ -204,3 +210,173 @@ async def test_rotate_secret_different_names_persists_requested_value_and_delete assert manager.storage.values[new_name] == new_value assert current_name not in manager.storage.values assert manager.storage.values[unrelated_secret_name] == unrelated_value + + +@dataclass(frozen=True, slots=True) +class FakeSecretsManagerState: + live: Mapping[str, str] + scheduled_for_deletion: frozenset[str] = frozenset() + actions: tuple[str, ...] = () + descriptions: Mapping[str, str] = MappingProxyType({}) + failing_actions: frozenset[str] = frozenset() + + +class FakeSecretsManagerService: + def __init__(self, state: FakeSecretsManagerState) -> None: + self.state = state + + def handle(self, request: httpx.Request) -> httpx.Response: + action: Final = request.headers["X-Amz-Target"].removeprefix("secretsmanager.") + body: Final = json.loads(request.content) + name: Final = str(body.get("Name") or body.get("SecretId")) + self.state = replace(self.state, actions=(*self.state.actions, f"{action}:{name}")) + if action in self.state.failing_actions: + return self._error("InternalServiceError", f"injected failure for {action}") + match action: + case "CreateSecret": + if name in self.state.live: + return self._error("ResourceExistsException", f"The secret {name} already exists") + self.state = replace( + self.state, + live=MappingProxyType({**self.state.live, name: str(body["SecretString"])}), + descriptions=MappingProxyType({**self.state.descriptions, name: str(body.get("Description", ""))}), + ) + return httpx.Response(200, json={"ARN": f"arn:fake:{name}", "Name": name}) + case "UpdateSecret": + if name in self.state.scheduled_for_deletion: + return self._error( + "InvalidRequestException", + "You can't perform this operation on the secret because it was marked for deletion.", + ) + self.state = replace( + self.state, + live=MappingProxyType({**self.state.live, name: str(body["SecretString"])}), + descriptions=MappingProxyType({**self.state.descriptions, name: str(body.get("Description", ""))}), + ) + return httpx.Response(200, json={"ARN": f"arn:fake:{name}", "Name": name}) + case "DescribeSecret": + if name not in self.state.live: + return self._error("ResourceNotFoundException", "Secrets Manager can't find the specified secret.") + deleted: Final = "2026-01-01T00:00:00Z" if name in self.state.scheduled_for_deletion else None + return httpx.Response(200, json={"ARN": f"arn:fake:{name}", "Name": name, "DeletedDate": deleted}) + case "RestoreSecret": + self.state = replace(self.state, scheduled_for_deletion=self.state.scheduled_for_deletion - {name}) + return httpx.Response(200, json={"ARN": f"arn:fake:{name}", "Name": name}) + case "PutSecretValue": + if name in self.state.scheduled_for_deletion: + return self._error( + "InvalidRequestException", + "You can't perform this operation on the secret because it was marked for deletion.", + ) + self.state = replace( + self.state, live=MappingProxyType({**self.state.live, name: str(body["SecretString"])}) + ) + return httpx.Response(200, json={"ARN": f"arn:fake:{name}", "Name": name}) + case "GetSecretValue": + if name not in self.state.live or name in self.state.scheduled_for_deletion: + return self._error("ResourceNotFoundException", "Secrets Manager can't find the specified secret.") + return httpx.Response(200, json={"SecretString": self.state.live[name]}) + case "DeleteSecret": + self.state = replace(self.state, scheduled_for_deletion=self.state.scheduled_for_deletion | {name}) + return httpx.Response(200, json={"ARN": f"arn:fake:{name}", "Name": name}) + return self._error("UnsupportedAction", action) + + @staticmethod + def _error(error_type: str, message: str) -> httpx.Response: + return httpx.Response(400, json={"__type": error_type, "message": message}) + + +@contextmanager +def fake_secrets_manager(monkeypatch: pytest.MonkeyPatch) -> Iterator[FakeSecretsManagerService]: + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "synthetic-access-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "synthetic-secret-key") + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + service: Final = FakeSecretsManagerService(FakeSecretsManagerState(live=MappingProxyType({}))) + cache_key: Final = "async_httpx_clienttimeout_None" + httpxSpecialProvider.SecretManager + litellm.in_memory_llm_clients_cache.set_cache( + key=cache_key, + value=AsyncHTTPHandler(transport=httpx.MockTransport(service.handle)), + ) + try: + yield service + finally: + litellm.in_memory_llm_clients_cache.delete_cache( + litellm.in_memory_llm_clients_cache.update_cache_key_with_event_loop(cache_key) + ) + + +@pytest.mark.asyncio +async def test_rotate_secret_back_to_name_inside_recovery_window_restores_and_stores_new_value( + monkeypatch: pytest.MonkeyPatch, +) -> None: + alias_a: Final = "synthetic/alias-a" + alias_b: Final = "synthetic/alias-b" + with fake_secrets_manager(monkeypatch) as fake: + manager: Final = AWSSecretsManagerV2(aws_region_name="us-east-1") + await manager.async_write_secret(secret_name=alias_a, secret_value="value-1") + await manager.async_rotate_secret( + current_secret_name=alias_a, new_secret_name=alias_b, new_secret_value="value-2" + ) + assert alias_a in fake.state.scheduled_for_deletion + + await manager.async_rotate_secret( + current_secret_name=alias_b, new_secret_name=alias_a, new_secret_value="value-3" + ) + + assert await manager.async_read_secret(secret_name=alias_a) == "value-3" + assert await manager.async_read_secret(secret_name=alias_b) is None + assert fake.state.scheduled_for_deletion == frozenset({alias_b}) + assert fake.state.descriptions[alias_a] == f"Rotated from {alias_b}" + + +@pytest.mark.asyncio +async def test_write_secret_to_name_inside_recovery_window_reschedules_deletion_when_update_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + alias: Final = "synthetic/deleted-alias" + with fake_secrets_manager(monkeypatch) as fake: + manager: Final = AWSSecretsManagerV2(aws_region_name="us-east-1") + await manager.async_write_secret(secret_name=alias, secret_value="value-1") + await manager.async_delete_secret(secret_name=alias, recovery_window_in_days=7) + fake.state = replace(fake.state, failing_actions=frozenset({"UpdateSecret"})) + + with pytest.raises(ValueError, match="injected failure for UpdateSecret"): + await manager.async_write_secret(secret_name=alias, secret_value="value-2") + + assert fake.state.scheduled_for_deletion == frozenset({alias}) + assert fake.state.live[alias] == "value-1" + + +@pytest.mark.asyncio +async def test_write_secret_to_name_inside_recovery_window_restores_and_stores_new_value( + monkeypatch: pytest.MonkeyPatch, +) -> None: + alias: Final = "synthetic/deleted-alias" + with fake_secrets_manager(monkeypatch) as fake: + manager: Final = AWSSecretsManagerV2(aws_region_name="us-east-1") + await manager.async_write_secret(secret_name=alias, secret_value="value-1") + await manager.async_delete_secret(secret_name=alias, recovery_window_in_days=7) + + assert await manager.async_write_secret(secret_name=alias, secret_value="value-2") == { + "ARN": f"arn:fake:{alias}", + "Name": alias, + } + + assert await manager.async_read_secret(secret_name=alias) == "value-2" + assert fake.state.scheduled_for_deletion == frozenset() + + +@pytest.mark.asyncio +async def test_write_secret_to_live_existing_name_still_fails_without_overwriting( + monkeypatch: pytest.MonkeyPatch, +) -> None: + alias: Final = "synthetic/live-alias" + with fake_secrets_manager(monkeypatch) as fake: + manager: Final = AWSSecretsManagerV2(aws_region_name="us-east-1") + await manager.async_write_secret(secret_name=alias, secret_value="value-1") + + with pytest.raises(ValueError, match="ResourceExistsException"): + await manager.async_write_secret(secret_name=alias, secret_value="value-2") + + assert await manager.async_read_secret(secret_name=alias) == "value-1" + assert f"RestoreSecret:{alias}" not in fake.state.actions From 05d7fb24bd1e9ced3a5877c82e2de00367744037 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:16:56 -0700 Subject: [PATCH 16/48] feat(rust): add python-compat crate for Python data formats (#42510) * feat(rust): add python-compat crate for Python data formats Add litellm-python-compat, a PyO3-free crate that reproduces the Python data formats LiteLLM persists, so Rust readers and writers can interoperate with state written by the Python proxy: - literal::literal_eval: a linear recursive-descent port of ast.literal_eval (prefixes, escapes, implicit concatenation, numeric underscores and radixes, single unary sign, real +/- complex with 3.14 mixed-mode rules, set(), Python-equality key dedup) - repr::{repr, to_str}: byte-exact repr()/str(), with a printable table generated from CPython's str.isprintable (Unicode 16.0.0) - json::{dumps, from_json, to_json}: json.dumps defaults and the json.loads mapping - pickle::{loads, dumps}: plain-data pickles via serde-pickle's serde interface, which keeps dict insertion order - truthy::truthy: bool() for plain data Tests replay fixtures generated by CPython 3.14 (values across every format and pickle protocol 0-5, plus 154 literal_eval source texts). Accepted divergences are pinned in a KNOWN table that fails once one starts matching. A criterion bench covers each format and literal_eval cost by nesting depth, guarding the linear parse: the py_literal grammar doubled per nested bracket (105 ms at 16 nested dicts; 19 us at 128 now). Co-Authored-By: Claude Opus 5 * refactor(rust): split python-compat modules and harden the pickle verifier - Disable class resolution in scripts/verify_rust_pickles.py, and truncate the export file once instead of removing and appending to it, so the verifier cannot be pointed at a pre-created file whose rows execute code through pickle.loads - Move Error to error.rs and Value to value.rs, leaving lib.rs as the crate overview, module list and MAX_DEPTH - Move the generator and verifier to scripts/, beside the Unicode table generator, leaving tests/ to the Rust tests - Group the bench by measured surface, give every case a Throughput so criterion reports bytes per second, and document baseline comparison Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Yujong Lee Co-authored-by: Claude Opus 5 --- litellm-rust/Cargo.lock | 15 + litellm-rust/crates/python-compat/AGENTS.md | 23 + litellm-rust/crates/python-compat/Cargo.toml | 24 + .../crates/python-compat/benches/formats.rs | 111 + .../python-compat/generated/nonprintable.rs | 745 ++++++ .../python-compat/generated/values.json | 2164 +++++++++++++++++ .../scripts/generate_fixtures.py | 352 +++ .../scripts/generate_nonprintable.py | 35 + .../scripts/verify_rust_pickles.py | 43 + .../crates/python-compat/src/error.rs | 25 + litellm-rust/crates/python-compat/src/json.rs | 168 ++ litellm-rust/crates/python-compat/src/lib.rs | 39 + .../crates/python-compat/src/literal.rs | 745 ++++++ .../crates/python-compat/src/pickle.rs | 187 ++ litellm-rust/crates/python-compat/src/repr.rs | 237 ++ .../crates/python-compat/src/truthy.rs | 18 + .../crates/python-compat/src/value.rs | 53 + .../crates/python-compat/tests/fixtures.rs | 343 +++ .../crates/python-compat/tests/limits.rs | 122 + 19 files changed, 5449 insertions(+) create mode 100644 litellm-rust/crates/python-compat/AGENTS.md create mode 100644 litellm-rust/crates/python-compat/Cargo.toml create mode 100644 litellm-rust/crates/python-compat/benches/formats.rs create mode 100644 litellm-rust/crates/python-compat/generated/nonprintable.rs create mode 100644 litellm-rust/crates/python-compat/generated/values.json create mode 100644 litellm-rust/crates/python-compat/scripts/generate_fixtures.py create mode 100644 litellm-rust/crates/python-compat/scripts/generate_nonprintable.py create mode 100644 litellm-rust/crates/python-compat/scripts/verify_rust_pickles.py create mode 100644 litellm-rust/crates/python-compat/src/error.rs create mode 100644 litellm-rust/crates/python-compat/src/json.rs create mode 100644 litellm-rust/crates/python-compat/src/lib.rs create mode 100644 litellm-rust/crates/python-compat/src/literal.rs create mode 100644 litellm-rust/crates/python-compat/src/pickle.rs create mode 100644 litellm-rust/crates/python-compat/src/repr.rs create mode 100644 litellm-rust/crates/python-compat/src/truthy.rs create mode 100644 litellm-rust/crates/python-compat/src/value.rs create mode 100644 litellm-rust/crates/python-compat/tests/fixtures.rs create mode 100644 litellm-rust/crates/python-compat/tests/limits.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 76004c5d978..ea63746d56d 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3072,6 +3072,21 @@ dependencies = [ "wiremock", ] +[[package]] +name = "litellm-python-compat" +version = "0.1.0" +dependencies = [ + "criterion", + "hex", + "num-bigint 0.4.8", + "num-traits", + "rstest", + "serde", + "serde-pickle", + "serde_json", + "thiserror 2.0.19", +] + [[package]] name = "litellm-secrets" version = "0.1.0" diff --git a/litellm-rust/crates/python-compat/AGENTS.md b/litellm-rust/crates/python-compat/AGENTS.md new file mode 100644 index 00000000000..0869568d33b --- /dev/null +++ b/litellm-rust/crates/python-compat/AGENTS.md @@ -0,0 +1,23 @@ +- Pure Python *data formats* in Rust, for state Python LiteLLM writes and Rust must read or write byte-compatibly + - No PyO3, no live objects: truthiness, `__str__`, descriptors of real Python objects belong to `python-bridge`'s coercion layer + - Format *choices* stay with callers: the `{timestamp, response}` envelope, diskcache modes, and the cache-key recipe live in the cache crates and only call into this crate +- Intended users + - `cache-response` codec: reading `str(dict)` values Python's sync Redis path writes (`literal_eval`) + - `cache-disk`: diskcache's pickled values (`pickle`), falsy-is-miss (`truthy`) + - Cache-key derivation: sha256 over `str(value)` must match Python byte for byte (`repr::to_str`) + - Byte-identical writes where Python compares raw values (`json::dumps`, `repr`) +- Relation to [`py_literal`](https://docs.rs/py_literal/latest/py_literal/): replaced, do not reintroduce + - Its pest grammar backtracks: parse time doubles per nested `[`/`{` (105 ms at depth 16); ours is linear (19 µs at depth 128) + - Its formatter is not `repr` (`2e-1`, always single quotes, escapes non-ASCII); it also lost `-0.0`, `(1+2j)`, `set()` + - `cache-response` and `cache-disk` still depend on it; migrate them here +- Relation to [`serde-pickle`](https://docs.rs/serde-pickle/latest/serde_pickle/): the pickle codec, used only through its serde interface + - Never `serde_pickle::Value`: its `BTreeMap` dicts reorder keys + - Accepted limits: ints beyond i64, `tuple`/`set`/`frozenset` decode as lists, class references (`GLOBAL`/`REDUCE`) fail; writes protocol 3 +- Every behavior is pinned by CPython output, not by reasoning; everything under `generated/` is script output, never hand-edited + - Regenerate `generated/values.json` with `scripts/generate_fixtures.py`; add a corpus row before changing behavior + - Divergences go in `KNOWN` in `tests/fixtures.rs` with a reason; an entry that starts matching fails until deleted + - Regenerate `generated/nonprintable.rs` with `scripts/generate_nonprintable.py` when the target Python's Unicode version changes + - `scripts/verify_rust_pickles.py` checks CPython reads Rust pickles, with class resolution disabled; CI does not run Python +- Decoders recurse, so they reject nesting beyond `MAX_DEPTH` for stack safety: deliberately stricter than CPython, whose parser takes ~200 levels and whose unpickler has no limit (pinned as `nested_150`) + - Formatters (`repr`, `json`) are unbounded; values from the decoders are already capped, a hand-built `Value` is the caller's responsibility + - `literal_eval` must stay linear in depth: `tests/limits.rs` times the deepest parse, `benches/formats.rs` measures the curve but is manual, since CI runs no Rust bench diff --git a/litellm-rust/crates/python-compat/Cargo.toml b/litellm-rust/crates/python-compat/Cargo.toml new file mode 100644 index 00000000000..2ab6fb29843 --- /dev/null +++ b/litellm-rust/crates/python-compat/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "litellm-python-compat" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Python data formats (repr, literal_eval, json.dumps, pickle) reproduced for interop with persisted LiteLLM state" + +[dependencies] +num-bigint = "0.4" +num-traits = "0.2" +serde.workspace = true +serde-pickle = "1.2" +serde_json = { workspace = true, features = ["preserve_order"] } +thiserror.workspace = true + +[dev-dependencies] +criterion.workspace = true +hex = "0.4" +rstest.workspace = true + +[[bench]] +name = "formats" +harness = false diff --git a/litellm-rust/crates/python-compat/benches/formats.rs b/litellm-rust/crates/python-compat/benches/formats.rs new file mode 100644 index 00000000000..f0b83cb1658 --- /dev/null +++ b/litellm-rust/crates/python-compat/benches/formats.rs @@ -0,0 +1,111 @@ +//! Throughput of each format on a cached chat completion, and `literal_eval` cost by nesting. +//! +//! Run one group with `cargo bench -p litellm-python-compat -- cached_completion`, and compare +//! against a stored run with `--save-baseline ` / `--baseline `. +//! +//! `literal_eval/nesting` guards against backtracking: the `py_literal` grammar this parser +//! replaced doubled its time per nested `[` or `{` (105 ms at depth 16), so cost must stay +//! linear in depth for every container shape. + +use std::{hint::black_box, time::Duration}; + +use criterion::{ + BatchSize, BenchmarkGroup, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main, + measurement::WallTime, +}; +use litellm_python_compat::{Value, json, literal::literal_eval, pickle, repr::repr}; + +/// `str(entry)` for the `{timestamp, response}` envelope Python's sync Redis path writes. +fn cached_completion() -> String { + let choices: Vec = (0..4) + .map(|index| { + format!( + "{{'finish_reason': 'stop', 'index': {index}, 'message': {{'content': \ + 'Benchmarks compare the same workload under controlled conditions, so a \ + change in time reflects the code rather than the environment. café 日本 \ + {index}', 'role': 'assistant', 'tool_calls': None, 'function_call': None}}, \ + 'logprobs': None}}" + ) + }) + .collect(); + format!( + "{{'timestamp': 1726000000.123, 'response': {{'id': 'chatcmpl-9x1', 'created': \ + 1726000000, 'model': 'gpt-4o-2024-08-06', 'object': 'chat.completion', \ + 'system_fingerprint': 'fp_1', 'choices': [{}], 'usage': {{'completion_tokens': 120, \ + 'prompt_tokens': 42, 'total_tokens': 162, 'completion_tokens_details': None}}}}}}", + choices.join(", ") + ) +} + +/// Every text format, measured against the source bytes it reads or writes. +fn text_formats(group: &mut BenchmarkGroup<'_, WallTime>, text: &str, value: &Value) { + group.throughput(Throughput::Bytes(text.len() as u64)); + group.bench_function("literal_eval", |bencher| { + bencher.iter(|| literal_eval(black_box(text))) + }); + group.bench_function("repr", |bencher| bencher.iter(|| repr(black_box(value)))); + group.bench_function("json_dumps", |bencher| { + bencher.iter(|| json::dumps(black_box(value))) + }); + group.bench_function("to_json", |bencher| { + bencher.iter(|| json::to_json(black_box(value))) + }); +} + +/// Pickle, measured against its own encoding rather than the source text. +fn binary_formats(group: &mut BenchmarkGroup<'_, WallTime>, value: &Value, pickled: &[u8]) { + group.throughput(Throughput::Bytes(pickled.len() as u64)); + group.bench_function("pickle_dumps", |bencher| { + bencher.iter(|| pickle::dumps(black_box(value))) + }); + group.bench_function("pickle_loads", |bencher| { + bencher.iter(|| pickle::loads(black_box(pickled))) + }); +} + +fn formats(c: &mut Criterion) { + let text = cached_completion(); + let value = literal_eval(&text).expect("benchmark payload is a literal"); + let pickled = pickle::dumps(&value).expect("benchmark payload pickles"); + let dumped = json::dumps(&value).expect("benchmark payload is JSON serializable"); + + let mut group = c.benchmark_group("cached_completion"); + text_formats(&mut group, &text, &value); + binary_formats(&mut group, &value, &pickled); + // `from_json` consumes its input, so each iteration gets a freshly parsed one. + group.throughput(Throughput::Bytes(dumped.len() as u64)); + group.bench_function("from_json", |bencher| { + bencher.iter_batched( + || serde_json::from_str::(&dumped).expect("dumps output parses"), + json::from_json, + BatchSize::SmallInput, + ) + }); + group.finish(); +} + +/// One nesting level of each container shape, as `(name, open, close)`. +const SHAPES: [(&str, &str, &str); 3] = [ + ("list", "[", "]"), + ("dict", "{'a': ", "}"), + ("tuple", "(", ",)"), +]; + +fn literal_nesting(c: &mut Criterion) { + let mut group = c.benchmark_group("literal_eval/nesting"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(3)); + for depth in [4, 16, 64, 128] { + for (shape, open, close) in SHAPES { + let text = format!("{}1{}", open.repeat(depth), close.repeat(depth)); + group.throughput(Throughput::Bytes(text.len() as u64)); + group.bench_with_input(BenchmarkId::new(shape, depth), &text, |bencher, text| { + bencher.iter(|| literal_eval(black_box(text))) + }); + } + } + group.finish(); +} + +criterion_group!(benches, formats, literal_nesting); +criterion_main!(benches); diff --git a/litellm-rust/crates/python-compat/generated/nonprintable.rs b/litellm-rust/crates/python-compat/generated/nonprintable.rs new file mode 100644 index 00000000000..a044210f07d --- /dev/null +++ b/litellm-rust/crates/python-compat/generated/nonprintable.rs @@ -0,0 +1,745 @@ +// Generated by scripts/generate_nonprintable.py from Python 3.14.7 +// (Unicode 16.0.0). Do not edit by hand. + +pub(crate) const UNICODE_VERSION: &str = "16.0.0"; + +/// Inclusive code point ranges for which Python's `str.isprintable()` is false. +pub(crate) const NONPRINTABLE: [(u32, u32); 737] = [ + (0x0000, 0x001F), + (0x007F, 0x00A0), + (0x00AD, 0x00AD), + (0x0378, 0x0379), + (0x0380, 0x0383), + (0x038B, 0x038B), + (0x038D, 0x038D), + (0x03A2, 0x03A2), + (0x0530, 0x0530), + (0x0557, 0x0558), + (0x058B, 0x058C), + (0x0590, 0x0590), + (0x05C8, 0x05CF), + (0x05EB, 0x05EE), + (0x05F5, 0x0605), + (0x061C, 0x061C), + (0x06DD, 0x06DD), + (0x070E, 0x070F), + (0x074B, 0x074C), + (0x07B2, 0x07BF), + (0x07FB, 0x07FC), + (0x082E, 0x082F), + (0x083F, 0x083F), + (0x085C, 0x085D), + (0x085F, 0x085F), + (0x086B, 0x086F), + (0x088F, 0x0896), + (0x08E2, 0x08E2), + (0x0984, 0x0984), + (0x098D, 0x098E), + (0x0991, 0x0992), + (0x09A9, 0x09A9), + (0x09B1, 0x09B1), + (0x09B3, 0x09B5), + (0x09BA, 0x09BB), + (0x09C5, 0x09C6), + (0x09C9, 0x09CA), + (0x09CF, 0x09D6), + (0x09D8, 0x09DB), + (0x09DE, 0x09DE), + (0x09E4, 0x09E5), + (0x09FF, 0x0A00), + (0x0A04, 0x0A04), + (0x0A0B, 0x0A0E), + (0x0A11, 0x0A12), + (0x0A29, 0x0A29), + (0x0A31, 0x0A31), + (0x0A34, 0x0A34), + (0x0A37, 0x0A37), + (0x0A3A, 0x0A3B), + (0x0A3D, 0x0A3D), + (0x0A43, 0x0A46), + (0x0A49, 0x0A4A), + (0x0A4E, 0x0A50), + (0x0A52, 0x0A58), + (0x0A5D, 0x0A5D), + (0x0A5F, 0x0A65), + (0x0A77, 0x0A80), + (0x0A84, 0x0A84), + (0x0A8E, 0x0A8E), + (0x0A92, 0x0A92), + (0x0AA9, 0x0AA9), + (0x0AB1, 0x0AB1), + (0x0AB4, 0x0AB4), + (0x0ABA, 0x0ABB), + (0x0AC6, 0x0AC6), + (0x0ACA, 0x0ACA), + (0x0ACE, 0x0ACF), + (0x0AD1, 0x0ADF), + (0x0AE4, 0x0AE5), + (0x0AF2, 0x0AF8), + (0x0B00, 0x0B00), + (0x0B04, 0x0B04), + (0x0B0D, 0x0B0E), + (0x0B11, 0x0B12), + (0x0B29, 0x0B29), + (0x0B31, 0x0B31), + (0x0B34, 0x0B34), + (0x0B3A, 0x0B3B), + (0x0B45, 0x0B46), + (0x0B49, 0x0B4A), + (0x0B4E, 0x0B54), + (0x0B58, 0x0B5B), + (0x0B5E, 0x0B5E), + (0x0B64, 0x0B65), + (0x0B78, 0x0B81), + (0x0B84, 0x0B84), + (0x0B8B, 0x0B8D), + (0x0B91, 0x0B91), + (0x0B96, 0x0B98), + (0x0B9B, 0x0B9B), + (0x0B9D, 0x0B9D), + (0x0BA0, 0x0BA2), + (0x0BA5, 0x0BA7), + (0x0BAB, 0x0BAD), + (0x0BBA, 0x0BBD), + (0x0BC3, 0x0BC5), + (0x0BC9, 0x0BC9), + (0x0BCE, 0x0BCF), + (0x0BD1, 0x0BD6), + (0x0BD8, 0x0BE5), + (0x0BFB, 0x0BFF), + (0x0C0D, 0x0C0D), + (0x0C11, 0x0C11), + (0x0C29, 0x0C29), + (0x0C3A, 0x0C3B), + (0x0C45, 0x0C45), + (0x0C49, 0x0C49), + (0x0C4E, 0x0C54), + (0x0C57, 0x0C57), + (0x0C5B, 0x0C5C), + (0x0C5E, 0x0C5F), + (0x0C64, 0x0C65), + (0x0C70, 0x0C76), + (0x0C8D, 0x0C8D), + (0x0C91, 0x0C91), + (0x0CA9, 0x0CA9), + (0x0CB4, 0x0CB4), + (0x0CBA, 0x0CBB), + (0x0CC5, 0x0CC5), + (0x0CC9, 0x0CC9), + (0x0CCE, 0x0CD4), + (0x0CD7, 0x0CDC), + (0x0CDF, 0x0CDF), + (0x0CE4, 0x0CE5), + (0x0CF0, 0x0CF0), + (0x0CF4, 0x0CFF), + (0x0D0D, 0x0D0D), + (0x0D11, 0x0D11), + (0x0D45, 0x0D45), + (0x0D49, 0x0D49), + (0x0D50, 0x0D53), + (0x0D64, 0x0D65), + (0x0D80, 0x0D80), + (0x0D84, 0x0D84), + (0x0D97, 0x0D99), + (0x0DB2, 0x0DB2), + (0x0DBC, 0x0DBC), + (0x0DBE, 0x0DBF), + (0x0DC7, 0x0DC9), + (0x0DCB, 0x0DCE), + (0x0DD5, 0x0DD5), + (0x0DD7, 0x0DD7), + (0x0DE0, 0x0DE5), + (0x0DF0, 0x0DF1), + (0x0DF5, 0x0E00), + (0x0E3B, 0x0E3E), + (0x0E5C, 0x0E80), + (0x0E83, 0x0E83), + (0x0E85, 0x0E85), + (0x0E8B, 0x0E8B), + (0x0EA4, 0x0EA4), + (0x0EA6, 0x0EA6), + (0x0EBE, 0x0EBF), + (0x0EC5, 0x0EC5), + (0x0EC7, 0x0EC7), + (0x0ECF, 0x0ECF), + (0x0EDA, 0x0EDB), + (0x0EE0, 0x0EFF), + (0x0F48, 0x0F48), + (0x0F6D, 0x0F70), + (0x0F98, 0x0F98), + (0x0FBD, 0x0FBD), + (0x0FCD, 0x0FCD), + (0x0FDB, 0x0FFF), + (0x10C6, 0x10C6), + (0x10C8, 0x10CC), + (0x10CE, 0x10CF), + (0x1249, 0x1249), + (0x124E, 0x124F), + (0x1257, 0x1257), + (0x1259, 0x1259), + (0x125E, 0x125F), + (0x1289, 0x1289), + (0x128E, 0x128F), + (0x12B1, 0x12B1), + (0x12B6, 0x12B7), + (0x12BF, 0x12BF), + (0x12C1, 0x12C1), + (0x12C6, 0x12C7), + (0x12D7, 0x12D7), + (0x1311, 0x1311), + (0x1316, 0x1317), + (0x135B, 0x135C), + (0x137D, 0x137F), + (0x139A, 0x139F), + (0x13F6, 0x13F7), + (0x13FE, 0x13FF), + (0x1680, 0x1680), + (0x169D, 0x169F), + (0x16F9, 0x16FF), + (0x1716, 0x171E), + (0x1737, 0x173F), + (0x1754, 0x175F), + (0x176D, 0x176D), + (0x1771, 0x1771), + (0x1774, 0x177F), + (0x17DE, 0x17DF), + (0x17EA, 0x17EF), + (0x17FA, 0x17FF), + (0x180E, 0x180E), + (0x181A, 0x181F), + (0x1879, 0x187F), + (0x18AB, 0x18AF), + (0x18F6, 0x18FF), + (0x191F, 0x191F), + (0x192C, 0x192F), + (0x193C, 0x193F), + (0x1941, 0x1943), + (0x196E, 0x196F), + (0x1975, 0x197F), + (0x19AC, 0x19AF), + (0x19CA, 0x19CF), + (0x19DB, 0x19DD), + (0x1A1C, 0x1A1D), + (0x1A5F, 0x1A5F), + (0x1A7D, 0x1A7E), + (0x1A8A, 0x1A8F), + (0x1A9A, 0x1A9F), + (0x1AAE, 0x1AAF), + (0x1ACF, 0x1AFF), + (0x1B4D, 0x1B4D), + (0x1BF4, 0x1BFB), + (0x1C38, 0x1C3A), + (0x1C4A, 0x1C4C), + (0x1C8B, 0x1C8F), + (0x1CBB, 0x1CBC), + (0x1CC8, 0x1CCF), + (0x1CFB, 0x1CFF), + (0x1F16, 0x1F17), + (0x1F1E, 0x1F1F), + (0x1F46, 0x1F47), + (0x1F4E, 0x1F4F), + (0x1F58, 0x1F58), + (0x1F5A, 0x1F5A), + (0x1F5C, 0x1F5C), + (0x1F5E, 0x1F5E), + (0x1F7E, 0x1F7F), + (0x1FB5, 0x1FB5), + (0x1FC5, 0x1FC5), + (0x1FD4, 0x1FD5), + (0x1FDC, 0x1FDC), + (0x1FF0, 0x1FF1), + (0x1FF5, 0x1FF5), + (0x1FFF, 0x200F), + (0x2028, 0x202F), + (0x205F, 0x206F), + (0x2072, 0x2073), + (0x208F, 0x208F), + (0x209D, 0x209F), + (0x20C1, 0x20CF), + (0x20F1, 0x20FF), + (0x218C, 0x218F), + (0x242A, 0x243F), + (0x244B, 0x245F), + (0x2B74, 0x2B75), + (0x2B96, 0x2B96), + (0x2CF4, 0x2CF8), + (0x2D26, 0x2D26), + (0x2D28, 0x2D2C), + (0x2D2E, 0x2D2F), + (0x2D68, 0x2D6E), + (0x2D71, 0x2D7E), + (0x2D97, 0x2D9F), + (0x2DA7, 0x2DA7), + (0x2DAF, 0x2DAF), + (0x2DB7, 0x2DB7), + (0x2DBF, 0x2DBF), + (0x2DC7, 0x2DC7), + (0x2DCF, 0x2DCF), + (0x2DD7, 0x2DD7), + (0x2DDF, 0x2DDF), + (0x2E5E, 0x2E7F), + (0x2E9A, 0x2E9A), + (0x2EF4, 0x2EFF), + (0x2FD6, 0x2FEF), + (0x3000, 0x3000), + (0x3040, 0x3040), + (0x3097, 0x3098), + (0x3100, 0x3104), + (0x3130, 0x3130), + (0x318F, 0x318F), + (0x31E6, 0x31EE), + (0x321F, 0x321F), + (0xA48D, 0xA48F), + (0xA4C7, 0xA4CF), + (0xA62C, 0xA63F), + (0xA6F8, 0xA6FF), + (0xA7CE, 0xA7CF), + (0xA7D2, 0xA7D2), + (0xA7D4, 0xA7D4), + (0xA7DD, 0xA7F1), + (0xA82D, 0xA82F), + (0xA83A, 0xA83F), + (0xA878, 0xA87F), + (0xA8C6, 0xA8CD), + (0xA8DA, 0xA8DF), + (0xA954, 0xA95E), + (0xA97D, 0xA97F), + (0xA9CE, 0xA9CE), + (0xA9DA, 0xA9DD), + (0xA9FF, 0xA9FF), + (0xAA37, 0xAA3F), + (0xAA4E, 0xAA4F), + (0xAA5A, 0xAA5B), + (0xAAC3, 0xAADA), + (0xAAF7, 0xAB00), + (0xAB07, 0xAB08), + (0xAB0F, 0xAB10), + (0xAB17, 0xAB1F), + (0xAB27, 0xAB27), + (0xAB2F, 0xAB2F), + (0xAB6C, 0xAB6F), + (0xABEE, 0xABEF), + (0xABFA, 0xABFF), + (0xD7A4, 0xD7AF), + (0xD7C7, 0xD7CA), + (0xD7FC, 0xF8FF), + (0xFA6E, 0xFA6F), + (0xFADA, 0xFAFF), + (0xFB07, 0xFB12), + (0xFB18, 0xFB1C), + (0xFB37, 0xFB37), + (0xFB3D, 0xFB3D), + (0xFB3F, 0xFB3F), + (0xFB42, 0xFB42), + (0xFB45, 0xFB45), + (0xFBC3, 0xFBD2), + (0xFD90, 0xFD91), + (0xFDC8, 0xFDCE), + (0xFDD0, 0xFDEF), + (0xFE1A, 0xFE1F), + (0xFE53, 0xFE53), + (0xFE67, 0xFE67), + (0xFE6C, 0xFE6F), + (0xFE75, 0xFE75), + (0xFEFD, 0xFF00), + (0xFFBF, 0xFFC1), + (0xFFC8, 0xFFC9), + (0xFFD0, 0xFFD1), + (0xFFD8, 0xFFD9), + (0xFFDD, 0xFFDF), + (0xFFE7, 0xFFE7), + (0xFFEF, 0xFFFB), + (0xFFFE, 0xFFFF), + (0x1000C, 0x1000C), + (0x10027, 0x10027), + (0x1003B, 0x1003B), + (0x1003E, 0x1003E), + (0x1004E, 0x1004F), + (0x1005E, 0x1007F), + (0x100FB, 0x100FF), + (0x10103, 0x10106), + (0x10134, 0x10136), + (0x1018F, 0x1018F), + (0x1019D, 0x1019F), + (0x101A1, 0x101CF), + (0x101FE, 0x1027F), + (0x1029D, 0x1029F), + (0x102D1, 0x102DF), + (0x102FC, 0x102FF), + (0x10324, 0x1032C), + (0x1034B, 0x1034F), + (0x1037B, 0x1037F), + (0x1039E, 0x1039E), + (0x103C4, 0x103C7), + (0x103D6, 0x103FF), + (0x1049E, 0x1049F), + (0x104AA, 0x104AF), + (0x104D4, 0x104D7), + (0x104FC, 0x104FF), + (0x10528, 0x1052F), + (0x10564, 0x1056E), + (0x1057B, 0x1057B), + (0x1058B, 0x1058B), + (0x10593, 0x10593), + (0x10596, 0x10596), + (0x105A2, 0x105A2), + (0x105B2, 0x105B2), + (0x105BA, 0x105BA), + (0x105BD, 0x105BF), + (0x105F4, 0x105FF), + (0x10737, 0x1073F), + (0x10756, 0x1075F), + (0x10768, 0x1077F), + (0x10786, 0x10786), + (0x107B1, 0x107B1), + (0x107BB, 0x107FF), + (0x10806, 0x10807), + (0x10809, 0x10809), + (0x10836, 0x10836), + (0x10839, 0x1083B), + (0x1083D, 0x1083E), + (0x10856, 0x10856), + (0x1089F, 0x108A6), + (0x108B0, 0x108DF), + (0x108F3, 0x108F3), + (0x108F6, 0x108FA), + (0x1091C, 0x1091E), + (0x1093A, 0x1093E), + (0x10940, 0x1097F), + (0x109B8, 0x109BB), + (0x109D0, 0x109D1), + (0x10A04, 0x10A04), + (0x10A07, 0x10A0B), + (0x10A14, 0x10A14), + (0x10A18, 0x10A18), + (0x10A36, 0x10A37), + (0x10A3B, 0x10A3E), + (0x10A49, 0x10A4F), + (0x10A59, 0x10A5F), + (0x10AA0, 0x10ABF), + (0x10AE7, 0x10AEA), + (0x10AF7, 0x10AFF), + (0x10B36, 0x10B38), + (0x10B56, 0x10B57), + (0x10B73, 0x10B77), + (0x10B92, 0x10B98), + (0x10B9D, 0x10BA8), + (0x10BB0, 0x10BFF), + (0x10C49, 0x10C7F), + (0x10CB3, 0x10CBF), + (0x10CF3, 0x10CF9), + (0x10D28, 0x10D2F), + (0x10D3A, 0x10D3F), + (0x10D66, 0x10D68), + (0x10D86, 0x10D8D), + (0x10D90, 0x10E5F), + (0x10E7F, 0x10E7F), + (0x10EAA, 0x10EAA), + (0x10EAE, 0x10EAF), + (0x10EB2, 0x10EC1), + (0x10EC5, 0x10EFB), + (0x10F28, 0x10F2F), + (0x10F5A, 0x10F6F), + (0x10F8A, 0x10FAF), + (0x10FCC, 0x10FDF), + (0x10FF7, 0x10FFF), + (0x1104E, 0x11051), + (0x11076, 0x1107E), + (0x110BD, 0x110BD), + (0x110C3, 0x110CF), + (0x110E9, 0x110EF), + (0x110FA, 0x110FF), + (0x11135, 0x11135), + (0x11148, 0x1114F), + (0x11177, 0x1117F), + (0x111E0, 0x111E0), + (0x111F5, 0x111FF), + (0x11212, 0x11212), + (0x11242, 0x1127F), + (0x11287, 0x11287), + (0x11289, 0x11289), + (0x1128E, 0x1128E), + (0x1129E, 0x1129E), + (0x112AA, 0x112AF), + (0x112EB, 0x112EF), + (0x112FA, 0x112FF), + (0x11304, 0x11304), + (0x1130D, 0x1130E), + (0x11311, 0x11312), + (0x11329, 0x11329), + (0x11331, 0x11331), + (0x11334, 0x11334), + (0x1133A, 0x1133A), + (0x11345, 0x11346), + (0x11349, 0x1134A), + (0x1134E, 0x1134F), + (0x11351, 0x11356), + (0x11358, 0x1135C), + (0x11364, 0x11365), + (0x1136D, 0x1136F), + (0x11375, 0x1137F), + (0x1138A, 0x1138A), + (0x1138C, 0x1138D), + (0x1138F, 0x1138F), + (0x113B6, 0x113B6), + (0x113C1, 0x113C1), + (0x113C3, 0x113C4), + (0x113C6, 0x113C6), + (0x113CB, 0x113CB), + (0x113D6, 0x113D6), + (0x113D9, 0x113E0), + (0x113E3, 0x113FF), + (0x1145C, 0x1145C), + (0x11462, 0x1147F), + (0x114C8, 0x114CF), + (0x114DA, 0x1157F), + (0x115B6, 0x115B7), + (0x115DE, 0x115FF), + (0x11645, 0x1164F), + (0x1165A, 0x1165F), + (0x1166D, 0x1167F), + (0x116BA, 0x116BF), + (0x116CA, 0x116CF), + (0x116E4, 0x116FF), + (0x1171B, 0x1171C), + (0x1172C, 0x1172F), + (0x11747, 0x117FF), + (0x1183C, 0x1189F), + (0x118F3, 0x118FE), + (0x11907, 0x11908), + (0x1190A, 0x1190B), + (0x11914, 0x11914), + (0x11917, 0x11917), + (0x11936, 0x11936), + (0x11939, 0x1193A), + (0x11947, 0x1194F), + (0x1195A, 0x1199F), + (0x119A8, 0x119A9), + (0x119D8, 0x119D9), + (0x119E5, 0x119FF), + (0x11A48, 0x11A4F), + (0x11AA3, 0x11AAF), + (0x11AF9, 0x11AFF), + (0x11B0A, 0x11BBF), + (0x11BE2, 0x11BEF), + (0x11BFA, 0x11BFF), + (0x11C09, 0x11C09), + (0x11C37, 0x11C37), + (0x11C46, 0x11C4F), + (0x11C6D, 0x11C6F), + (0x11C90, 0x11C91), + (0x11CA8, 0x11CA8), + (0x11CB7, 0x11CFF), + (0x11D07, 0x11D07), + (0x11D0A, 0x11D0A), + (0x11D37, 0x11D39), + (0x11D3B, 0x11D3B), + (0x11D3E, 0x11D3E), + (0x11D48, 0x11D4F), + (0x11D5A, 0x11D5F), + (0x11D66, 0x11D66), + (0x11D69, 0x11D69), + (0x11D8F, 0x11D8F), + (0x11D92, 0x11D92), + (0x11D99, 0x11D9F), + (0x11DAA, 0x11EDF), + (0x11EF9, 0x11EFF), + (0x11F11, 0x11F11), + (0x11F3B, 0x11F3D), + (0x11F5B, 0x11FAF), + (0x11FB1, 0x11FBF), + (0x11FF2, 0x11FFE), + (0x1239A, 0x123FF), + (0x1246F, 0x1246F), + (0x12475, 0x1247F), + (0x12544, 0x12F8F), + (0x12FF3, 0x12FFF), + (0x13430, 0x1343F), + (0x13456, 0x1345F), + (0x143FB, 0x143FF), + (0x14647, 0x160FF), + (0x1613A, 0x167FF), + (0x16A39, 0x16A3F), + (0x16A5F, 0x16A5F), + (0x16A6A, 0x16A6D), + (0x16ABF, 0x16ABF), + (0x16ACA, 0x16ACF), + (0x16AEE, 0x16AEF), + (0x16AF6, 0x16AFF), + (0x16B46, 0x16B4F), + (0x16B5A, 0x16B5A), + (0x16B62, 0x16B62), + (0x16B78, 0x16B7C), + (0x16B90, 0x16D3F), + (0x16D7A, 0x16E3F), + (0x16E9B, 0x16EFF), + (0x16F4B, 0x16F4E), + (0x16F88, 0x16F8E), + (0x16FA0, 0x16FDF), + (0x16FE5, 0x16FEF), + (0x16FF2, 0x16FFF), + (0x187F8, 0x187FF), + (0x18CD6, 0x18CFE), + (0x18D09, 0x1AFEF), + (0x1AFF4, 0x1AFF4), + (0x1AFFC, 0x1AFFC), + (0x1AFFF, 0x1AFFF), + (0x1B123, 0x1B131), + (0x1B133, 0x1B14F), + (0x1B153, 0x1B154), + (0x1B156, 0x1B163), + (0x1B168, 0x1B16F), + (0x1B2FC, 0x1BBFF), + (0x1BC6B, 0x1BC6F), + (0x1BC7D, 0x1BC7F), + (0x1BC89, 0x1BC8F), + (0x1BC9A, 0x1BC9B), + (0x1BCA0, 0x1CBFF), + (0x1CCFA, 0x1CCFF), + (0x1CEB4, 0x1CEFF), + (0x1CF2E, 0x1CF2F), + (0x1CF47, 0x1CF4F), + (0x1CFC4, 0x1CFFF), + (0x1D0F6, 0x1D0FF), + (0x1D127, 0x1D128), + (0x1D173, 0x1D17A), + (0x1D1EB, 0x1D1FF), + (0x1D246, 0x1D2BF), + (0x1D2D4, 0x1D2DF), + (0x1D2F4, 0x1D2FF), + (0x1D357, 0x1D35F), + (0x1D379, 0x1D3FF), + (0x1D455, 0x1D455), + (0x1D49D, 0x1D49D), + (0x1D4A0, 0x1D4A1), + (0x1D4A3, 0x1D4A4), + (0x1D4A7, 0x1D4A8), + (0x1D4AD, 0x1D4AD), + (0x1D4BA, 0x1D4BA), + (0x1D4BC, 0x1D4BC), + (0x1D4C4, 0x1D4C4), + (0x1D506, 0x1D506), + (0x1D50B, 0x1D50C), + (0x1D515, 0x1D515), + (0x1D51D, 0x1D51D), + (0x1D53A, 0x1D53A), + (0x1D53F, 0x1D53F), + (0x1D545, 0x1D545), + (0x1D547, 0x1D549), + (0x1D551, 0x1D551), + (0x1D6A6, 0x1D6A7), + (0x1D7CC, 0x1D7CD), + (0x1DA8C, 0x1DA9A), + (0x1DAA0, 0x1DAA0), + (0x1DAB0, 0x1DEFF), + (0x1DF1F, 0x1DF24), + (0x1DF2B, 0x1DFFF), + (0x1E007, 0x1E007), + (0x1E019, 0x1E01A), + (0x1E022, 0x1E022), + (0x1E025, 0x1E025), + (0x1E02B, 0x1E02F), + (0x1E06E, 0x1E08E), + (0x1E090, 0x1E0FF), + (0x1E12D, 0x1E12F), + (0x1E13E, 0x1E13F), + (0x1E14A, 0x1E14D), + (0x1E150, 0x1E28F), + (0x1E2AF, 0x1E2BF), + (0x1E2FA, 0x1E2FE), + (0x1E300, 0x1E4CF), + (0x1E4FA, 0x1E5CF), + (0x1E5FB, 0x1E5FE), + (0x1E600, 0x1E7DF), + (0x1E7E7, 0x1E7E7), + (0x1E7EC, 0x1E7EC), + (0x1E7EF, 0x1E7EF), + (0x1E7FF, 0x1E7FF), + (0x1E8C5, 0x1E8C6), + (0x1E8D7, 0x1E8FF), + (0x1E94C, 0x1E94F), + (0x1E95A, 0x1E95D), + (0x1E960, 0x1EC70), + (0x1ECB5, 0x1ED00), + (0x1ED3E, 0x1EDFF), + (0x1EE04, 0x1EE04), + (0x1EE20, 0x1EE20), + (0x1EE23, 0x1EE23), + (0x1EE25, 0x1EE26), + (0x1EE28, 0x1EE28), + (0x1EE33, 0x1EE33), + (0x1EE38, 0x1EE38), + (0x1EE3A, 0x1EE3A), + (0x1EE3C, 0x1EE41), + (0x1EE43, 0x1EE46), + (0x1EE48, 0x1EE48), + (0x1EE4A, 0x1EE4A), + (0x1EE4C, 0x1EE4C), + (0x1EE50, 0x1EE50), + (0x1EE53, 0x1EE53), + (0x1EE55, 0x1EE56), + (0x1EE58, 0x1EE58), + (0x1EE5A, 0x1EE5A), + (0x1EE5C, 0x1EE5C), + (0x1EE5E, 0x1EE5E), + (0x1EE60, 0x1EE60), + (0x1EE63, 0x1EE63), + (0x1EE65, 0x1EE66), + (0x1EE6B, 0x1EE6B), + (0x1EE73, 0x1EE73), + (0x1EE78, 0x1EE78), + (0x1EE7D, 0x1EE7D), + (0x1EE7F, 0x1EE7F), + (0x1EE8A, 0x1EE8A), + (0x1EE9C, 0x1EEA0), + (0x1EEA4, 0x1EEA4), + (0x1EEAA, 0x1EEAA), + (0x1EEBC, 0x1EEEF), + (0x1EEF2, 0x1EFFF), + (0x1F02C, 0x1F02F), + (0x1F094, 0x1F09F), + (0x1F0AF, 0x1F0B0), + (0x1F0C0, 0x1F0C0), + (0x1F0D0, 0x1F0D0), + (0x1F0F6, 0x1F0FF), + (0x1F1AE, 0x1F1E5), + (0x1F203, 0x1F20F), + (0x1F23C, 0x1F23F), + (0x1F249, 0x1F24F), + (0x1F252, 0x1F25F), + (0x1F266, 0x1F2FF), + (0x1F6D8, 0x1F6DB), + (0x1F6ED, 0x1F6EF), + (0x1F6FD, 0x1F6FF), + (0x1F777, 0x1F77A), + (0x1F7DA, 0x1F7DF), + (0x1F7EC, 0x1F7EF), + (0x1F7F1, 0x1F7FF), + (0x1F80C, 0x1F80F), + (0x1F848, 0x1F84F), + (0x1F85A, 0x1F85F), + (0x1F888, 0x1F88F), + (0x1F8AE, 0x1F8AF), + (0x1F8BC, 0x1F8BF), + (0x1F8C2, 0x1F8FF), + (0x1FA54, 0x1FA5F), + (0x1FA6E, 0x1FA6F), + (0x1FA7D, 0x1FA7F), + (0x1FA8A, 0x1FA8E), + (0x1FAC7, 0x1FACD), + (0x1FADD, 0x1FADE), + (0x1FAEA, 0x1FAEF), + (0x1FAF9, 0x1FAFF), + (0x1FB93, 0x1FB93), + (0x1FBFA, 0x1FFFF), + (0x2A6E0, 0x2A6FF), + (0x2B73A, 0x2B73F), + (0x2B81E, 0x2B81F), + (0x2CEA2, 0x2CEAF), + (0x2EBE1, 0x2EBEF), + (0x2EE5E, 0x2F7FF), + (0x2FA1E, 0x2FFFF), + (0x3134B, 0x3134F), + (0x323B0, 0xE00FF), + (0xE01F0, 0x10FFFF), +]; diff --git a/litellm-rust/crates/python-compat/generated/values.json b/litellm-rust/crates/python-compat/generated/values.json new file mode 100644 index 00000000000..7f7e1e35a7d --- /dev/null +++ b/litellm-rust/crates/python-compat/generated/values.json @@ -0,0 +1,2164 @@ +{ + "python": "3.14.7", + "rows": [ + { + "name": "None", + "source": "None", + "literal": true, + "plain": true, + "repr": "None", + "str": "None", + "truthy": false, + "json": "null", + "pickle": { + "0": "4e2e", + "1": "4e2e", + "2": "80024e2e", + "3": "80034e2e", + "4": "80044e2e", + "5": "80054e2e" + }, + "view": "None" + }, + { + "name": "True", + "source": "True", + "literal": true, + "plain": true, + "repr": "True", + "str": "True", + "truthy": true, + "json": "true", + "pickle": { + "0": "4930310a2e", + "1": "4930310a2e", + "2": "8002882e", + "3": "8003882e", + "4": "8004882e", + "5": "8005882e" + }, + "view": "True" + }, + { + "name": "False", + "source": "False", + "literal": true, + "plain": true, + "repr": "False", + "str": "False", + "truthy": false, + "json": "false", + "pickle": { + "0": "4930300a2e", + "1": "4930300a2e", + "2": "8002892e", + "3": "8003892e", + "4": "8004892e", + "5": "8005892e" + }, + "view": "False" + }, + { + "name": "0", + "source": "0", + "literal": true, + "plain": true, + "repr": "0", + "str": "0", + "truthy": false, + "json": "0", + "pickle": { + "0": "49300a2e", + "1": "4b002e", + "2": "80024b002e", + "3": "80034b002e", + "4": "80044b002e", + "5": "80054b002e" + }, + "view": "0" + }, + { + "name": "-7", + "source": "-7", + "literal": true, + "plain": true, + "repr": "-7", + "str": "-7", + "truthy": true, + "json": "-7", + "pickle": { + "0": "492d370a2e", + "1": "4af9ffffff2e", + "2": "80024af9ffffff2e", + "3": "80034af9ffffff2e", + "4": "80049506000000000000004af9ffffff2e", + "5": "80059506000000000000004af9ffffff2e" + }, + "view": "-7" + }, + { + "name": "2**63 - 1", + "source": "2**63 - 1", + "literal": true, + "plain": true, + "repr": "9223372036854775807", + "str": "9223372036854775807", + "truthy": true, + "json": "9223372036854775807", + "pickle": { + "0": "4c393232333337323033363835343737353830374c0a2e", + "1": "4c393232333337323033363835343737353830374c0a2e", + "2": "80028a08ffffffffffffff7f2e", + "3": "80038a08ffffffffffffff7f2e", + "4": "8004950b000000000000008a08ffffffffffffff7f2e", + "5": "8005950b000000000000008a08ffffffffffffff7f2e" + }, + "view": "9223372036854775807" + }, + { + "name": "-(2**63)", + "source": "-(2**63)", + "literal": true, + "plain": true, + "repr": "-9223372036854775808", + "str": "-9223372036854775808", + "truthy": true, + "json": "-9223372036854775808", + "pickle": { + "0": "4c2d393232333337323033363835343737353830384c0a2e", + "1": "4c2d393232333337323033363835343737353830384c0a2e", + "2": "80028a0800000000000000802e", + "3": "80038a0800000000000000802e", + "4": "8004950b000000000000008a0800000000000000802e", + "5": "8005950b000000000000008a0800000000000000802e" + }, + "view": "-9223372036854775808" + }, + { + "name": "2**64", + "source": "2**64", + "literal": true, + "plain": true, + "repr": "18446744073709551616", + "str": "18446744073709551616", + "truthy": true, + "json": "18446744073709551616", + "pickle": { + "0": "4c31383434363734343037333730393535313631364c0a2e", + "1": "4c31383434363734343037333730393535313631364c0a2e", + "2": "80028a090000000000000000012e", + "3": "80038a090000000000000000012e", + "4": "8004950c000000000000008a090000000000000000012e", + "5": "8005950c000000000000008a090000000000000000012e" + }, + "view": "18446744073709551616" + }, + { + "name": "-(2**70)", + "source": "-(2**70)", + "literal": true, + "plain": true, + "repr": "-1180591620717411303424", + "str": "-1180591620717411303424", + "truthy": true, + "json": "-1180591620717411303424", + "pickle": { + "0": "4c2d313138303539313632303731373431313330333432344c0a2e", + "1": "4c2d313138303539313632303731373431313330333432344c0a2e", + "2": "80028a090000000000000000c02e", + "3": "80038a090000000000000000c02e", + "4": "8004950c000000000000008a090000000000000000c02e", + "5": "8005950c000000000000008a090000000000000000c02e" + }, + "view": "-1180591620717411303424" + }, + { + "name": "0.0", + "source": "0.0", + "literal": true, + "plain": true, + "repr": "0.0", + "str": "0.0", + "truthy": false, + "json": "0.0", + "pickle": { + "0": "46302e300a2e", + "1": "4700000000000000002e", + "2": "80024700000000000000002e", + "3": "80034700000000000000002e", + "4": "8004950a000000000000004700000000000000002e", + "5": "8005950a000000000000004700000000000000002e" + }, + "view": "0.0" + }, + { + "name": "-0.0", + "source": "-0.0", + "literal": true, + "plain": true, + "repr": "-0.0", + "str": "-0.0", + "truthy": false, + "json": "-0.0", + "pickle": { + "0": "462d302e300a2e", + "1": "4780000000000000002e", + "2": "80024780000000000000002e", + "3": "80034780000000000000002e", + "4": "8004950a000000000000004780000000000000002e", + "5": "8005950a000000000000004780000000000000002e" + }, + "view": "-0.0" + }, + { + "name": "0.2", + "source": "0.2", + "literal": true, + "plain": true, + "repr": "0.2", + "str": "0.2", + "truthy": true, + "json": "0.2", + "pickle": { + "0": "46302e320a2e", + "1": "473fc999999999999a2e", + "2": "8002473fc999999999999a2e", + "3": "8003473fc999999999999a2e", + "4": "8004950a00000000000000473fc999999999999a2e", + "5": "8005950a00000000000000473fc999999999999a2e" + }, + "view": "0.2" + }, + { + "name": "1.0", + "source": "1.0", + "literal": true, + "plain": true, + "repr": "1.0", + "str": "1.0", + "truthy": true, + "json": "1.0", + "pickle": { + "0": "46312e300a2e", + "1": "473ff00000000000002e", + "2": "8002473ff00000000000002e", + "3": "8003473ff00000000000002e", + "4": "8004950a00000000000000473ff00000000000002e", + "5": "8005950a00000000000000473ff00000000000002e" + }, + "view": "1.0" + }, + { + "name": "-1.5", + "source": "-1.5", + "literal": true, + "plain": true, + "repr": "-1.5", + "str": "-1.5", + "truthy": true, + "json": "-1.5", + "pickle": { + "0": "462d312e350a2e", + "1": "47bff80000000000002e", + "2": "800247bff80000000000002e", + "3": "800347bff80000000000002e", + "4": "8004950a0000000000000047bff80000000000002e", + "5": "8005950a0000000000000047bff80000000000002e" + }, + "view": "-1.5" + }, + { + "name": "0.1 + 0.2", + "source": "0.1 + 0.2", + "literal": true, + "plain": true, + "repr": "0.30000000000000004", + "str": "0.30000000000000004", + "truthy": true, + "json": "0.30000000000000004", + "pickle": { + "0": "46302e33303030303030303030303030303030340a2e", + "1": "473fd33333333333342e", + "2": "8002473fd33333333333342e", + "3": "8003473fd33333333333342e", + "4": "8004950a00000000000000473fd33333333333342e", + "5": "8005950a00000000000000473fd33333333333342e" + }, + "view": "0.30000000000000004" + }, + { + "name": "123456789.123", + "source": "123456789.123", + "literal": true, + "plain": true, + "repr": "123456789.123", + "str": "123456789.123", + "truthy": true, + "json": "123456789.123", + "pickle": { + "0": "463132333435363738392e3132330a2e", + "1": "47419d6f34547df3b62e", + "2": "800247419d6f34547df3b62e", + "3": "800347419d6f34547df3b62e", + "4": "8004950a0000000000000047419d6f34547df3b62e", + "5": "8005950a0000000000000047419d6f34547df3b62e" + }, + "view": "123456789.123" + }, + { + "name": "1e15", + "source": "1e15", + "literal": true, + "plain": true, + "repr": "1000000000000000.0", + "str": "1000000000000000.0", + "truthy": true, + "json": "1000000000000000.0", + "pickle": { + "0": "46313030303030303030303030303030302e300a2e", + "1": "47430c6bf5263400002e", + "2": "800247430c6bf5263400002e", + "3": "800347430c6bf5263400002e", + "4": "8004950a0000000000000047430c6bf5263400002e", + "5": "8005950a0000000000000047430c6bf5263400002e" + }, + "view": "1000000000000000.0" + }, + { + "name": "1e16", + "source": "1e16", + "literal": true, + "plain": true, + "repr": "1e+16", + "str": "1e+16", + "truthy": true, + "json": "1e+16", + "pickle": { + "0": "4631652b31360a2e", + "1": "474341c37937e080002e", + "2": "8002474341c37937e080002e", + "3": "8003474341c37937e080002e", + "4": "8004950a00000000000000474341c37937e080002e", + "5": "8005950a00000000000000474341c37937e080002e" + }, + "view": "1e+16" + }, + { + "name": "1.5e16", + "source": "1.5e16", + "literal": true, + "plain": true, + "repr": "1.5e+16", + "str": "1.5e+16", + "truthy": true, + "json": "1.5e+16", + "pickle": { + "0": "46312e35652b31360a2e", + "1": "47434aa535d3d0c0002e", + "2": "800247434aa535d3d0c0002e", + "3": "800347434aa535d3d0c0002e", + "4": "8004950a0000000000000047434aa535d3d0c0002e", + "5": "8005950a0000000000000047434aa535d3d0c0002e" + }, + "view": "1.5e+16" + }, + { + "name": "9999999999999998.0", + "source": "9999999999999998.0", + "literal": true, + "plain": true, + "repr": "9999999999999998.0", + "str": "9999999999999998.0", + "truthy": true, + "json": "9999999999999998.0", + "pickle": { + "0": "46393939393939393939393939393939382e300a2e", + "1": "474341c37937e07fff2e", + "2": "8002474341c37937e07fff2e", + "3": "8003474341c37937e07fff2e", + "4": "8004950a00000000000000474341c37937e07fff2e", + "5": "8005950a00000000000000474341c37937e07fff2e" + }, + "view": "9999999999999998.0" + }, + { + "name": "0.0001", + "source": "0.0001", + "literal": true, + "plain": true, + "repr": "0.0001", + "str": "0.0001", + "truthy": true, + "json": "0.0001", + "pickle": { + "0": "46302e303030310a2e", + "1": "473f1a36e2eb1c432d2e", + "2": "8002473f1a36e2eb1c432d2e", + "3": "8003473f1a36e2eb1c432d2e", + "4": "8004950a00000000000000473f1a36e2eb1c432d2e", + "5": "8005950a00000000000000473f1a36e2eb1c432d2e" + }, + "view": "0.0001" + }, + { + "name": "1e-05", + "source": "1e-05", + "literal": true, + "plain": true, + "repr": "1e-05", + "str": "1e-05", + "truthy": true, + "json": "1e-05", + "pickle": { + "0": "4631652d30350a2e", + "1": "473ee4f8b588e368f12e", + "2": "8002473ee4f8b588e368f12e", + "3": "8003473ee4f8b588e368f12e", + "4": "8004950a00000000000000473ee4f8b588e368f12e", + "5": "8005950a00000000000000473ee4f8b588e368f12e" + }, + "view": "1e-05" + }, + { + "name": "1.25e-07", + "source": "1.25e-07", + "literal": true, + "plain": true, + "repr": "1.25e-07", + "str": "1.25e-07", + "truthy": true, + "json": "1.25e-07", + "pickle": { + "0": "46312e3235652d30370a2e", + "1": "473e80c6f7a0b5ed8d2e", + "2": "8002473e80c6f7a0b5ed8d2e", + "3": "8003473e80c6f7a0b5ed8d2e", + "4": "8004950a00000000000000473e80c6f7a0b5ed8d2e", + "5": "8005950a00000000000000473e80c6f7a0b5ed8d2e" + }, + "view": "1.25e-07" + }, + { + "name": "5e-324", + "source": "5e-324", + "literal": true, + "plain": true, + "repr": "5e-324", + "str": "5e-324", + "truthy": true, + "json": "5e-324", + "pickle": { + "0": "4635652d3332340a2e", + "1": "4700000000000000012e", + "2": "80024700000000000000012e", + "3": "80034700000000000000012e", + "4": "8004950a000000000000004700000000000000012e", + "5": "8005950a000000000000004700000000000000012e" + }, + "view": "5e-324" + }, + { + "name": "1.7976931348623157e308", + "source": "1.7976931348623157e308", + "literal": true, + "plain": true, + "repr": "1.7976931348623157e+308", + "str": "1.7976931348623157e+308", + "truthy": true, + "json": "1.7976931348623157e+308", + "pickle": { + "0": "46312e37393736393331333438363233313537652b3330380a2e", + "1": "477fefffffffffffff2e", + "2": "8002477fefffffffffffff2e", + "3": "8003477fefffffffffffff2e", + "4": "8004950a00000000000000477fefffffffffffff2e", + "5": "8005950a00000000000000477fefffffffffffff2e" + }, + "view": "1.7976931348623157e+308" + }, + { + "name": "1e22", + "source": "1e22", + "literal": true, + "plain": true, + "repr": "1e+22", + "str": "1e+22", + "truthy": true, + "json": "1e+22", + "pickle": { + "0": "4631652b32320a2e", + "1": "474480f0cf064dd5922e", + "2": "8002474480f0cf064dd5922e", + "3": "8003474480f0cf064dd5922e", + "4": "8004950a00000000000000474480f0cf064dd5922e", + "5": "8005950a00000000000000474480f0cf064dd5922e" + }, + "view": "1e+22" + }, + { + "name": "float('inf')", + "source": "float('inf')", + "literal": false, + "plain": true, + "repr": "inf", + "str": "inf", + "truthy": true, + "json": "Infinity", + "pickle": { + "0": "46696e660a2e", + "1": "477ff00000000000002e", + "2": "8002477ff00000000000002e", + "3": "8003477ff00000000000002e", + "4": "8004950a00000000000000477ff00000000000002e", + "5": "8005950a00000000000000477ff00000000000002e" + }, + "view": "inf" + }, + { + "name": "float('-inf')", + "source": "float('-inf')", + "literal": false, + "plain": true, + "repr": "-inf", + "str": "-inf", + "truthy": true, + "json": "-Infinity", + "pickle": { + "0": "462d696e660a2e", + "1": "47fff00000000000002e", + "2": "800247fff00000000000002e", + "3": "800347fff00000000000002e", + "4": "8004950a0000000000000047fff00000000000002e", + "5": "8005950a0000000000000047fff00000000000002e" + }, + "view": "-inf" + }, + { + "name": "float('nan')", + "source": "float('nan')", + "literal": false, + "plain": true, + "repr": "nan", + "str": "nan", + "truthy": true, + "json": "NaN", + "pickle": { + "0": "466e616e0a2e", + "1": "477ff80000000000002e", + "2": "8002477ff80000000000002e", + "3": "8003477ff80000000000002e", + "4": "8004950a00000000000000477ff80000000000002e", + "5": "8005950a00000000000000477ff80000000000002e" + }, + "view": "nan" + }, + { + "name": "1j", + "source": "1j", + "literal": true, + "plain": false, + "repr": "1j", + "str": "1j", + "truthy": true, + "json_error": "Object of type complex is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a636f6d706c65780a70300a2846302e300a46312e300a7470310a5270320a2e", + "1": "635f5f6275696c74696e5f5f0a636f6d706c65780a710028470000000000000000473ff00000000000007471015271022e", + "2": "8002635f5f6275696c74696e5f5f0a636f6d706c65780a7100470000000000000000473ff00000000000008671015271022e", + "3": "8003636275696c74696e730a636f6d706c65780a7100470000000000000000473ff00000000000008671015271022e", + "4": "8004952e000000000000008c086275696c74696e73948c07636f6d706c6578949394470000000000000000473ff0000000000000869452942e", + "5": "8005952e000000000000008c086275696c74696e73948c07636f6d706c6578949394470000000000000000473ff0000000000000869452942e" + }, + "view": "1j" + }, + { + "name": "-1j", + "source": "-1j", + "literal": false, + "plain": false, + "repr": "(-0-1j)", + "str": "(-0-1j)", + "truthy": true, + "json_error": "Object of type complex is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a636f6d706c65780a70300a28462d302e300a462d312e300a7470310a5270320a2e", + "1": "635f5f6275696c74696e5f5f0a636f6d706c65780a71002847800000000000000047bff00000000000007471015271022e", + "2": "8002635f5f6275696c74696e5f5f0a636f6d706c65780a710047800000000000000047bff00000000000008671015271022e", + "3": "8003636275696c74696e730a636f6d706c65780a710047800000000000000047bff00000000000008671015271022e", + "4": "8004952e000000000000008c086275696c74696e73948c07636f6d706c657894939447800000000000000047bff0000000000000869452942e", + "5": "8005952e000000000000008c086275696c74696e73948c07636f6d706c657894939447800000000000000047bff0000000000000869452942e" + }, + "view": "(-0-1j)" + }, + { + "name": "complex(0, -1)", + "source": "complex(0, -1)", + "literal": false, + "plain": false, + "repr": "-1j", + "str": "-1j", + "truthy": true, + "json_error": "Object of type complex is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a636f6d706c65780a70300a2846302e300a462d312e300a7470310a5270320a2e", + "1": "635f5f6275696c74696e5f5f0a636f6d706c65780a71002847000000000000000047bff00000000000007471015271022e", + "2": "8002635f5f6275696c74696e5f5f0a636f6d706c65780a710047000000000000000047bff00000000000008671015271022e", + "3": "8003636275696c74696e730a636f6d706c65780a710047000000000000000047bff00000000000008671015271022e", + "4": "8004952e000000000000008c086275696c74696e73948c07636f6d706c657894939447000000000000000047bff0000000000000869452942e", + "5": "8005952e000000000000008c086275696c74696e73948c07636f6d706c657894939447000000000000000047bff0000000000000869452942e" + }, + "view": "-1j" + }, + { + "name": "1+2j", + "source": "1+2j", + "literal": true, + "plain": false, + "repr": "(1+2j)", + "str": "(1+2j)", + "truthy": true, + "json_error": "Object of type complex is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a636f6d706c65780a70300a2846312e300a46322e300a7470310a5270320a2e", + "1": "635f5f6275696c74696e5f5f0a636f6d706c65780a710028473ff00000000000004740000000000000007471015271022e", + "2": "8002635f5f6275696c74696e5f5f0a636f6d706c65780a7100473ff00000000000004740000000000000008671015271022e", + "3": "8003636275696c74696e730a636f6d706c65780a7100473ff00000000000004740000000000000008671015271022e", + "4": "8004952e000000000000008c086275696c74696e73948c07636f6d706c6578949394473ff0000000000000474000000000000000869452942e", + "5": "8005952e000000000000008c086275696c74696e73948c07636f6d706c6578949394473ff0000000000000474000000000000000869452942e" + }, + "view": "(1+2j)" + }, + { + "name": "-1.5-0.5j", + "source": "-1.5-0.5j", + "literal": true, + "plain": false, + "repr": "(-1.5-0.5j)", + "str": "(-1.5-0.5j)", + "truthy": true, + "json_error": "Object of type complex is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a636f6d706c65780a70300a28462d312e350a462d302e350a7470310a5270320a2e", + "1": "635f5f6275696c74696e5f5f0a636f6d706c65780a71002847bff800000000000047bfe00000000000007471015271022e", + "2": "8002635f5f6275696c74696e5f5f0a636f6d706c65780a710047bff800000000000047bfe00000000000008671015271022e", + "3": "8003636275696c74696e730a636f6d706c65780a710047bff800000000000047bfe00000000000008671015271022e", + "4": "8004952e000000000000008c086275696c74696e73948c07636f6d706c657894939447bff800000000000047bfe0000000000000869452942e", + "5": "8005952e000000000000008c086275696c74696e73948c07636f6d706c657894939447bff800000000000047bfe0000000000000869452942e" + }, + "view": "(-1.5-0.5j)" + }, + { + "name": "complex(0.0, 1e16)", + "source": "complex(0.0, 1e16)", + "literal": true, + "plain": false, + "repr": "1e+16j", + "str": "1e+16j", + "truthy": true, + "json_error": "Object of type complex is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a636f6d706c65780a70300a2846302e300a4631652b31360a7470310a5270320a2e", + "1": "635f5f6275696c74696e5f5f0a636f6d706c65780a710028470000000000000000474341c37937e080007471015271022e", + "2": "8002635f5f6275696c74696e5f5f0a636f6d706c65780a7100470000000000000000474341c37937e080008671015271022e", + "3": "8003636275696c74696e730a636f6d706c65780a7100470000000000000000474341c37937e080008671015271022e", + "4": "8004952e000000000000008c086275696c74696e73948c07636f6d706c6578949394470000000000000000474341c37937e08000869452942e", + "5": "8005952e000000000000008c086275696c74696e73948c07636f6d706c6578949394470000000000000000474341c37937e08000869452942e" + }, + "view": "1e+16j" + }, + { + "name": "''", + "source": "''", + "literal": true, + "plain": true, + "repr": "''", + "str": "", + "truthy": false, + "json": "\"\"", + "pickle": { + "0": "560a70300a2e", + "1": "580000000071002e", + "2": "8002580000000071002e", + "3": "8003580000000071002e", + "4": "80049504000000000000008c00942e", + "5": "80059504000000000000008c00942e" + }, + "view": "''" + }, + { + "name": "'plain'", + "source": "'plain'", + "literal": true, + "plain": true, + "repr": "'plain'", + "str": "plain", + "truthy": true, + "json": "\"plain\"", + "pickle": { + "0": "56706c61696e0a70300a2e", + "1": "5805000000706c61696e71002e", + "2": "80025805000000706c61696e71002e", + "3": "80035805000000706c61696e71002e", + "4": "80049509000000000000008c05706c61696e942e", + "5": "80059509000000000000008c05706c61696e942e" + }, + "view": "'plain'" + }, + { + "name": "\"it's\"", + "source": "\"it's\"", + "literal": true, + "plain": true, + "repr": "\"it's\"", + "str": "it's", + "truthy": true, + "json": "\"it's\"", + "pickle": { + "0": "56697427730a70300a2e", + "1": "58040000006974277371002e", + "2": "800258040000006974277371002e", + "3": "800358040000006974277371002e", + "4": "80049508000000000000008c0469742773942e", + "5": "80059508000000000000008c0469742773942e" + }, + "view": "\"it's\"" + }, + { + "name": "'say \"hi\"'", + "source": "'say \"hi\"'", + "literal": true, + "plain": true, + "repr": "'say \"hi\"'", + "str": "say \"hi\"", + "truthy": true, + "json": "\"say \\\"hi\\\"\"", + "pickle": { + "0": "5673617920226869220a70300a2e", + "1": "5808000000736179202268692271002e", + "2": "80025808000000736179202268692271002e", + "3": "80035808000000736179202268692271002e", + "4": "8004950c000000000000008c087361792022686922942e", + "5": "8005950c000000000000008c087361792022686922942e" + }, + "view": "'say \"hi\"'" + }, + { + "name": "'both \\' and \"'", + "source": "'both \\' and \"'", + "literal": true, + "plain": true, + "repr": "'both \\' and \"'", + "str": "both ' and \"", + "truthy": true, + "json": "\"both ' and \\\"\"", + "pickle": { + "0": "56626f7468202720616e6420220a70300a2e", + "1": "580c000000626f7468202720616e64202271002e", + "2": "8002580c000000626f7468202720616e64202271002e", + "3": "8003580c000000626f7468202720616e64202271002e", + "4": "80049510000000000000008c0c626f7468202720616e642022942e", + "5": "80059510000000000000008c0c626f7468202720616e642022942e" + }, + "view": "'both \\' and \"'" + }, + { + "name": "'back\\\\slash'", + "source": "'back\\\\slash'", + "literal": true, + "plain": true, + "repr": "'back\\\\slash'", + "str": "back\\slash", + "truthy": true, + "json": "\"back\\\\slash\"", + "pickle": { + "0": "566261636b5c7530303563736c6173680a70300a2e", + "1": "580a0000006261636b5c736c61736871002e", + "2": "8002580a0000006261636b5c736c61736871002e", + "3": "8003580a0000006261636b5c736c61736871002e", + "4": "8004950e000000000000008c0a6261636b5c736c617368942e", + "5": "8005950e000000000000008c0a6261636b5c736c617368942e" + }, + "view": "'back\\\\slash'" + }, + { + "name": "'\\t\\n\\r'", + "source": "'\\t\\n\\r'", + "literal": true, + "plain": true, + "repr": "'\\t\\n\\r'", + "str": "\t\n\r", + "truthy": true, + "json": "\"\\t\\n\\r\"", + "pickle": { + "0": "56095c75303030615c75303030640a70300a2e", + "1": "5803000000090a0d71002e", + "2": "80025803000000090a0d71002e", + "3": "80035803000000090a0d71002e", + "4": "80049507000000000000008c03090a0d942e", + "5": "80059507000000000000008c03090a0d942e" + }, + "view": "'\\t\\n\\r'" + }, + { + "name": "'\\x00\\x1f\\x7f'", + "source": "'\\x00\\x1f\\x7f'", + "literal": true, + "plain": true, + "repr": "'\\x00\\x1f\\x7f'", + "str": "\u0000\u001f\u007f", + "truthy": true, + "json": "\"\\u0000\\u001f\\u007f\"", + "pickle": { + "0": "565c75303030301f7f0a70300a2e", + "1": "5803000000001f7f71002e", + "2": "80025803000000001f7f71002e", + "3": "80035803000000001f7f71002e", + "4": "80049507000000000000008c03001f7f942e", + "5": "80059507000000000000008c03001f7f942e" + }, + "view": "'\\x00\\x1f\\x7f'" + }, + { + "name": "'\\x85\\xa0\\xad'", + "source": "'\\x85\\xa0\\xad'", + "literal": true, + "plain": true, + "repr": "'\\x85\\xa0\\xad'", + "str": "\u0085\u00a0\u00ad", + "truthy": true, + "json": "\"\\u0085\\u00a0\\u00ad\"", + "pickle": { + "0": "5685a0ad0a70300a2e", + "1": "5806000000c285c2a0c2ad71002e", + "2": "80025806000000c285c2a0c2ad71002e", + "3": "80035806000000c285c2a0c2ad71002e", + "4": "8004950a000000000000008c06c285c2a0c2ad942e", + "5": "8005950a000000000000008c06c285c2a0c2ad942e" + }, + "view": "'\\x85\\xa0\\xad'" + }, + { + "name": "'caf\\xe9'", + "source": "'caf\\xe9'", + "literal": true, + "plain": true, + "repr": "'caf\u00e9'", + "str": "caf\u00e9", + "truthy": true, + "json": "\"caf\\u00e9\"", + "pickle": { + "0": "56636166e90a70300a2e", + "1": "5805000000636166c3a971002e", + "2": "80025805000000636166c3a971002e", + "3": "80035805000000636166c3a971002e", + "4": "80049509000000000000008c05636166c3a9942e", + "5": "80059509000000000000008c05636166c3a9942e" + }, + "view": "'caf\u00e9'" + }, + { + "name": "'\\u65e5\\u672c'", + "source": "'\\u65e5\\u672c'", + "literal": true, + "plain": true, + "repr": "'\u65e5\u672c'", + "str": "\u65e5\u672c", + "truthy": true, + "json": "\"\\u65e5\\u672c\"", + "pickle": { + "0": "565c75363565355c75363732630a70300a2e", + "1": "5806000000e697a5e69cac71002e", + "2": "80025806000000e697a5e69cac71002e", + "3": "80035806000000e697a5e69cac71002e", + "4": "8004950a000000000000008c06e697a5e69cac942e", + "5": "8005950a000000000000008c06e697a5e69cac942e" + }, + "view": "'\u65e5\u672c'" + }, + { + "name": "'\\u200b\\u2028\\u3000'", + "source": "'\\u200b\\u2028\\u3000'", + "literal": true, + "plain": true, + "repr": "'\\u200b\\u2028\\u3000'", + "str": "\u200b\u2028\u3000", + "truthy": true, + "json": "\"\\u200b\\u2028\\u3000\"", + "pickle": { + "0": "565c75323030625c75323032385c75333030300a70300a2e", + "1": "5809000000e2808be280a8e3808071002e", + "2": "80025809000000e2808be280a8e3808071002e", + "3": "80035809000000e2808be280a8e3808071002e", + "4": "8004950d000000000000008c09e2808be280a8e38080942e", + "5": "8005950d000000000000008c09e2808be280a8e38080942e" + }, + "view": "'\\u200b\\u2028\\u3000'" + }, + { + "name": "'\\U0001f600'", + "source": "'\\U0001f600'", + "literal": true, + "plain": true, + "repr": "'\ud83d\ude00'", + "str": "\ud83d\ude00", + "truthy": true, + "json": "\"\\ud83d\\ude00\"", + "pickle": { + "0": "565c5530303031663630300a70300a2e", + "1": "5804000000f09f988071002e", + "2": "80025804000000f09f988071002e", + "3": "80035804000000f09f988071002e", + "4": "80049508000000000000008c04f09f9880942e", + "5": "80059508000000000000008c04f09f9880942e" + }, + "view": "'\ud83d\ude00'" + }, + { + "name": "'\\U000e0001\\U0010ffff'", + "source": "'\\U000e0001\\U0010ffff'", + "literal": true, + "plain": true, + "repr": "'\\U000e0001\\U0010ffff'", + "str": "\udb40\udc01\udbff\udfff", + "truthy": true, + "json": "\"\\udb40\\udc01\\udbff\\udfff\"", + "pickle": { + "0": "565c5530303065303030315c5530303130666666660a70300a2e", + "1": "5808000000f3a08081f48fbfbf71002e", + "2": "80025808000000f3a08081f48fbfbf71002e", + "3": "80035808000000f3a08081f48fbfbf71002e", + "4": "8004950c000000000000008c08f3a08081f48fbfbf942e", + "5": "8005950c000000000000008c08f3a08081f48fbfbf942e" + }, + "view": "'\\U000e0001\\U0010ffff'" + }, + { + "name": "'\\b\\f'", + "source": "'\\b\\f'", + "literal": true, + "plain": true, + "repr": "'\\x08\\x0c'", + "str": "\b\f", + "truthy": true, + "json": "\"\\b\\f\"", + "pickle": { + "0": "56080c0a70300a2e", + "1": "5802000000080c71002e", + "2": "80025802000000080c71002e", + "3": "80035802000000080c71002e", + "4": "80049506000000000000008c02080c942e", + "5": "80059506000000000000008c02080c942e" + }, + "view": "'\\x08\\x0c'" + }, + { + "name": "b''", + "source": "b''", + "literal": true, + "plain": true, + "repr": "b''", + "str": "b''", + "truthy": false, + "json_error": "Object of type bytes is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a62797465730a70300a28745270310a2e", + "1": "635f5f6275696c74696e5f5f0a62797465730a7100295271012e", + "2": "8002635f5f6275696c74696e5f5f0a62797465730a7100295271012e", + "3": "8003430071002e", + "4": "80049504000000000000004300942e", + "5": "80059504000000000000004300942e" + }, + "view": "b''" + }, + { + "name": "b'abc'", + "source": "b'abc'", + "literal": true, + "plain": true, + "repr": "b'abc'", + "str": "b'abc'", + "truthy": true, + "json_error": "Object of type bytes is not JSON serializable", + "pickle": { + "0": "635f636f646563730a656e636f64650a70300a28566162630a70310a566c6174696e310a70320a7470330a5270340a2e", + "1": "635f636f646563730a656e636f64650a7100285803000000616263710158060000006c6174696e3171027471035271042e", + "2": "8002635f636f646563730a656e636f64650a71005803000000616263710158060000006c6174696e3171028671035271042e", + "3": "8003430361626371002e", + "4": "80049507000000000000004303616263942e", + "5": "80059507000000000000004303616263942e" + }, + "view": "b'abc'" + }, + { + "name": "b\"a'b\"", + "source": "b\"a'b\"", + "literal": true, + "plain": true, + "repr": "b\"a'b\"", + "str": "b\"a'b\"", + "truthy": true, + "json_error": "Object of type bytes is not JSON serializable", + "pickle": { + "0": "635f636f646563730a656e636f64650a70300a28566127620a70310a566c6174696e310a70320a7470330a5270340a2e", + "1": "635f636f646563730a656e636f64650a7100285803000000612762710158060000006c6174696e3171027471035271042e", + "2": "8002635f636f646563730a656e636f64650a71005803000000612762710158060000006c6174696e3171028671035271042e", + "3": "8003430361276271002e", + "4": "80049507000000000000004303612762942e", + "5": "80059507000000000000004303612762942e" + }, + "view": "b\"a'b\"" + }, + { + "name": "b'a\"b\\'c'", + "source": "b'a\"b\\'c'", + "literal": true, + "plain": true, + "repr": "b'a\"b\\'c'", + "str": "b'a\"b\\'c'", + "truthy": true, + "json_error": "Object of type bytes is not JSON serializable", + "pickle": { + "0": "635f636f646563730a656e636f64650a70300a285661226227630a70310a566c6174696e310a70320a7470330a5270340a2e", + "1": "635f636f646563730a656e636f64650a71002858050000006122622763710158060000006c6174696e3171027471035271042e", + "2": "8002635f636f646563730a656e636f64650a710058050000006122622763710158060000006c6174696e3171028671035271042e", + "3": "80034305612262276371002e", + "4": "800495090000000000000043056122622763942e", + "5": "800595090000000000000043056122622763942e" + }, + "view": "b'a\"b\\'c'" + }, + { + "name": "b'\\x00\\t\\n\\r\\x7f\\x80\\xff'", + "source": "b'\\x00\\t\\n\\r\\x7f\\x80\\xff'", + "literal": true, + "plain": true, + "repr": "b'\\x00\\t\\n\\r\\x7f\\x80\\xff'", + "str": "b'\\x00\\t\\n\\r\\x7f\\x80\\xff'", + "truthy": true, + "json_error": "Object of type bytes is not JSON serializable", + "pickle": { + "0": "635f636f646563730a656e636f64650a70300a28565c7530303030095c75303030615c75303030647f80ff0a70310a566c6174696e310a70320a7470330a5270340a2e", + "1": "635f636f646563730a656e636f64650a710028580900000000090a0d7fc280c3bf710158060000006c6174696e3171027471035271042e", + "2": "8002635f636f646563730a656e636f64650a7100580900000000090a0d7fc280c3bf710158060000006c6174696e3171028671035271042e", + "3": "8003430700090a0d7f80ff71002e", + "4": "8004950b00000000000000430700090a0d7f80ff942e", + "5": "8005950b00000000000000430700090a0d7f80ff942e" + }, + "view": "b'\\x00\\t\\n\\r\\x7f\\x80\\xff'" + }, + { + "name": "[]", + "source": "[]", + "literal": true, + "plain": true, + "repr": "[]", + "str": "[]", + "truthy": false, + "json": "[]", + "pickle": { + "0": "286c70300a2e", + "1": "5d71002e", + "2": "80025d71002e", + "3": "80035d71002e", + "4": "80045d942e", + "5": "80055d942e" + }, + "view": "[]" + }, + { + "name": "[1, 'a', None, True]", + "source": "[1, 'a', None, True]", + "literal": true, + "plain": true, + "repr": "[1, 'a', None, True]", + "str": "[1, 'a', None, True]", + "truthy": true, + "json": "[1, \"a\", null, true]", + "pickle": { + "0": "286c70300a49310a6156610a70310a614e614930310a612e", + "1": "5d7100284b0158010000006171014e4930310a652e", + "2": "80025d7100284b0158010000006171014e88652e", + "3": "80035d7100284b0158010000006171014e88652e", + "4": "8004950d000000000000005d94284b018c0161944e88652e", + "5": "8005950d000000000000005d94284b018c0161944e88652e" + }, + "view": "[1, 'a', None, True]" + }, + { + "name": "()", + "source": "()", + "literal": true, + "plain": true, + "repr": "()", + "str": "()", + "truthy": false, + "json": "[]", + "pickle": { + "0": "28742e", + "1": "292e", + "2": "8002292e", + "3": "8003292e", + "4": "8004292e", + "5": "8005292e" + }, + "view": "[]" + }, + { + "name": "(1,)", + "source": "(1,)", + "literal": true, + "plain": true, + "repr": "(1,)", + "str": "(1,)", + "truthy": true, + "json": "[1]", + "pickle": { + "0": "2849310a7470300a2e", + "1": "284b017471002e", + "2": "80024b018571002e", + "3": "80034b018571002e", + "4": "80049505000000000000004b0185942e", + "5": "80059505000000000000004b0185942e" + }, + "view": "[1]" + }, + { + "name": "(1, (2, 3))", + "source": "(1, (2, 3))", + "literal": true, + "plain": true, + "repr": "(1, (2, 3))", + "str": "(1, (2, 3))", + "truthy": true, + "json": "[1, [2, 3]]", + "pickle": { + "0": "2849310a2849320a49330a7470300a7470310a2e", + "1": "284b01284b024b037471007471012e", + "2": "80024b014b024b038671008671012e", + "3": "80034b014b024b038671008671012e", + "4": "8004950b000000000000004b014b024b03869486942e", + "5": "8005950b000000000000004b014b024b03869486942e" + }, + "view": "[1, [2, 3]]" + }, + { + "name": "{}", + "source": "{}", + "literal": true, + "plain": true, + "repr": "{}", + "str": "{}", + "truthy": false, + "json": "{}", + "pickle": { + "0": "286470300a2e", + "1": "7d71002e", + "2": "80027d71002e", + "3": "80037d71002e", + "4": "80047d942e", + "5": "80057d942e" + }, + "view": "{}" + }, + { + "name": "{'a': 1, 'b': [1.0, 2.5]}", + "source": "{'a': 1, 'b': [1.0, 2.5]}", + "literal": true, + "plain": true, + "repr": "{'a': 1, 'b': [1.0, 2.5]}", + "str": "{'a': 1, 'b': [1.0, 2.5]}", + "truthy": true, + "json": "{\"a\": 1, \"b\": [1.0, 2.5]}", + "pickle": { + "0": "286470300a56610a70310a49310a7356620a70320a286c70330a46312e300a6146322e350a61732e", + "1": "7d71002858010000006171014b0158010000006271025d710328473ff000000000000047400400000000000065752e", + "2": "80027d71002858010000006171014b0158010000006271025d710328473ff000000000000047400400000000000065752e", + "3": "80037d71002858010000006171014b0158010000006271025d710328473ff000000000000047400400000000000065752e", + "4": "80049525000000000000007d94288c0161944b018c0162945d9428473ff000000000000047400400000000000065752e", + "5": "80059525000000000000007d94288c0161944b018c0162945d9428473ff000000000000047400400000000000065752e" + }, + "view": "{'a': 1, 'b': [1.0, 2.5]}" + }, + { + "name": "{'z': 1, 'a': 2, 'm': 3}", + "source": "{'z': 1, 'a': 2, 'm': 3}", + "literal": true, + "plain": true, + "repr": "{'z': 1, 'a': 2, 'm': 3}", + "str": "{'z': 1, 'a': 2, 'm': 3}", + "truthy": true, + "json": "{\"z\": 1, \"a\": 2, \"m\": 3}", + "pickle": { + "0": "286470300a567a0a70310a49310a7356610a70320a49320a73566d0a70330a49330a732e", + "1": "7d71002858010000007a71014b0158010000006171024b0258010000006d71034b03752e", + "2": "80027d71002858010000007a71014b0158010000006171024b0258010000006d71034b03752e", + "3": "80037d71002858010000007a71014b0158010000006171024b0258010000006d71034b03752e", + "4": "80049517000000000000007d94288c017a944b018c0161944b028c016d944b03752e", + "5": "80059517000000000000007d94288c017a944b018c0161944b028c016d944b03752e" + }, + "view": "{'z': 1, 'a': 2, 'm': 3}" + }, + { + "name": "{1: 'int', 2.5: 'float', True: 'bool', None: 'none'}", + "source": "{1: 'int', 2.5: 'float', True: 'bool', None: 'none'}", + "literal": true, + "plain": true, + "repr": "{1: 'bool', 2.5: 'float', None: 'none'}", + "str": "{1: 'bool', 2.5: 'float', None: 'none'}", + "truthy": true, + "json": "{\"1\": \"bool\", \"2.5\": \"float\", \"null\": \"none\"}", + "pickle": { + "0": "286470300a49310a56626f6f6c0a70310a7346322e350a56666c6f61740a70320a734e566e6f6e650a70330a732e", + "1": "7d7100284b015804000000626f6f6c71014740040000000000005805000000666c6f617471024e58040000006e6f6e657103752e", + "2": "80027d7100284b015804000000626f6f6c71014740040000000000005805000000666c6f617471024e58040000006e6f6e657103752e", + "3": "80037d7100284b015804000000626f6f6c71014740040000000000005805000000666c6f617471024e58040000006e6f6e657103752e", + "4": "80049527000000000000007d94284b018c04626f6f6c944740040000000000008c05666c6f6174944e8c046e6f6e6594752e", + "5": "80059527000000000000007d94284b018c04626f6f6c944740040000000000008c05666c6f6174944e8c046e6f6e6594752e" + }, + "view": "{1: 'bool', 2.5: 'float', None: 'none'}" + }, + { + "name": "{(1, 2): 'tuple key'}", + "source": "{(1, 2): 'tuple key'}", + "literal": true, + "plain": true, + "repr": "{(1, 2): 'tuple key'}", + "str": "{(1, 2): 'tuple key'}", + "truthy": true, + "json_error": "keys must be str, int, float, bool or None, not tuple", + "pickle": { + "0": "286470300a2849310a49320a7470310a567475706c65206b65790a70320a732e", + "1": "7d7100284b014b0274710158090000007475706c65206b65797102732e", + "2": "80027d71004b014b0286710158090000007475706c65206b65797102732e", + "3": "80037d71004b014b0286710158090000007475706c65206b65797102732e", + "4": "80049516000000000000007d944b014b0286948c097475706c65206b657994732e", + "5": "80059516000000000000007d944b014b0286948c097475706c65206b657994732e" + }, + "view": "{[1, 2]: 'tuple key'}" + }, + { + "name": "{'nested': {'deeper': {'deepest': [{}]}}}", + "source": "{'nested': {'deeper': {'deepest': [{}]}}}", + "literal": true, + "plain": true, + "repr": "{'nested': {'deeper': {'deepest': [{}]}}}", + "str": "{'nested': {'deeper': {'deepest': [{}]}}}", + "truthy": true, + "json": "{\"nested\": {\"deeper\": {\"deepest\": [{}]}}}", + "pickle": { + "0": "286470300a566e65737465640a70310a286470320a566465657065720a70330a286470340a56646565706573740a70350a286c70360a286470370a617373732e", + "1": "7d710058060000006e657374656471017d7102580600000064656570657271037d710458070000006465657065737471055d71067d7107617373732e", + "2": "80027d710058060000006e657374656471017d7102580600000064656570657271037d710458070000006465657065737471055d71067d7107617373732e", + "3": "80037d710058060000006e657374656471017d7102580600000064656570657271037d710458070000006465657065737471055d71067d7107617373732e", + "4": "8004952b000000000000007d948c066e6573746564947d948c06646565706572947d948c0764656570657374945d947d94617373732e", + "5": "8005952b000000000000007d948c066e6573746564947d948c06646565706572947d948c0764656570657374945d947d94617373732e" + }, + "view": "{'nested': {'deeper': {'deepest': [{}]}}}" + }, + { + "name": "{1}", + "source": "{1}", + "literal": true, + "plain": true, + "repr": "{1}", + "str": "{1}", + "truthy": true, + "json_error": "Object of type set is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a7365740a70300a28286c70310a49310a617470320a5270330a2e", + "1": "635f5f6275696c74696e5f5f0a7365740a7100285d71014b01617471025271032e", + "2": "8002635f5f6275696c74696e5f5f0a7365740a71005d71014b01618571025271032e", + "3": "8003636275696c74696e730a7365740a71005d71014b01618571025271032e", + "4": "80049507000000000000008f94284b01902e", + "5": "80059507000000000000008f94284b01902e" + }, + "view": "[1]" + }, + { + "name": "set()", + "source": "set()", + "literal": true, + "plain": true, + "repr": "set()", + "str": "set()", + "truthy": false, + "json_error": "Object of type set is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a7365740a70300a28286c70310a7470320a5270330a2e", + "1": "635f5f6275696c74696e5f5f0a7365740a7100285d71017471025271032e", + "2": "8002635f5f6275696c74696e5f5f0a7365740a71005d71018571025271032e", + "3": "8003636275696c74696e730a7365740a71005d71018571025271032e", + "4": "80048f942e", + "5": "80058f942e" + }, + "view": "[]" + }, + { + "name": "frozenset({1})", + "source": "frozenset({1})", + "literal": false, + "plain": true, + "repr": "frozenset({1})", + "str": "frozenset({1})", + "truthy": true, + "json_error": "Object of type frozenset is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a66726f7a656e7365740a70300a28286c70310a49310a617470320a5270330a2e", + "1": "635f5f6275696c74696e5f5f0a66726f7a656e7365740a7100285d71014b01617471025271032e", + "2": "8002635f5f6275696c74696e5f5f0a66726f7a656e7365740a71005d71014b01618571025271032e", + "3": "8003636275696c74696e730a66726f7a656e7365740a71005d71014b01618571025271032e", + "4": "8004950600000000000000284b0191942e", + "5": "8005950600000000000000284b0191942e" + }, + "view": "[1]" + }, + { + "name": "[[[[[[[[[[1]]]]]]]]]]", + "source": "[[[[[[[[[[1]]]]]]]]]]", + "literal": true, + "plain": true, + "repr": "[[[[[[[[[[1]]]]]]]]]]", + "str": "[[[[[[[[[[1]]]]]]]]]]", + "truthy": true, + "json": "[[[[[[[[[[1]]]]]]]]]]", + "pickle": { + "0": "286c70300a286c70310a286c70320a286c70330a286c70340a286c70350a286c70360a286c70370a286c70380a286c70390a49310a616161616161616161612e", + "1": "5d71005d71015d71025d71035d71045d71055d71065d71075d71085d71094b01616161616161616161612e", + "2": "80025d71005d71015d71025d71035d71045d71055d71065d71075d71085d71094b01616161616161616161612e", + "3": "80035d71005d71015d71025d71035d71045d71055d71065d71075d71085d71094b01616161616161616161612e", + "4": "80049521000000000000005d945d945d945d945d945d945d945d945d945d944b01616161616161616161612e", + "5": "80059521000000000000005d945d945d945d945d945d945d945d945d945d944b01616161616161616161612e" + }, + "view": "[[[[[[[[[[1]]]]]]]]]]" + }, + { + "name": "nested_150", + "source": "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]", + "literal": true, + "plain": true, + "repr": "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]", + "str": "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]", + "truthy": true, + "json": "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]", + "pickle": { + "0": "286c70300a286c70310a286c70320a286c70330a286c70340a286c70350a286c70360a286c70370a286c70380a286c70390a286c7031300a286c7031310a286c7031320a286c7031330a286c7031340a286c7031350a286c7031360a286c7031370a286c7031380a286c7031390a286c7032300a286c7032310a286c7032320a286c7032330a286c7032340a286c7032350a286c7032360a286c7032370a286c7032380a286c7032390a286c7033300a286c7033310a286c7033320a286c7033330a286c7033340a286c7033350a286c7033360a286c7033370a286c7033380a286c7033390a286c7034300a286c7034310a286c7034320a286c7034330a286c7034340a286c7034350a286c7034360a286c7034370a286c7034380a286c7034390a286c7035300a286c7035310a286c7035320a286c7035330a286c7035340a286c7035350a286c7035360a286c7035370a286c7035380a286c7035390a286c7036300a286c7036310a286c7036320a286c7036330a286c7036340a286c7036350a286c7036360a286c7036370a286c7036380a286c7036390a286c7037300a286c7037310a286c7037320a286c7037330a286c7037340a286c7037350a286c7037360a286c7037370a286c7037380a286c7037390a286c7038300a286c7038310a286c7038320a286c7038330a286c7038340a286c7038350a286c7038360a286c7038370a286c7038380a286c7038390a286c7039300a286c7039310a286c7039320a286c7039330a286c7039340a286c7039350a286c7039360a286c7039370a286c7039380a286c7039390a286c703130300a286c703130310a286c703130320a286c703130330a286c703130340a286c703130350a286c703130360a286c703130370a286c703130380a286c703130390a286c703131300a286c703131310a286c703131320a286c703131330a286c703131340a286c703131350a286c703131360a286c703131370a286c703131380a286c703131390a286c703132300a286c703132310a286c703132320a286c703132330a286c703132340a286c703132350a286c703132360a286c703132370a286c703132380a286c703132390a286c703133300a286c703133310a286c703133320a286c703133330a286c703133340a286c703133350a286c703133360a286c703133370a286c703133380a286c703133390a286c703134300a286c703134310a286c703134320a286c703134330a286c703134340a286c703134350a286c703134360a286c703134370a286c703134380a286c703134390a49310a6161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e", + "1": "5d71005d71015d71025d71035d71045d71055d71065d71075d71085d71095d710a5d710b5d710c5d710d5d710e5d710f5d71105d71115d71125d71135d71145d71155d71165d71175d71185d71195d711a5d711b5d711c5d711d5d711e5d711f5d71205d71215d71225d71235d71245d71255d71265d71275d71285d71295d712a5d712b5d712c5d712d5d712e5d712f5d71305d71315d71325d71335d71345d71355d71365d71375d71385d71395d713a5d713b5d713c5d713d5d713e5d713f5d71405d71415d71425d71435d71445d71455d71465d71475d71485d71495d714a5d714b5d714c5d714d5d714e5d714f5d71505d71515d71525d71535d71545d71555d71565d71575d71585d71595d715a5d715b5d715c5d715d5d715e5d715f5d71605d71615d71625d71635d71645d71655d71665d71675d71685d71695d716a5d716b5d716c5d716d5d716e5d716f5d71705d71715d71725d71735d71745d71755d71765d71775d71785d71795d717a5d717b5d717c5d717d5d717e5d717f5d71805d71815d71825d71835d71845d71855d71865d71875d71885d71895d718a5d718b5d718c5d718d5d718e5d718f5d71905d71915d71925d71935d71945d71954b016161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e", + "2": "80025d71005d71015d71025d71035d71045d71055d71065d71075d71085d71095d710a5d710b5d710c5d710d5d710e5d710f5d71105d71115d71125d71135d71145d71155d71165d71175d71185d71195d711a5d711b5d711c5d711d5d711e5d711f5d71205d71215d71225d71235d71245d71255d71265d71275d71285d71295d712a5d712b5d712c5d712d5d712e5d712f5d71305d71315d71325d71335d71345d71355d71365d71375d71385d71395d713a5d713b5d713c5d713d5d713e5d713f5d71405d71415d71425d71435d71445d71455d71465d71475d71485d71495d714a5d714b5d714c5d714d5d714e5d714f5d71505d71515d71525d71535d71545d71555d71565d71575d71585d71595d715a5d715b5d715c5d715d5d715e5d715f5d71605d71615d71625d71635d71645d71655d71665d71675d71685d71695d716a5d716b5d716c5d716d5d716e5d716f5d71705d71715d71725d71735d71745d71755d71765d71775d71785d71795d717a5d717b5d717c5d717d5d717e5d717f5d71805d71815d71825d71835d71845d71855d71865d71875d71885d71895d718a5d718b5d718c5d718d5d718e5d718f5d71905d71915d71925d71935d71945d71954b016161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e", + "3": "80035d71005d71015d71025d71035d71045d71055d71065d71075d71085d71095d710a5d710b5d710c5d710d5d710e5d710f5d71105d71115d71125d71135d71145d71155d71165d71175d71185d71195d711a5d711b5d711c5d711d5d711e5d711f5d71205d71215d71225d71235d71245d71255d71265d71275d71285d71295d712a5d712b5d712c5d712d5d712e5d712f5d71305d71315d71325d71335d71345d71355d71365d71375d71385d71395d713a5d713b5d713c5d713d5d713e5d713f5d71405d71415d71425d71435d71445d71455d71465d71475d71485d71495d714a5d714b5d714c5d714d5d714e5d714f5d71505d71515d71525d71535d71545d71555d71565d71575d71585d71595d715a5d715b5d715c5d715d5d715e5d715f5d71605d71615d71625d71635d71645d71655d71665d71675d71685d71695d716a5d716b5d716c5d716d5d716e5d716f5d71705d71715d71725d71735d71745d71755d71765d71775d71785d71795d717a5d717b5d717c5d717d5d717e5d717f5d71805d71815d71825d71835d71845d71855d71865d71875d71885d71895d718a5d718b5d718c5d718d5d718e5d718f5d71905d71915d71925d71935d71945d71954b016161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e", + "4": "800495c5010000000000005d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d944b016161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e", + "5": "800595c5010000000000005d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d944b016161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e" + }, + "view": "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]" + }, + { + "name": "{'timestamp': 1726000000.123, 'response': '{\"id\": \"chatcmpl-1\", \"object\": \"chat.completion\"}'}", + "source": "{'timestamp': 1726000000.123, 'response': '{\"id\": \"chatcmpl-1\", \"object\": \"chat.completion\"}'}", + "literal": true, + "plain": true, + "repr": "{'timestamp': 1726000000.123, 'response': '{\"id\": \"chatcmpl-1\", \"object\": \"chat.completion\"}'}", + "str": "{'timestamp': 1726000000.123, 'response': '{\"id\": \"chatcmpl-1\", \"object\": \"chat.completion\"}'}", + "truthy": true, + "json": "{\"timestamp\": 1726000000.123, \"response\": \"{\\\"id\\\": \\\"chatcmpl-1\\\", \\\"object\\\": \\\"chat.completion\\\"}\"}", + "pickle": { + "0": "286470300a5674696d657374616d700a70310a46313732363030303030302e3132330a7356726573706f6e73650a70320a567b226964223a202263686174636d706c2d31222c20226f626a656374223a2022636861742e636f6d706c6574696f6e227d0a70330a732e", + "1": "7d710028580900000074696d657374616d7071014741d9b82ae007df3b5808000000726573706f6e7365710258310000007b226964223a202263686174636d706c2d31222c20226f626a656374223a2022636861742e636f6d706c6574696f6e227d7103752e", + "2": "80027d710028580900000074696d657374616d7071014741d9b82ae007df3b5808000000726573706f6e7365710258310000007b226964223a202263686174636d706c2d31222c20226f626a656374223a2022636861742e636f6d706c6574696f6e227d7103752e", + "3": "80037d710028580900000074696d657374616d7071014741d9b82ae007df3b5808000000726573706f6e7365710258310000007b226964223a202263686174636d706c2d31222c20226f626a656374223a2022636861742e636f6d706c6574696f6e227d7103752e", + "4": "80049559000000000000007d94288c0974696d657374616d70944741d9b82ae007df3b8c08726573706f6e7365948c317b226964223a202263686174636d706c2d31222c20226f626a656374223a2022636861742e636f6d706c6574696f6e227d94752e", + "5": "80059559000000000000007d94288c0974696d657374616d70944741d9b82ae007df3b8c08726573706f6e7365948c317b226964223a202263686174636d706c2d31222c20226f626a656374223a2022636861742e636f6d706c6574696f6e227d94752e" + }, + "view": "{'timestamp': 1726000000.123, 'response': '{\"id\": \"chatcmpl-1\", \"object\": \"chat.completion\"}'}" + }, + { + "name": "{'model': 'gpt-4o', 'messages': [{'role': 'user', 'content': 'hi'}], 'temperature': 0.2, 'stream': False}", + "source": "{'model': 'gpt-4o', 'messages': [{'role': 'user', 'content': 'hi'}], 'temperature': 0.2, 'stream': False}", + "literal": true, + "plain": true, + "repr": "{'model': 'gpt-4o', 'messages': [{'role': 'user', 'content': 'hi'}], 'temperature': 0.2, 'stream': False}", + "str": "{'model': 'gpt-4o', 'messages': [{'role': 'user', 'content': 'hi'}], 'temperature': 0.2, 'stream': False}", + "truthy": true, + "json": "{\"model\": \"gpt-4o\", \"messages\": [{\"role\": \"user\", \"content\": \"hi\"}], \"temperature\": 0.2, \"stream\": false}", + "pickle": { + "0": "286470300a566d6f64656c0a70310a566770742d346f0a70320a73566d657373616765730a70330a286c70340a286470350a56726f6c650a70360a56757365720a70370a7356636f6e74656e740a70380a5668690a70390a7361735674656d70657261747572650a7031300a46302e320a735673747265616d0a7031310a4930300a732e", + "1": "7d71002858050000006d6f64656c710158060000006770742d346f710258080000006d6573736167657371035d71047d7105285804000000726f6c65710658040000007573657271075807000000636f6e74656e7471085802000000686971097561580b00000074656d7065726174757265710a473fc999999999999a580600000073747265616d710b4930300a752e", + "2": "80027d71002858050000006d6f64656c710158060000006770742d346f710258080000006d6573736167657371035d71047d7105285804000000726f6c65710658040000007573657271075807000000636f6e74656e7471085802000000686971097561580b00000074656d7065726174757265710a473fc999999999999a580600000073747265616d710b89752e", + "3": "80037d71002858050000006d6f64656c710158060000006770742d346f710258080000006d6573736167657371035d71047d7105285804000000726f6c65710658040000007573657271075807000000636f6e74656e7471085802000000686971097561580b00000074656d7065726174757265710a473fc999999999999a580600000073747265616d710b89752e", + "4": "80049566000000000000007d94288c056d6f64656c948c066770742d346f948c086d65737361676573945d947d94288c04726f6c65948c0475736572948c07636f6e74656e74948c0268699475618c0b74656d706572617475726594473fc999999999999a8c0673747265616d9489752e", + "5": "80059566000000000000007d94288c056d6f64656c948c066770742d346f948c086d65737361676573945d947d94288c04726f6c65948c0475736572948c07636f6e74656e74948c0268699475618c0b74656d706572617475726594473fc999999999999a8c0673747265616d9489752e" + }, + "view": "{'model': 'gpt-4o', 'messages': [{'role': 'user', 'content': 'hi'}], 'temperature': 0.2, 'stream': False}" + } + ], + "sources": [ + { + "name": "1", + "source": "1", + "repr": "1" + }, + { + "name": " 1", + "source": " 1", + "repr": "1" + }, + { + "name": "\t1", + "source": "\t1", + "repr": "1" + }, + { + "name": "\n1", + "source": "\n1", + "repr": "1" + }, + { + "name": " \n 1", + "source": " \n 1", + "error": "IndentationError" + }, + { + "name": "1\n", + "source": "1\n", + "repr": "1" + }, + { + "name": "1 # comment", + "source": "1 # comment", + "repr": "1" + }, + { + "name": "# c\n1", + "source": "# c\n1", + "repr": "1" + }, + { + "name": "1 \\\n", + "source": "1 \\\n", + "error": "SyntaxError" + }, + { + "name": "1,", + "source": "1,", + "repr": "(1,)" + }, + { + "name": "1, 2", + "source": "1, 2", + "repr": "(1, 2)" + }, + { + "name": "1,\n2", + "source": "1,\n2", + "error": "SyntaxError" + }, + { + "name": "(1,\n2)", + "source": "(1,\n2)", + "repr": "(1, 2)" + }, + { + "name": "[1,\n 2,\n]", + "source": "[1,\n 2,\n]", + "repr": "[1, 2]" + }, + { + "name": "()", + "source": "()", + "repr": "()" + }, + { + "name": "(1)", + "source": "(1)", + "repr": "1" + }, + { + "name": "((1,))", + "source": "((1,))", + "repr": "(1,)" + }, + { + "name": "(,)", + "source": "(,)", + "error": "SyntaxError" + }, + { + "name": "[,]", + "source": "[,]", + "error": "SyntaxError" + }, + { + "name": "{,}", + "source": "{,}", + "error": "SyntaxError" + }, + { + "name": "[1,]", + "source": "[1,]", + "repr": "[1]" + }, + { + "name": "{'a': 1,}", + "source": "{'a': 1,}", + "repr": "{'a': 1}" + }, + { + "name": "{1,}", + "source": "{1,}", + "repr": "{1}" + }, + { + "name": "{'a': 1 'b': 2}", + "source": "{'a': 1 'b': 2}", + "error": "SyntaxError" + }, + { + "name": "{1: 'a', True: 'b'}", + "source": "{1: 'a', True: 'b'}", + "repr": "{1: 'b'}" + }, + { + "name": "{1, True, 1.0}", + "source": "{1, True, 1.0}", + "repr": "{1}" + }, + { + "name": "{(1, 2): 'x', (1.0, 2): 'y'}", + "source": "{(1, 2): 'x', (1.0, 2): 'y'}", + "repr": "{(1, 2): 'y'}" + }, + { + "name": "{[1]: 2}", + "source": "{[1]: 2}", + "error": "TypeError" + }, + { + "name": "{{1}}", + "source": "{{1}}", + "error": "TypeError" + }, + { + "name": "{(1, [2])}", + "source": "{(1, [2])}", + "error": "TypeError" + }, + { + "name": "set()", + "source": "set()", + "repr": "set()" + }, + { + "name": "set( )", + "source": "set( )", + "repr": "set()" + }, + { + "name": "set([1])", + "source": "set([1])", + "error": "ValueError" + }, + { + "name": "frozenset()", + "source": "frozenset()", + "error": "ValueError" + }, + { + "name": "True", + "source": "True", + "repr": "True" + }, + { + "name": "False", + "source": "False", + "repr": "False" + }, + { + "name": "None", + "source": "None", + "repr": "None" + }, + { + "name": "Truex", + "source": "Truex", + "error": "ValueError" + }, + { + "name": "true", + "source": "true", + "error": "ValueError" + }, + { + "name": "...", + "source": "...", + "repr": "Ellipsis" + }, + { + "name": "0", + "source": "0", + "repr": "0" + }, + { + "name": "00", + "source": "00", + "repr": "0" + }, + { + "name": "0_0", + "source": "0_0", + "repr": "0" + }, + { + "name": "01", + "source": "01", + "error": "SyntaxError" + }, + { + "name": "007", + "source": "007", + "error": "SyntaxError" + }, + { + "name": "1_000", + "source": "1_000", + "repr": "1000" + }, + { + "name": "1_", + "source": "1_", + "error": "SyntaxError" + }, + { + "name": "1__0", + "source": "1__0", + "error": "SyntaxError" + }, + { + "name": "_1", + "source": "_1", + "error": "ValueError" + }, + { + "name": "0x1F", + "source": "0x1F", + "repr": "31" + }, + { + "name": "0X_1f", + "source": "0X_1f", + "repr": "31" + }, + { + "name": "0o17", + "source": "0o17", + "repr": "15" + }, + { + "name": "0b101", + "source": "0b101", + "repr": "5" + }, + { + "name": "0b102", + "source": "0b102", + "error": "SyntaxError" + }, + { + "name": "0x", + "source": "0x", + "error": "SyntaxError" + }, + { + "name": "1e3", + "source": "1e3", + "repr": "1000.0" + }, + { + "name": "1E-3", + "source": "1E-3", + "repr": "0.001" + }, + { + "name": "1e", + "source": "1e", + "error": "SyntaxError" + }, + { + "name": "1.e5", + "source": "1.e5", + "repr": "100000.0" + }, + { + "name": ".5", + "source": ".5", + "repr": "0.5" + }, + { + "name": "5.", + "source": "5.", + "repr": "5.0" + }, + { + "name": "1..", + "source": "1..", + "error": "SyntaxError" + }, + { + "name": "1.5.2", + "source": "1.5.2", + "error": "SyntaxError" + }, + { + "name": "1_0.0_1e1_0", + "source": "1_0.0_1e1_0", + "repr": "100100000000.0" + }, + { + "name": "1e999", + "source": "1e999", + "repr": "inf" + }, + { + "name": "-1e999", + "source": "-1e999", + "repr": "-inf" + }, + { + "name": "1j", + "source": "1j", + "repr": "1j" + }, + { + "name": "1.5J", + "source": "1.5J", + "repr": "1.5j" + }, + { + "name": "010j", + "source": "010j", + "repr": "10j" + }, + { + "name": "010.5", + "source": "010.5", + "repr": "10.5" + }, + { + "name": "1a", + "source": "1a", + "error": "SyntaxError" + }, + { + "name": "0x1g", + "source": "0x1g", + "error": "SyntaxError" + }, + { + "name": "-1", + "source": "-1", + "repr": "-1" + }, + { + "name": "+1", + "source": "+1", + "repr": "1" + }, + { + "name": "- 1", + "source": "- 1", + "repr": "-1" + }, + { + "name": "--1", + "source": "--1", + "error": "ValueError" + }, + { + "name": "-+1", + "source": "-+1", + "error": "ValueError" + }, + { + "name": "-(1)", + "source": "-(1)", + "repr": "-1" + }, + { + "name": "-(-1)", + "source": "-(-1)", + "error": "ValueError" + }, + { + "name": "-(1+2j)", + "source": "-(1+2j)", + "error": "ValueError" + }, + { + "name": "-True", + "source": "-True", + "error": "ValueError" + }, + { + "name": "-'a'", + "source": "-'a'", + "error": "ValueError" + }, + { + "name": "-[1]", + "source": "-[1]", + "error": "ValueError" + }, + { + "name": "1+2j", + "source": "1+2j", + "repr": "(1+2j)" + }, + { + "name": "1-2j", + "source": "1-2j", + "repr": "(1-2j)" + }, + { + "name": "1 + 2j", + "source": "1 + 2j", + "repr": "(1+2j)" + }, + { + "name": "(1)+(2j)", + "source": "(1)+(2j)", + "repr": "(1+2j)" + }, + { + "name": "1+2", + "source": "1+2", + "error": "ValueError" + }, + { + "name": "1+2+3j", + "source": "1+2+3j", + "error": "ValueError" + }, + { + "name": "1+-2j", + "source": "1+-2j", + "error": "ValueError" + }, + { + "name": "2j+1", + "source": "2j+1", + "error": "ValueError" + }, + { + "name": "True+1j", + "source": "True+1j", + "error": "ValueError" + }, + { + "name": "1-0j", + "source": "1-0j", + "repr": "(1-0j)" + }, + { + "name": "0.0-0j", + "source": "0.0-0j", + "repr": "-0j" + }, + { + "name": "-0.0+1j", + "source": "-0.0+1j", + "repr": "1j" + }, + { + "name": "-0.0", + "source": "-0.0", + "repr": "-0.0" + }, + { + "name": "-0", + "source": "-0", + "repr": "0" + }, + { + "name": "(-0.0)", + "source": "(-0.0)", + "repr": "-0.0" + }, + { + "name": "[-0.0, (-0.0)]", + "source": "[-0.0, (-0.0)]", + "repr": "[-0.0, -0.0]" + }, + { + "name": "2**3", + "source": "2**3", + "error": "ValueError" + }, + { + "name": "1*2", + "source": "1*2", + "error": "ValueError" + }, + { + "name": "1 if 1 else 2", + "source": "1 if 1 else 2", + "error": "ValueError" + }, + { + "name": "(1,)(2)", + "source": "(1,)(2)", + "error": "ValueError" + }, + { + "name": "''", + "source": "''", + "repr": "''" + }, + { + "name": "\"\"", + "source": "\"\"", + "repr": "''" + }, + { + "name": "'a' 'b'", + "source": "'a' 'b'", + "repr": "'ab'" + }, + { + "name": "'a' \"b\" '''c'''", + "source": "'a' \"b\" '''c'''", + "repr": "'abc'" + }, + { + "name": "'a' b'b'", + "source": "'a' b'b'", + "error": "SyntaxError" + }, + { + "name": "b'a' b'b'", + "source": "b'a' b'b'", + "repr": "b'ab'" + }, + { + "name": "u'x'", + "source": "u'x'", + "repr": "'x'" + }, + { + "name": "U'x'", + "source": "U'x'", + "repr": "'x'" + }, + { + "name": "r'x'", + "source": "r'x'", + "repr": "'x'" + }, + { + "name": "R'x'", + "source": "R'x'", + "repr": "'x'" + }, + { + "name": "b'x'", + "source": "b'x'", + "repr": "b'x'" + }, + { + "name": "B'x'", + "source": "B'x'", + "repr": "b'x'" + }, + { + "name": "br'x'", + "source": "br'x'", + "repr": "b'x'" + }, + { + "name": "Rb'x'", + "source": "Rb'x'", + "repr": "b'x'" + }, + { + "name": "rB'x'", + "source": "rB'x'", + "repr": "b'x'" + }, + { + "name": "ur'x'", + "source": "ur'x'", + "error": "SyntaxError" + }, + { + "name": "bu'x'", + "source": "bu'x'", + "error": "SyntaxError" + }, + { + "name": "f'x'", + "source": "f'x'", + "error": "ValueError" + }, + { + "name": "rf'x'", + "source": "rf'x'", + "error": "ValueError" + }, + { + "name": "'''a\nb'''", + "source": "'''a\nb'''", + "repr": "'a\\nb'" + }, + { + "name": "\"\"\"a\\\"\"\"\"", + "source": "\"\"\"a\\\"\"\"\"", + "repr": "'a\"'" + }, + { + "name": "'a\nb'", + "source": "'a\nb'", + "error": "SyntaxError" + }, + { + "name": "'a\\\nb'", + "source": "'a\\\nb'", + "repr": "'ab'" + }, + { + "name": "r'a\\\nb'", + "source": "r'a\\\nb'", + "repr": "'a\\\\\\nb'" + }, + { + "name": "'unterminated", + "source": "'unterminated", + "error": "SyntaxError" + }, + { + "name": "'\\a\\b\\f\\n\\r\\t\\v'", + "source": "'\\a\\b\\f\\n\\r\\t\\v'", + "repr": "'\\x07\\x08\\x0c\\n\\r\\t\\x0b'" + }, + { + "name": "'\\0\\12\\101\\1011'", + "source": "'\\0\\12\\101\\1011'", + "repr": "'\\x00\\nAA1'" + }, + { + "name": "'\\777'", + "source": "'\\777'", + "repr": "'\u01ff'" + }, + { + "name": "b'\\777'", + "source": "b'\\777'", + "repr": "b'\\xff'" + }, + { + "name": "b'\\400'", + "source": "b'\\400'", + "repr": "b'\\x00'" + }, + { + "name": "'\\x41'", + "source": "'\\x41'", + "repr": "'A'" + }, + { + "name": "'\\x4'", + "source": "'\\x4'", + "error": "SyntaxError" + }, + { + "name": "'\\u00e9'", + "source": "'\\u00e9'", + "repr": "'\u00e9'" + }, + { + "name": "'\\u00e'", + "source": "'\\u00e'", + "error": "SyntaxError" + }, + { + "name": "'\\U0001F600'", + "source": "'\\U0001F600'", + "repr": "'\ud83d\ude00'" + }, + { + "name": "'\\U00110000'", + "source": "'\\U00110000'", + "error": "SyntaxError" + }, + { + "name": "'\\ud800'", + "source": "'\\ud800'", + "repr": "'\\ud800'" + }, + { + "name": "'\\N{BULLET}'", + "source": "'\\N{BULLET}'", + "repr": "'\u2022'" + }, + { + "name": "'\\q'", + "source": "'\\q'", + "repr": "'\\\\q'" + }, + { + "name": "'\\\\'", + "source": "'\\\\'", + "repr": "'\\\\'" + }, + { + "name": "'\\''", + "source": "'\\''", + "repr": "\"'\"" + }, + { + "name": "\"\\\"\"", + "source": "\"\\\"\"", + "repr": "'\"'" + }, + { + "name": "b'\\u0041'", + "source": "b'\\u0041'", + "repr": "b'\\\\u0041'" + }, + { + "name": "b'\\x41\\xff'", + "source": "b'\\x41\\xff'", + "repr": "b'A\\xff'" + }, + { + "name": "b'caf\u00e9'", + "source": "b'caf\u00e9'", + "error": "SyntaxError" + }, + { + "name": "'caf\u00e9'", + "source": "'caf\u00e9'", + "repr": "'caf\u00e9'" + }, + { + "name": "r'\\d'", + "source": "r'\\d'", + "repr": "'\\\\d'" + }, + { + "name": "r'\\''", + "source": "r'\\''", + "repr": "\"\\\\'\"" + }, + { + "name": "rb'\\d'", + "source": "rb'\\d'", + "repr": "b'\\\\d'" + }, + { + "name": "r'\\'", + "source": "r'\\'", + "error": "SyntaxError" + }, + { + "name": "'\u65e5\ud83d\ude00'", + "source": "'\u65e5\ud83d\ude00'", + "repr": "'\u65e5\ud83d\ude00'" + } + ] +} diff --git a/litellm-rust/crates/python-compat/scripts/generate_fixtures.py b/litellm-rust/crates/python-compat/scripts/generate_fixtures.py new file mode 100644 index 00000000000..26456a638f6 --- /dev/null +++ b/litellm-rust/crates/python-compat/scripts/generate_fixtures.py @@ -0,0 +1,352 @@ +"""Regenerate generated/values.json: what CPython produces for each value in CORPUS. + + python scripts/generate_fixtures.py > generated/values.json + +Each row records `repr`, `str`, `json.dumps` (or its error), `bool`, and `pickle.dumps` at +every protocol. `literal` says whether `ast.literal_eval(repr(value))` gives the value back, +which is how Python reads `str(dict)` text back from a cache; the Rust tests reach the other +rows only through pickle. `view` is `repr` of the value as `pickle::loads` decodes it, with +tuples, sets, and frozensets rendered as lists. `sources` records `ast.literal_eval` on raw +source texts: its result, or the exception it raises. +""" + +import ast +import json +import pickle +import sys +import warnings + +# Entries are source texts, or `(name, source)` when the source is too long to read in a +# test report. `name` is what the Rust `KNOWN` table keys on. +CORPUS = [ + # Scalars + "None", + "True", + "False", + "0", + "-7", + "2**63 - 1", + "-(2**63)", + "2**64", + "-(2**70)", + # Floats around CPython's repr thresholds + "0.0", + "-0.0", + "0.2", + "1.0", + "-1.5", + "0.1 + 0.2", + "123456789.123", + "1e15", + "1e16", + "1.5e16", + "9999999999999998.0", + "0.0001", + "1e-05", + "1.25e-07", + "5e-324", + "1.7976931348623157e308", + "1e22", + "float('inf')", + "float('-inf')", + "float('nan')", + # Complex + "1j", + "-1j", + "complex(0, -1)", + "1+2j", + "-1.5-0.5j", + "complex(0.0, 1e16)", + # Strings: quote selection, escapes, printable and non-printable non-ASCII + "''", + "'plain'", + '"it\'s"', + "'say \"hi\"'", + "'both \\' and \"'", + "'back\\\\slash'", + "'\\t\\n\\r'", + "'\\x00\\x1f\\x7f'", + "'\\x85\\xa0\\xad'", + "'caf\\xe9'", + "'\\u65e5\\u672c'", + "'\\u200b\\u2028\\u3000'", + "'\\U0001f600'", + "'\\U000e0001\\U0010ffff'", + "'\\b\\f'", + # Bytes + "b''", + "b'abc'", + 'b"a\'b"', + "b'a\"b\\'c'", + "b'\\x00\\t\\n\\r\\x7f\\x80\\xff'", + # Containers + "[]", + "[1, 'a', None, True]", + "()", + "(1,)", + "(1, (2, 3))", + "{}", + "{'a': 1, 'b': [1.0, 2.5]}", + "{'z': 1, 'a': 2, 'm': 3}", + "{1: 'int', 2.5: 'float', True: 'bool', None: 'none'}", + "{(1, 2): 'tuple key'}", + "{'nested': {'deeper': {'deepest': [{}]}}}", + "{1}", + "set()", + "frozenset({1})", + "[[[[[[[[[[1]]]]]]]]]]", + # Deeper than the Rust decoders allow: CPython's parser accepts ~200 nested brackets + # and its unpickler has no limit, so this row records a deliberate divergence. + ("nested_150", "[" * 150 + "1" + "]" * 150), + # The shape LiteLLM caches + '{\'timestamp\': 1726000000.123, \'response\': \'{"id": "chatcmpl-1", "object": "chat.completion"}\'}', + "{'model': 'gpt-4o', 'messages': [{'role': 'user', 'content': 'hi'}], 'temperature': 0.2, 'stream': False}", +] + +# Source texts for `literal_eval` itself: tokenizer and evaluator edge cases, recorded with +# CPython's result or the exception it raises. Raw strings keep backslashes literal. +SOURCES = [ + # Layout: leading/trailing whitespace, comments, newlines, continuations + "1", + " 1", + "\t1", + "\n1", + " \n 1", + "1\n", + "1 # comment", + "# c\n1", + "1 \\\n", + "1,", + "1, 2", + "1,\n2", + "(1,\n2)", + "[1,\n 2,\n]", + # Containers and grouping + "()", + "(1)", + "((1,))", + "(,)", + "[,]", + "{,}", + "[1,]", + "{'a': 1,}", + "{1,}", + "{'a': 1 'b': 2}", + "{1: 'a', True: 'b'}", + "{1, True, 1.0}", + "{(1, 2): 'x', (1.0, 2): 'y'}", + "{[1]: 2}", + "{{1}}", + "{(1, [2])}", + "set()", + "set( )", + "set([1])", + "frozenset()", + # Names + "True", + "False", + "None", + "Truex", + "true", + "...", + # Integers and floats + "0", + "00", + "0_0", + "01", + "007", + "1_000", + "1_", + "1__0", + "_1", + "0x1F", + "0X_1f", + "0o17", + "0b101", + "0b102", + "0x", + "1e3", + "1E-3", + "1e", + "1.e5", + ".5", + "5.", + "1..", + "1.5.2", + "1_0.0_1e1_0", + "1e999", + "-1e999", + "1j", + "1.5J", + "010j", + "010.5", + "1a", + "0x1g", + # Signs and complex sums + "-1", + "+1", + "- 1", + "--1", + "-+1", + "-(1)", + "-(-1)", + "-(1+2j)", + "-True", + "-'a'", + "-[1]", + "1+2j", + "1-2j", + "1 + 2j", + "(1)+(2j)", + "1+2", + "1+2+3j", + "1+-2j", + "2j+1", + "True+1j", + "1-0j", + "0.0-0j", + "-0.0+1j", + "-0.0", + "-0", + "(-0.0)", + "[-0.0, (-0.0)]", + "2**3", + "1*2", + "1 if 1 else 2", + "(1,)(2)", + # String prefixes, quoting, and concatenation + "''", + '""', + "'a' 'b'", + "'a' \"b\" '''c'''", + "'a' b'b'", + "b'a' b'b'", + "u'x'", + "U'x'", + "r'x'", + "R'x'", + "b'x'", + "B'x'", + "br'x'", + "Rb'x'", + "rB'x'", + "ur'x'", + "bu'x'", + "f'x'", + "rf'x'", + "'''a\nb'''", + '"""a\\""""', + "'a\nb'", + "'a\\\nb'", + "r'a\\\nb'", + "'unterminated", + # Escapes + r"'\a\b\f\n\r\t\v'", + r"'\0\12\101\1011'", + r"'\777'", + r"b'\777'", + r"b'\400'", + r"'\x41'", + r"'\x4'", + r"'\u00e9'", + r"'\u00e'", + r"'\U0001F600'", + r"'\U00110000'", + r"'\ud800'", + r"'\N{BULLET}'", + r"'\q'", + r"'\\'", + r"'\''", + r'"\""', + r"b'\u0041'", + r"b'\x41\xff'", + "b'café'", + "'café'", + r"r'\d'", + r"r'\''", + r"rb'\d'", + r"r'\'", + "'日\U0001f600'", +] + + +def named(entry): + """Split a corpus entry into its report name and its source text.""" + if isinstance(entry, tuple): + return entry + return entry, entry + + +def evaluate(entry): + name, source = named(entry) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + try: + return {"name": name, "source": source, "repr": repr(ast.literal_eval(source))} + except Exception as error: # noqa: BLE001 - recorded, not raised + return {"name": name, "source": source, "error": type(error).__name__} + + +def view(value): + if isinstance(value, (list, tuple, set, frozenset)): + return "[" + ", ".join(view(item) for item in value) + "]" + if isinstance(value, dict): + return "{" + ", ".join(f"{view(key)}: {view(item)}" for key, item in value.items()) + "}" + return repr(value) + + +def plain(value): + """Whether pickle can encode the value without a class reference such as `complex`.""" + if isinstance(value, complex): + return False + if isinstance(value, (list, tuple, set, frozenset)): + return all(plain(item) for item in value) + if isinstance(value, dict): + return all(plain(key) and plain(item) for key, item in value.items()) + return True + + +def is_literal(value): + try: + parsed = ast.literal_eval(repr(value)) + except (ValueError, SyntaxError): + return False + return repr(parsed) == repr(value) + + +def row(entry): + name, source = named(entry) + value = eval(source) + entry = { + "name": name, + "source": source, + "literal": is_literal(value), + "plain": plain(value), + "repr": repr(value), + "str": str(value), + "truthy": bool(value), + } + try: + entry["json"] = json.dumps(value) + except (TypeError, ValueError) as error: + entry["json_error"] = str(error) + try: + entry["pickle"] = {str(protocol): pickle.dumps(value, protocol=protocol).hex() for protocol in range(6)} + entry["view"] = view(value) + except Exception as error: # noqa: BLE001 - recorded, not raised + entry["pickle_error"] = f"{type(error).__name__}: {error}" + return entry + + +if __name__ == "__main__": + json.dump( + { + "python": sys.version.split()[0], + "rows": [row(entry) for entry in CORPUS], + "sources": [evaluate(entry) for entry in SOURCES], + }, + sys.stdout, + indent=2, + ensure_ascii=True, + ) + sys.stdout.write("\n") diff --git a/litellm-rust/crates/python-compat/scripts/generate_nonprintable.py b/litellm-rust/crates/python-compat/scripts/generate_nonprintable.py new file mode 100644 index 00000000000..70d1fbd2dd1 --- /dev/null +++ b/litellm-rust/crates/python-compat/scripts/generate_nonprintable.py @@ -0,0 +1,35 @@ +"""Regenerate generated/nonprintable.rs from this interpreter's `str.isprintable`. + +`repr(str)` escapes exactly the characters for which `str.isprintable()` is false, so the +table must come from the Python version the gateway interoperates with. + + python scripts/generate_nonprintable.py > generated/nonprintable.rs +""" + +import sys +import unicodedata + +ranges = [] +start = None +for code in range(0x110000): + printable = chr(code).isprintable() + if not printable and start is None: + start = code + elif printable and start is not None: + ranges.append((start, code - 1)) + start = None +if start is not None: + ranges.append((start, 0x10FFFF)) + +lines = [ + f"// Generated by scripts/generate_nonprintable.py from Python {sys.version.split()[0]}", + f"// (Unicode {unicodedata.unidata_version}). Do not edit by hand.", + "", + f'pub(crate) const UNICODE_VERSION: &str = "{unicodedata.unidata_version}";', + "", + "/// Inclusive code point ranges for which Python's `str.isprintable()` is false.", + f"pub(crate) const NONPRINTABLE: [(u32, u32); {len(ranges)}] = [", + *(f" (0x{low:04X}, 0x{high:04X})," for low, high in ranges), + "];", +] +sys.stdout.write("\n".join(lines) + "\n") diff --git a/litellm-rust/crates/python-compat/scripts/verify_rust_pickles.py b/litellm-rust/crates/python-compat/scripts/verify_rust_pickles.py new file mode 100644 index 00000000000..793acad61f9 --- /dev/null +++ b/litellm-rust/crates/python-compat/scripts/verify_rust_pickles.py @@ -0,0 +1,43 @@ +"""Check that CPython unpickles what `pickle::dumps` writes, to the value it was given. + + PYTHON_COMPAT_RUST_PICKLES=rust.tsv cargo test -p litellm-python-compat --test fixtures + python scripts/verify_rust_pickles.py rust.tsv + +The rows are plain data by construction, so this refuses to resolve any class rather than +handing file-controlled bytes to an unrestricted `pickle.loads`. +""" + +import ast +import io +import pickle +import sys + + +class PlainDataUnpickler(pickle.Unpickler): + """An unpickler with `GLOBAL`/`REDUCE` disabled, mirroring `pickle::loads` in Rust.""" + + def find_class(self, module, name): + raise pickle.UnpicklingError(f"refusing to resolve {module}.{name}") + + +def loads(data): + return PlainDataUnpickler(io.BytesIO(data)).load() + + +def main(path): + failures = 0 + rows = 0 + with open(path, encoding="utf-8") as lines: + for line in lines: + data, expected = line.rstrip("\n").split("\t", 1) + rows += 1 + actual = repr(loads(bytes.fromhex(data))) + if actual != repr(ast.literal_eval(expected)): + failures += 1 + sys.stdout.write(f"mismatch: expected {expected}, got {actual}\n") + sys.stdout.write(f"{rows} Rust pickles checked, {failures} mismatches\n") + return 1 if failures or not rows else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1])) diff --git a/litellm-rust/crates/python-compat/src/error.rs b/litellm-rust/crates/python-compat/src/error.rs new file mode 100644 index 00000000000..43990eb7602 --- /dev/null +++ b/litellm-rust/crates/python-compat/src/error.rs @@ -0,0 +1,25 @@ +use crate::MAX_DEPTH; + +/// A failure to read or write a Python format. Messages quote CPython's own wording where +/// the Python side raises (`TypeError`, `ValueError`), so callers can log them as is. +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("malformed Python literal at byte {0}")] + InvalidLiteral(usize), + #[error("unhashable type: '{0}'")] + Unhashable(&'static str), + #[error("value nests deeper than {MAX_DEPTH} levels")] + TooDeep, + #[error("invalid pickle: {0}")] + InvalidPickle(String), + #[error("Object of type {0} is not JSON serializable")] + NotJsonSerializable(&'static str), + #[error("keys must be str, int, float, bool or None, not {0}")] + InvalidJsonKey(&'static str), + #[error("Out of range float values are not JSON compliant")] + NonFiniteFloat, + #[error("integer does not fit in the JSON number range")] + IntegerOutOfRange, + #[error("Object of type {0} cannot be pickled as plain data")] + NotPicklable(&'static str), +} diff --git a/litellm-rust/crates/python-compat/src/json.rs b/litellm-rust/crates/python-compat/src/json.rs new file mode 100644 index 00000000000..d3436519ac4 --- /dev/null +++ b/litellm-rust/crates/python-compat/src/json.rs @@ -0,0 +1,168 @@ +//! `json.dumps` with CPython's default options, and the JSON value `json.loads` returns. +//! +//! Defaults are `ensure_ascii=True`, `allow_nan=True`, separators `(", ", ": ")`, and no key +//! sorting. Python's `json.loads` is mapped by [`from_json`]; a `serde_json` number that does +//! not fit `i64` or `u64` arrives as a float, where Python would keep an `int`. + +use std::fmt::Write; + +use serde_json::{Map, Number}; + +use crate::{Error, Value, repr::float_repr}; + +/// `json.dumps(value)`. +pub fn dumps(value: &Value) -> Result { + let mut out = String::new(); + write_value(&mut out, value)?; + Ok(out) +} + +/// The Python value `json.loads` returns for a JSON document. +pub fn from_json(value: serde_json::Value) -> Value { + match value { + serde_json::Value::Null => Value::None, + serde_json::Value::Bool(value) => Value::Bool(value), + serde_json::Value::Number(number) => { + if let Some(value) = number.as_i64() { + Value::Int(value.into()) + } else if let Some(value) = number.as_u64() { + Value::Int(value.into()) + } else { + Value::Float(number.as_f64().unwrap_or(f64::NAN)) + } + } + serde_json::Value::String(text) => Value::Str(text), + serde_json::Value::Array(values) => { + Value::List(values.into_iter().map(from_json).collect()) + } + serde_json::Value::Object(entries) => Value::Dict( + entries + .into_iter() + .map(|(key, value)| (Value::Str(key), from_json(value))) + .collect(), + ), + } +} + +/// `json.loads(json.dumps(value))` as a `serde_json` value: tuples become arrays and dict +/// keys are coerced to strings as `json.dumps` does. Non-finite floats, which Python writes +/// as `NaN` and `Infinity`, have no `serde_json` form and fail with [`Error::NonFiniteFloat`]. +pub fn to_json(value: &Value) -> Result { + Ok(match value { + Value::None => serde_json::Value::Null, + Value::Bool(value) => serde_json::Value::Bool(*value), + Value::Int(value) => { + let text = value.to_string(); + serde_json::Value::Number( + text.parse::() + .map(Number::from) + .or_else(|_| text.parse::().map(Number::from)) + .map_err(|_| Error::IntegerOutOfRange)?, + ) + } + Value::Float(value) => { + serde_json::Value::Number(Number::from_f64(*value).ok_or(Error::NonFiniteFloat)?) + } + Value::Str(text) => serde_json::Value::String(text.clone()), + Value::List(values) | Value::Tuple(values) => { + serde_json::Value::Array(values.iter().map(to_json).collect::, _>>()?) + } + Value::Dict(entries) => serde_json::Value::Object( + entries + .iter() + .map(|(key, value)| Ok((json_key(key)?, to_json(value)?))) + .collect::, Error>>()?, + ), + value @ (Value::Bytes(_) | Value::Set(_) | Value::Complex { .. }) => { + return Err(Error::NotJsonSerializable(value.type_name())); + } + }) +} + +fn write_value(out: &mut String, value: &Value) -> Result<(), Error> { + match value { + Value::None => out.push_str("null"), + Value::Bool(true) => out.push_str("true"), + Value::Bool(false) => out.push_str("false"), + Value::Int(value) => { + let _ = write!(out, "{value}"); + } + Value::Float(value) => out.push_str(&float_text(*value)), + Value::Str(text) => write_string(out, text), + Value::List(values) | Value::Tuple(values) => { + out.push('['); + for (index, value) in values.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + write_value(out, value)?; + } + out.push(']'); + } + Value::Dict(entries) => { + out.push('{'); + for (index, (key, value)) in entries.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + write_string(out, &json_key(key)?); + out.push_str(": "); + write_value(out, value)?; + } + out.push('}'); + } + value @ (Value::Bytes(_) | Value::Set(_) | Value::Complex { .. }) => { + return Err(Error::NotJsonSerializable(value.type_name())); + } + } + Ok(()) +} + +/// `json.encoder.JSONEncoder.iterencode`'s `floatstr` with `allow_nan=True`. +fn float_text(value: f64) -> String { + if value.is_nan() { + "NaN".to_owned() + } else if value.is_infinite() { + if value > 0.0 { "Infinity" } else { "-Infinity" }.to_owned() + } else { + float_repr(value) + } +} + +/// Dict key coercion in `json.dumps`: scalars become their JSON text, other keys fail. +fn json_key(key: &Value) -> Result { + Ok(match key { + Value::Str(text) => text.clone(), + Value::Int(value) => value.to_string(), + Value::Float(value) => float_text(*value), + Value::Bool(true) => "true".to_owned(), + Value::Bool(false) => "false".to_owned(), + Value::None => "null".to_owned(), + key => return Err(Error::InvalidJsonKey(key.type_name())), + }) +} + +/// `py_encode_basestring_ascii`: escape `"`, `\`, control characters, and everything outside +/// printable ASCII as `\uXXXX`, with surrogate pairs above the BMP. +fn write_string(out: &mut String, text: &str) { + out.push('"'); + for ch in text.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + '\u{8}' => out.push_str("\\b"), + '\u{c}' => out.push_str("\\f"), + ' '..='~' => out.push(ch), + ch => { + let mut units = [0u16; 2]; + for unit in ch.encode_utf16(&mut units) { + let _ = write!(out, "\\u{unit:04x}"); + } + } + } + } + out.push('"'); +} diff --git a/litellm-rust/crates/python-compat/src/lib.rs b/litellm-rust/crates/python-compat/src/lib.rs new file mode 100644 index 00000000000..b23aff409f2 --- /dev/null +++ b/litellm-rust/crates/python-compat/src/lib.rs @@ -0,0 +1,39 @@ +//! Python data formats reproduced in Rust, for state that Python LiteLLM writes and reads. +//! +//! Each module mirrors one Python operation over plain data values, and its tests replay +//! fixtures generated by that operation in CPython (`scripts/generate_fixtures.py`): +//! +//! | Module | Python operation | +//! |---|---| +//! | [`literal`] | `ast.literal_eval(text)` | +//! | [`repr`] | `repr(value)` and `str(value)` | +//! | [`json`] | `json.dumps(value)`, and `json.loads(json.dumps(value))` as a JSON value | +//! | [`pickle`] | `pickle.loads(data)` and `pickle.dumps(value)` for plain data | +//! | [`truthy`] | `bool(value)` | +//! +//! [`Value`] is the closed data model these formats share. Live Python objects +//! (descriptors, `__bool__`, `__str__`, callbacks) are out of scope: those belong to the +//! PyO3 boundary in `litellm-python-bridge`, which runs the real protocol. +//! +//! Known limits, each pinned by a test: +//! - `set` iteration order follows Python's hash order, which this crate does not model +//! (string hashes are randomized per process). Sets keep their literal order. +//! - `str` values are Rust `String`s, so lone surrogates cannot be represented. +//! - [`pickle::loads`] decodes `tuple`, `set`, and `frozenset` as lists. + +mod error; +pub mod json; +pub mod literal; +pub mod pickle; +pub mod repr; +pub mod truthy; +mod value; + +pub use error::Error; +pub use num_bigint::BigInt; +pub use value::Value; + +/// Nesting limit for the decoders, which recurse. It keeps untrusted persisted data from +/// overflowing the Rust stack, and is deliberately stricter than CPython, whose parser takes +/// about 200 nested brackets and whose unpickler has no limit. +pub const MAX_DEPTH: usize = 128; diff --git a/litellm-rust/crates/python-compat/src/literal.rs b/litellm-rust/crates/python-compat/src/literal.rs new file mode 100644 index 00000000000..12e9036572f --- /dev/null +++ b/litellm-rust/crates/python-compat/src/literal.rs @@ -0,0 +1,745 @@ +//! `ast.literal_eval(text)`, as a single-pass recursive-descent parser. +//! +//! Tokens follow CPython's tokenizer (string prefixes, escapes, implicit concatenation, +//! numeric underscores and radixes, comments, line continuations). Expressions follow +//! `ast.literal_eval`'s evaluator: +//! +//! - one unary `+`/`-`, applied only to a numeric constant (`-(1)` is fine, `--1` is not) +//! - `a + b` / `a - b` only as a signed real plus or minus a complex constant, with 3.14's +//! mixed-mode rules (`1 - 0j` is `(1-0j)`) +//! - parentheses group without making a tuple; `set()` is the only call +//! - dict keys and set members are deduplicated with Python equality (`1 == 1.0 == True`) +//! +//! Not supported, each pinned by a fixture: `\N{NAME}` escapes, `...`, and escapes that +//! produce lone surrogates. + +use std::collections::{HashMap, hash_map::Entry}; + +use num_bigint::BigInt; +use num_traits::{FromPrimitive, ToPrimitive}; + +use crate::{Error, MAX_DEPTH, Value}; + +pub fn literal_eval(text: &str) -> Result { + let mut parser = Parser { + bytes: text.trim_start_matches([' ', '\t']).as_bytes(), + offset: text.len() - text.trim_start_matches([' ', '\t']).len(), + pos: 0, + depth: 0, + brackets: 0, + }; + parser.skip_leading_lines()?; + let value = parser.top_level()?.value; + parser.skip_trivia(true); + if parser.pos != parser.bytes.len() { + return Err(parser.error()); + } + Ok(value) +} + +/// How a parsed term may take part in `+`/`-`, per `ast.literal_eval`'s `_convert_num` +/// (`Constant`) and `_convert_signed_num` (`Signed`). `Other` is any other node. +#[derive(Clone, Copy, PartialEq)] +enum Kind { + Constant, + Signed, + Other, +} + +struct Term { + value: Value, + kind: Kind, +} + +impl Term { + fn other(value: Value) -> Self { + Self { + value, + kind: Kind::Other, + } + } + + fn is_number(&self) -> bool { + matches!( + self.value, + Value::Int(_) | Value::Float(_) | Value::Complex { .. } + ) + } +} + +struct Parser<'a> { + bytes: &'a [u8], + /// Bytes stripped before `bytes` starts, so errors report offsets into the input. + offset: usize, + pos: usize, + depth: usize, + /// Open brackets: newlines are insignificant only inside them. + brackets: usize, +} + +impl Parser<'_> { + fn error(&self) -> Error { + Error::InvalidLiteral(self.offset + self.pos) + } + + fn peek(&self) -> Option { + self.bytes.get(self.pos).copied() + } + + fn peek_at(&self, ahead: usize) -> Option { + self.bytes.get(self.pos + ahead).copied() + } + + fn expect(&mut self, byte: u8) -> Result<(), Error> { + if self.peek() != Some(byte) { + return Err(self.error()); + } + self.pos += 1; + Ok(()) + } + + fn enter(&mut self) -> Result<(), Error> { + self.depth += 1; + if self.depth > MAX_DEPTH { + return Err(Error::TooDeep); + } + Ok(()) + } + + /// Whitespace, comments, and backslash continuations; newlines too when `newlines`. + fn skip_trivia(&mut self, newlines: bool) { + while let Some(byte) = self.peek() { + match byte { + b' ' | b'\t' | b'\x0c' => self.pos += 1, + b'#' => { + while !matches!(self.peek(), None | Some(b'\n' | b'\r')) { + self.pos += 1; + } + } + // A continuation joins two lines; one that ends the input is an EOF error. + b'\\' if matches!(self.peek_at(1), Some(b'\n' | b'\r')) => { + let len = match (self.peek_at(1), self.peek_at(2)) { + (Some(b'\r'), Some(b'\n')) => 3, + _ => 2, + }; + if self.pos + len >= self.bytes.len() { + break; + } + self.pos += len; + } + b'\n' | b'\r' if newlines => self.pos += 1, + _ => break, + } + } + } + + /// Blank and comment-only lines may precede the expression, whose own line must not be + /// indented (CPython raises `IndentationError`). + fn skip_leading_lines(&mut self) -> Result<(), Error> { + loop { + let line_start = self.pos; + self.skip_trivia(false); + match self.peek() { + Some(b'\n' | b'\r') => self.pos += 1, + Some(_) if line_start > 0 && self.pos > line_start => { + self.pos = line_start; + return Err(self.error()); + } + _ => return Ok(()), + } + } + } + + fn at_logical_line_end(&self) -> bool { + matches!(self.peek(), None | Some(b'\n' | b'\r')) + } + + /// The `eval` input: an expression, or a tuple without parentheses. + fn top_level(&mut self) -> Result { + let first = self.expression()?; + self.skip_trivia(false); + if self.peek() != Some(b',') { + return Ok(first); + } + let mut values = vec![first.value]; + while self.peek() == Some(b',') { + self.pos += 1; + self.skip_trivia(false); + if self.at_logical_line_end() { + break; + } + values.push(self.expression()?.value); + self.skip_trivia(false); + } + Ok(Term::other(Value::Tuple(values))) + } + + /// A sum of unary terms, checked as `ast.literal_eval` checks `BinOp`. + fn expression(&mut self) -> Result { + let mut left = self.unary()?; + loop { + self.skip_trivia(self.brackets > 0); + let subtract = match self.peek() { + Some(b'+') => false, + Some(b'-') => true, + _ => return Ok(left), + }; + let at = self.pos; + self.pos += 1; + let right = self.unary()?; + left = complex_sum(left, subtract, right) + .ok_or(Error::InvalidLiteral(self.offset + at))?; + } + } + + fn unary(&mut self) -> Result { + self.skip_trivia(self.brackets > 0); + let negative = match self.peek() { + Some(b'+') => false, + Some(b'-') => true, + _ => return self.primary(), + }; + let at = self.pos; + self.pos += 1; + self.enter()?; + let operand = self.unary()?; + self.depth -= 1; + if operand.kind != Kind::Constant || !operand.is_number() { + return Err(Error::InvalidLiteral(self.offset + at)); + } + let value = if !negative { + operand.value + } else { + match operand.value { + Value::Int(value) => Value::Int(-value), + Value::Float(value) => Value::Float(-value), + Value::Complex { re, im } => Value::Complex { re: -re, im: -im }, + _ => unreachable!("checked numeric above"), + } + }; + Ok(Term { + value, + kind: Kind::Signed, + }) + } + + fn primary(&mut self) -> Result { + match self.peek() { + Some(b'(') => self.parenthesized(), + Some(b'[') => self.list(), + Some(b'{') => self.braced(), + Some(b'0'..=b'9') => self.number(), + Some(b'.') if matches!(self.peek_at(1), Some(b'0'..=b'9')) => self.number(), + Some(b'\'' | b'"') => self.strings(), + Some(byte) if byte.is_ascii_alphabetic() || byte == b'_' => { + if self.string_prefix_len().is_some() { + return self.strings(); + } + self.name() + } + _ => Err(self.error()), + } + } + + fn open(&mut self) -> Result<(), Error> { + self.enter()?; + self.brackets += 1; + self.pos += 1; + Ok(()) + } + + fn close(&mut self, byte: u8) -> Result<(), Error> { + self.skip_trivia(true); + self.expect(byte)?; + self.brackets -= 1; + self.depth -= 1; + Ok(()) + } + + /// Comma-separated expressions up to `close`, with an optional trailing comma. + fn elements(&mut self, close: u8) -> Result, Error> { + let mut values = Vec::new(); + loop { + self.skip_trivia(true); + if self.peek() == Some(close) { + return Ok(values); + } + values.push(self.expression()?.value); + self.skip_trivia(true); + if self.peek() != Some(b',') { + return Ok(values); + } + self.pos += 1; + } + } + + fn parenthesized(&mut self) -> Result { + self.open()?; + self.skip_trivia(true); + if self.peek() == Some(b')') { + self.close(b')')?; + return Ok(Term::other(Value::Tuple(Vec::new()))); + } + let first = self.expression()?; + self.skip_trivia(true); + if self.peek() != Some(b',') { + self.close(b')')?; + return Ok(first); + } + self.pos += 1; + let mut values = vec![first.value]; + values.extend(self.elements(b')')?); + self.close(b')')?; + Ok(Term::other(Value::Tuple(values))) + } + + fn list(&mut self) -> Result { + self.open()?; + let values = self.elements(b']')?; + self.close(b']')?; + Ok(Term::other(Value::List(values))) + } + + fn braced(&mut self) -> Result { + self.open()?; + self.skip_trivia(true); + if self.peek() == Some(b'}') { + self.close(b'}')?; + return Ok(Term::other(Value::Dict(Vec::new()))); + } + let first = self.expression()?.value; + self.skip_trivia(true); + if self.peek() != Some(b':') { + let mut members = UniqueValues::default(); + members.insert(first, None)?; + if self.peek() == Some(b',') { + self.pos += 1; + for member in self.elements(b'}')? { + members.insert(member, None)?; + } + } + self.close(b'}')?; + return Ok(Term::other(Value::Set(members.keys))); + } + let mut entries = UniqueValues::default(); + let mut key = first; + loop { + self.expect(b':')?; + let value = self.expression()?.value; + entries.insert(key, Some(value))?; + self.skip_trivia(true); + if self.peek() != Some(b',') { + break; + } + self.pos += 1; + self.skip_trivia(true); + if self.peek() == Some(b'}') { + break; + } + key = self.expression()?.value; + self.skip_trivia(true); + } + self.close(b'}')?; + Ok(Term::other(Value::Dict(entries.into_entries()))) + } + + fn identifier(&mut self) -> &[u8] { + let start = self.pos; + while matches!(self.peek(), Some(byte) if byte.is_ascii_alphanumeric() || byte == b'_') { + self.pos += 1; + } + &self.bytes[start..self.pos] + } + + fn name(&mut self) -> Result { + let at = self.pos; + let value = match self.identifier() { + b"True" => Value::Bool(true), + b"False" => Value::Bool(false), + b"None" => Value::None, + b"set" => { + self.skip_trivia(self.brackets > 0); + self.expect(b'(')?; + self.skip_trivia(true); + self.expect(b')')?; + return Ok(Term::other(Value::Set(Vec::new()))); + } + _ => return Err(Error::InvalidLiteral(self.offset + at)), + }; + Ok(Term { + value, + kind: Kind::Constant, + }) + } + + /// Digits with single underscores between them, as CPython's `digitpart`. + fn digits(&mut self, radix: u32, out: &mut String) -> Result<(), Error> { + let start = out.len(); + loop { + match self.peek() { + Some(byte) if (byte as char).is_digit(radix) => { + out.push(byte as char); + self.pos += 1; + } + Some(b'_') + if out.len() > start + && matches!(self.peek_at(1), Some(next) if (next as char).is_digit(radix)) => + { + self.pos += 1; + } + _ => break, + } + } + if out.len() == start { + return Err(self.error()); + } + Ok(()) + } + + fn number(&mut self) -> Result { + let start = self.pos; + let radix = match ( + self.peek(), + self.peek_at(1).map(|byte| byte.to_ascii_lowercase()), + ) { + (Some(b'0'), Some(b'x')) => Some(16), + (Some(b'0'), Some(b'o')) => Some(8), + (Some(b'0'), Some(b'b')) => Some(2), + _ => None, + }; + let mut text = String::new(); + let value = if let Some(radix) = radix { + self.pos += 2; + if self.peek() == Some(b'_') { + self.pos += 1; + } + self.digits(radix, &mut text)?; + Value::Int(BigInt::parse_bytes(text.as_bytes(), radix).ok_or(self.error())?) + } else { + let mut is_float = false; + if self.peek() != Some(b'.') { + self.digits(10, &mut text)?; + } + let integer_digits = text.clone(); + if self.peek() == Some(b'.') { + is_float = true; + self.pos += 1; + text.push('.'); + if matches!(self.peek(), Some(b'0'..=b'9')) { + self.digits(10, &mut text)?; + } + } + if matches!(self.peek(), Some(b'e' | b'E')) { + is_float = true; + self.pos += 1; + text.push('e'); + if let Some(sign @ (b'+' | b'-')) = self.peek() { + text.push(sign as char); + self.pos += 1; + } + self.digits(10, &mut text)?; + } + if matches!(self.peek(), Some(b'j' | b'J')) { + self.pos += 1; + let im = text.parse::().map_err(|_| self.error())?; + Value::Complex { re: 0.0, im } + } else if is_float { + Value::Float(text.parse::().map_err(|_| self.error())?) + } else { + if integer_digits.len() > 1 + && integer_digits.starts_with('0') + && integer_digits.bytes().any(|digit| digit != b'0') + { + return Err(Error::InvalidLiteral(self.offset + start)); + } + Value::Int(BigInt::parse_bytes(integer_digits.as_bytes(), 10).ok_or(self.error())?) + } + }; + if matches!(self.peek(), Some(byte) if byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'.') + { + return Err(self.error()); + } + Ok(Term { + value, + kind: Kind::Constant, + }) + } + + /// The length of a valid string prefix (`r`, `u`, `b`, `br`, `rb`, any case) directly + /// followed by a quote. + fn string_prefix_len(&self) -> Option { + let mut len = 0; + while matches!(self.peek_at(len), Some(byte) if byte.is_ascii_alphabetic()) && len < 3 { + len += 1; + } + if !matches!(self.peek_at(len), Some(b'\'' | b'"')) { + return None; + } + let prefix: Vec = self.bytes[self.pos..self.pos + len] + .iter() + .map(u8::to_ascii_lowercase) + .collect(); + matches!(prefix.as_slice(), b"" | b"r" | b"u" | b"b" | b"br" | b"rb").then_some(len) + } + + /// Adjacent string literals concatenate; `str` and `bytes` cannot mix. + fn strings(&mut self) -> Result { + let mut text: Option = None; + let mut bytes: Option> = None; + loop { + let at = self.pos; + let Some(prefix_len) = self.string_prefix_len() else { + break; + }; + let prefix = &self.bytes[self.pos..self.pos + prefix_len]; + let raw = prefix.iter().any(|byte| byte.eq_ignore_ascii_case(&b'r')); + let is_bytes = prefix.iter().any(|byte| byte.eq_ignore_ascii_case(&b'b')); + self.pos += prefix_len; + let mut out = Vec::new(); + self.string_body(raw, is_bytes, &mut out)?; + if is_bytes { + if text.is_some() { + return Err(Error::InvalidLiteral(self.offset + at)); + } + bytes.get_or_insert_with(Vec::new).extend(out); + } else { + if bytes.is_some() { + return Err(Error::InvalidLiteral(self.offset + at)); + } + let piece = + String::from_utf8(out).map_err(|_| Error::InvalidLiteral(self.offset + at))?; + text.get_or_insert_with(String::new).push_str(&piece); + } + self.skip_trivia(self.brackets > 0); + } + let value = match (text, bytes) { + (Some(text), None) => Value::Str(text), + (None, Some(bytes)) => Value::Bytes(bytes), + _ => return Err(self.error()), + }; + Ok(Term { + value, + kind: Kind::Constant, + }) + } + + /// One quoted body, decoded into UTF-8 (`str`) or raw bytes (`bytes`). + fn string_body(&mut self, raw: bool, is_bytes: bool, out: &mut Vec) -> Result<(), Error> { + let quote = self.bytes[self.pos]; + let triple = self.peek_at(1) == Some(quote) && self.peek_at(2) == Some(quote); + self.pos += if triple { 3 } else { 1 }; + loop { + let Some(byte) = self.peek() else { + return Err(self.error()); + }; + if byte == quote + && (!triple || (self.peek_at(1) == Some(quote) && self.peek_at(2) == Some(quote))) + { + self.pos += if triple { 3 } else { 1 }; + return Ok(()); + } + match byte { + b'\n' | b'\r' if !triple => return Err(self.error()), + b'\\' if raw => { + let Some(next) = self.peek_at(1) else { + return Err(self.error()); + }; + out.push(b'\\'); + self.pos += 1; + if next == b'\n' || next == b'\r' || next == quote || next == b'\\' { + out.push(next); + self.pos += 1; + } + } + b'\\' => { + self.pos += 1; + self.escape(is_bytes, out)?; + } + byte if is_bytes && !byte.is_ascii() => return Err(self.error()), + byte => { + out.push(byte); + self.pos += 1; + } + } + } + } + + fn escape(&mut self, is_bytes: bool, out: &mut Vec) -> Result<(), Error> { + let Some(byte) = self.peek() else { + return Err(self.error()); + }; + self.pos += 1; + let simple = match byte { + b'\n' => return Ok(()), + b'\r' => { + if self.peek() == Some(b'\n') { + self.pos += 1; + } + return Ok(()); + } + b'\\' | b'\'' | b'"' => byte, + b'a' => 0x07, + b'b' => 0x08, + b'f' => 0x0c, + b'n' => b'\n', + b'r' => b'\r', + b't' => b'\t', + b'v' => 0x0b, + b'0'..=b'7' => { + let mut code = u32::from(byte - b'0'); + for _ in 0..2 { + match self.peek() { + Some(digit @ b'0'..=b'7') => { + code = code * 8 + u32::from(digit - b'0'); + self.pos += 1; + } + _ => break, + } + } + // Bytes keep the low eight bits of `\400`-`\777`, as CPython does. + return self.push_code(if is_bytes { code & 0xff } else { code }, is_bytes, out); + } + b'x' => { + let code = self.hex(2)?; + return self.push_code(code, is_bytes, out); + } + b'u' if !is_bytes => { + let code = self.hex(4)?; + return self.push_code(code, is_bytes, out); + } + b'U' if !is_bytes => { + let code = self.hex(8)?; + return self.push_code(code, is_bytes, out); + } + b'N' if !is_bytes => return Err(self.error()), + _ => { + // Unknown escapes keep the backslash (a `SyntaxWarning` in CPython). + out.push(b'\\'); + self.pos -= 1; + return Ok(()); + } + }; + out.push(simple); + Ok(()) + } + + fn hex(&mut self, count: usize) -> Result { + let mut code = 0u32; + for _ in 0..count { + let digit = self + .peek() + .and_then(|byte| (byte as char).to_digit(16)) + .ok_or(self.error())?; + code = code * 16 + digit; + self.pos += 1; + } + Ok(code) + } + + fn push_code(&self, code: u32, is_bytes: bool, out: &mut Vec) -> Result<(), Error> { + if is_bytes { + out.push(u8::try_from(code).map_err(|_| self.error())?); + return Ok(()); + } + let ch = char::from_u32(code).ok_or(self.error())?; + let mut buffer = [0u8; 4]; + out.extend_from_slice(ch.encode_utf8(&mut buffer).as_bytes()); + Ok(()) + } +} + +/// `left + right` or `left - right` as `ast.literal_eval` permits: a signed real on the +/// left and an unsigned complex constant on the right, combined with CPython 3.14's +/// mixed-mode rules, which leave the imaginary part untouched by the real operand. +fn complex_sum(left: Term, subtract: bool, right: Term) -> Option { + if left.kind == Kind::Other || right.kind != Kind::Constant { + return None; + } + let real = match &left.value { + Value::Int(value) => value.to_f64().filter(|value| value.is_finite())?, + Value::Float(value) => *value, + _ => return None, + }; + let Value::Complex { re, im } = right.value else { + return None; + }; + let value = if subtract { + Value::Complex { + re: real - re, + im: -im, + } + } else { + Value::Complex { re: real + re, im } + }; + Some(Term::other(value)) +} + +/// Python equality for hashable literal values: numbers compare by value across `bool`, +/// `int`, `float`, and `complex`, so `1`, `1.0`, `True`, and `(1+0j)` are one key. +#[derive(Hash, PartialEq, Eq)] +enum KeyId { + None, + Int(BigInt), + Float(u64), + Complex(u64, u64), + Str(String), + Bytes(Vec), + Tuple(Vec), +} + +fn float_key(value: f64) -> KeyId { + if value.fract() == 0.0 + && let Some(integer) = BigInt::from_f64(value) + { + return KeyId::Int(integer); + } + KeyId::Float(value.to_bits()) +} + +fn key_id(value: &Value) -> Result { + Ok(match value { + Value::None => KeyId::None, + Value::Bool(value) => KeyId::Int(BigInt::from(u8::from(*value))), + Value::Int(value) => KeyId::Int(value.clone()), + Value::Float(value) => float_key(*value), + Value::Complex { re, im } if *im == 0.0 => float_key(*re), + Value::Complex { re, im } => KeyId::Complex((re + 0.0).to_bits(), (im + 0.0).to_bits()), + Value::Str(text) => KeyId::Str(text.clone()), + Value::Bytes(bytes) => KeyId::Bytes(bytes.clone()), + Value::Tuple(values) => KeyId::Tuple(values.iter().map(key_id).collect::>()?), + value @ (Value::List(_) | Value::Dict(_) | Value::Set(_)) => { + return Err(Error::Unhashable(value.type_name())); + } + }) +} + +/// Dict entries or set members in first-seen order: a repeated key keeps its first +/// position and, for dicts, takes the latest value. +#[derive(Default)] +struct UniqueValues { + keys: Vec, + values: Vec, + index: HashMap, +} + +impl UniqueValues { + fn insert(&mut self, key: Value, value: Option) -> Result<(), Error> { + match self.index.entry(key_id(&key)?) { + Entry::Occupied(slot) => { + if let Some(value) = value { + self.values[*slot.get()] = value; + } + } + Entry::Vacant(slot) => { + slot.insert(self.keys.len()); + self.keys.push(key); + self.values.extend(value); + } + } + Ok(()) + } + + fn into_entries(self) -> Vec<(Value, Value)> { + self.keys.into_iter().zip(self.values).collect() + } +} diff --git a/litellm-rust/crates/python-compat/src/pickle.rs b/litellm-rust/crates/python-compat/src/pickle.rs new file mode 100644 index 00000000000..bff7442dc6b --- /dev/null +++ b/litellm-rust/crates/python-compat/src/pickle.rs @@ -0,0 +1,187 @@ +//! `pickle.loads` and `pickle.dumps` for plain data, as diskcache stores LiteLLM values. +//! +//! Both directions go through `serde-pickle`'s serde interface rather than +//! `serde_pickle::Value`, because that value type keeps dicts in a `BTreeMap` and would +//! reorder keys. The serde interface keeps insertion order, at the cost of reporting +//! `tuple`, `set`, and `frozenset` as sequences: [`loads`] decodes all three as lists. +//! Python objects that need a class (`GLOBAL`/`REDUCE`) and recursive structures fail. + +use std::fmt; + +use serde::{ + Deserializer, Serialize, Serializer, + de::{self, DeserializeSeed, MapAccess, SeqAccess, Visitor}, + ser::{SerializeMap, SerializeSeq, SerializeTuple}, +}; +use serde_pickle::{DeOptions, SerOptions}; + +use crate::{Error, MAX_DEPTH, Value}; + +/// `pickle.loads(data)` for any protocol from 0 to 5. +pub fn loads(data: &[u8]) -> Result { + let mut deserializer = serde_pickle::Deserializer::new(data, DeOptions::new()); + let value = Seed { depth: 0 } + .deserialize(&mut deserializer) + .map_err(|error| Error::InvalidPickle(error.to_string()))?; + deserializer + .end() + .map_err(|error| Error::InvalidPickle(error.to_string()))?; + Ok(value) +} + +/// `pickle.dumps(value, protocol=3)`. Every Python 3 reads protocol 3, whatever its own +/// default. Sets and complex numbers are rejected rather than silently changing type. +pub fn dumps(value: &Value) -> Result, Error> { + check_picklable(value, 0)?; + serde_pickle::to_vec(&Pickled(value), SerOptions::new()) + .map_err(|error| Error::InvalidPickle(error.to_string())) +} + +fn check_picklable(value: &Value, depth: usize) -> Result<(), Error> { + if depth > MAX_DEPTH { + return Err(Error::TooDeep); + } + match value { + Value::Int(value) if i64::try_from(value).is_err() => Err(Error::IntegerOutOfRange), + value @ (Value::Set(_) | Value::Complex { .. }) => { + Err(Error::NotPicklable(value.type_name())) + } + Value::List(values) | Value::Tuple(values) => values + .iter() + .try_for_each(|value| check_picklable(value, depth + 1)), + Value::Dict(entries) => entries.iter().try_for_each(|(key, value)| { + check_picklable(key, depth + 1)?; + check_picklable(value, depth + 1) + }), + _ => Ok(()), + } +} + +struct Pickled<'a>(&'a Value); + +impl Serialize for Pickled<'_> { + fn serialize(&self, serializer: S) -> Result { + match self.0 { + Value::None => serializer.serialize_unit(), + Value::Bool(value) => serializer.serialize_bool(*value), + Value::Int(value) => { + let value = i64::try_from(value) + .map_err(|_| serde::ser::Error::custom("integer out of i64 range"))?; + serializer.serialize_i64(value) + } + Value::Float(value) => serializer.serialize_f64(*value), + Value::Str(text) => serializer.serialize_str(text), + Value::Bytes(bytes) => serializer.serialize_bytes(bytes), + Value::List(values) => { + let mut seq = serializer.serialize_seq(Some(values.len()))?; + for value in values { + seq.serialize_element(&Pickled(value))?; + } + seq.end() + } + Value::Tuple(values) => { + let mut tuple = serializer.serialize_tuple(values.len())?; + for value in values { + tuple.serialize_element(&Pickled(value))?; + } + tuple.end() + } + Value::Dict(entries) => { + let mut map = serializer.serialize_map(Some(entries.len()))?; + for (key, value) in entries { + map.serialize_entry(&Pickled(key), &Pickled(value))?; + } + map.end() + } + value @ (Value::Set(_) | Value::Complex { .. }) => Err(serde::ser::Error::custom( + format!("{} cannot be pickled as plain data", value.type_name()), + )), + } + } +} + +#[derive(Clone, Copy)] +struct Seed { + depth: usize, +} + +impl Seed { + fn child(self) -> Result { + if self.depth >= MAX_DEPTH { + return Err(E::custom(Error::TooDeep)); + } + Ok(Self { + depth: self.depth + 1, + }) + } +} + +impl<'de> DeserializeSeed<'de> for Seed { + type Value = Value; + + fn deserialize>(self, deserializer: D) -> Result { + deserializer.deserialize_any(self) + } +} + +impl<'de> Visitor<'de> for Seed { + type Value = Value; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a plain Python data value") + } + + fn visit_unit(self) -> Result { + Ok(Value::None) + } + + fn visit_bool(self, value: bool) -> Result { + Ok(Value::Bool(value)) + } + + fn visit_i64(self, value: i64) -> Result { + Ok(Value::Int(value.into())) + } + + fn visit_u64(self, value: u64) -> Result { + Ok(Value::Int(value.into())) + } + + fn visit_f64(self, value: f64) -> Result { + Ok(Value::Float(value)) + } + + fn visit_str(self, value: &str) -> Result { + Ok(Value::Str(value.to_owned())) + } + + fn visit_string(self, value: String) -> Result { + Ok(Value::Str(value)) + } + + fn visit_bytes(self, value: &[u8]) -> Result { + Ok(Value::Bytes(value.to_vec())) + } + + fn visit_byte_buf(self, value: Vec) -> Result { + Ok(Value::Bytes(value)) + } + + fn visit_seq>(self, mut seq: A) -> Result { + let child = self.child()?; + let mut values = Vec::with_capacity(seq.size_hint().unwrap_or(0).min(4096)); + while let Some(value) = seq.next_element_seed(child)? { + values.push(value); + } + Ok(Value::List(values)) + } + + fn visit_map>(self, mut map: A) -> Result { + let child = self.child()?; + let mut entries = Vec::with_capacity(map.size_hint().unwrap_or(0).min(4096)); + while let Some(key) = map.next_key_seed(child)? { + entries.push((key, map.next_value_seed(child)?)); + } + Ok(Value::Dict(entries)) + } +} diff --git a/litellm-rust/crates/python-compat/src/repr.rs b/litellm-rust/crates/python-compat/src/repr.rs new file mode 100644 index 00000000000..1cf20d87716 --- /dev/null +++ b/litellm-rust/crates/python-compat/src/repr.rs @@ -0,0 +1,237 @@ +//! `repr(value)` and `str(value)` byte for byte. +//! +//! LiteLLM hashes `str(value)` into cache keys and writes `str(dict)` into Redis, so these +//! strings are persisted identifiers rather than display text: every quote choice, escape, +//! and float digit must match CPython. + +use std::fmt::Write; + +use crate::Value; + +/// Generated by `scripts/generate_nonprintable.py`; see `generated/nonprintable.rs`. +mod nonprintable { + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/generated/nonprintable.rs" + )); +} + +/// Unicode version of the printable-character table, from the Python that generated it. +pub const UNICODE_VERSION: &str = nonprintable::UNICODE_VERSION; + +/// `repr(value)`. +pub fn repr(value: &Value) -> String { + let mut out = String::new(); + write_repr(&mut out, value); + out +} + +/// `str(value)`: a string's own contents, and `repr` for every other value. +pub fn to_str(value: &Value) -> String { + match value { + Value::Str(text) => text.clone(), + value => repr(value), + } +} + +/// `repr(float)`, the shortest round-trip form with CPython's exponent thresholds. +pub fn float_repr(value: f64) -> String { + format_float(value, true) +} + +fn write_repr(out: &mut String, value: &Value) { + match value { + Value::None => out.push_str("None"), + Value::Bool(true) => out.push_str("True"), + Value::Bool(false) => out.push_str("False"), + Value::Int(value) => { + let _ = write!(out, "{value}"); + } + Value::Float(value) => out.push_str(&float_repr(*value)), + Value::Complex { re, im } => write_complex(out, *re, *im), + Value::Str(text) => write_str(out, text), + Value::Bytes(bytes) => write_bytes(out, bytes), + Value::List(values) => write_sequence(out, '[', values, ']'), + Value::Tuple(values) if values.len() == 1 => { + out.push('('); + write_repr(out, &values[0]); + out.push_str(",)"); + } + Value::Tuple(values) => write_sequence(out, '(', values, ')'), + Value::Set(values) if values.is_empty() => out.push_str("set()"), + Value::Set(values) => write_sequence(out, '{', values, '}'), + Value::Dict(entries) => { + out.push('{'); + for (index, (key, value)) in entries.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + write_repr(out, key); + out.push_str(": "); + write_repr(out, value); + } + out.push('}'); + } + } +} + +fn write_sequence(out: &mut String, open: char, values: &[Value], close: char) { + out.push(open); + for (index, value) in values.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + write_repr(out, value); + } + out.push(close); +} + +/// `complex.__repr__`: a `+0.0` real part prints only the imaginary part, without parens. +fn write_complex(out: &mut String, re: f64, im: f64) { + if re == 0.0 && re.is_sign_positive() { + out.push_str(&format_float(im, false)); + out.push('j'); + return; + } + out.push('('); + out.push_str(&format_float(re, false)); + let im_text = format_float(im, false); + if !im_text.starts_with('-') { + out.push('+'); + } + out.push_str(&im_text); + out.push_str("j)"); +} + +/// `PyOS_double_to_string(value, 'r', 0, flags)`: scientific notation below 1e-4 and from +/// 1e16 up, with a signed exponent of at least two digits. `add_dot_0` is +/// `Py_DTSF_ADD_DOT_0`, which `float` sets and `complex` does not. +fn format_float(value: f64, add_dot_0: bool) -> String { + if value.is_nan() { + return "nan".to_owned(); + } + if value.is_infinite() { + return if value > 0.0 { "inf" } else { "-inf" }.to_owned(); + } + // Rust's `{:e}` prints the shortest round-trip digits, like CPython's 'r' mode. + let scientific = format!("{value:e}"); + let (mantissa, exponent) = scientific + .split_once('e') + .expect("`{:e}` always prints an exponent"); + let exponent: i32 = exponent.parse().expect("`{:e}` exponent is an integer"); + let (sign, mantissa) = match mantissa.strip_prefix('-') { + Some(mantissa) => ("-", mantissa), + None => ("", mantissa), + }; + let digits: String = mantissa.chars().filter(|ch| *ch != '.').collect(); + + let mut out = String::from(sign); + if !(-4..16).contains(&exponent) { + out.push_str(&digits[..1]); + if digits.len() > 1 { + out.push('.'); + out.push_str(&digits[1..]); + } + let _ = write!( + out, + "e{}{:02}", + if exponent < 0 { '-' } else { '+' }, + exponent.unsigned_abs() + ); + } else if exponent < 0 { + out.push_str("0."); + out.extend(std::iter::repeat_n('0', (-exponent - 1) as usize)); + out.push_str(&digits); + } else { + let integer_digits = exponent as usize + 1; + if digits.len() > integer_digits { + out.push_str(&digits[..integer_digits]); + out.push('.'); + out.push_str(&digits[integer_digits..]); + } else { + out.push_str(&digits); + out.extend(std::iter::repeat_n('0', integer_digits - digits.len())); + if add_dot_0 { + out.push_str(".0"); + } + } + } + out +} + +/// `unicode_repr`: single quotes unless the text has a `'` and no `"`. Printable non-ASCII +/// stays literal; everything `str.isprintable()` rejects is escaped. +fn write_str(out: &mut String, text: &str) { + let quote = if text.contains('\'') && !text.contains('"') { + '"' + } else { + '\'' + }; + out.push(quote); + for ch in text.chars() { + match ch { + '\\' => out.push_str("\\\\"), + '\t' => out.push_str("\\t"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + ch if ch == quote => { + out.push('\\'); + out.push(ch); + } + ch if is_printable(ch) => out.push(ch), + ch => { + let code = ch as u32; + let _ = match code { + 0..=0xff => write!(out, "\\x{code:02x}"), + 0x100..=0xffff => write!(out, "\\u{code:04x}"), + _ => write!(out, "\\U{code:08x}"), + }; + } + } + } + out.push(quote); +} + +/// `bytes.__repr__`: the same quote rule as `str`, with every byte outside printable ASCII +/// escaped as `\xhh`. +fn write_bytes(out: &mut String, bytes: &[u8]) { + let quote = if bytes.contains(&b'\'') && !bytes.contains(&b'"') { + b'"' + } else { + b'\'' + }; + out.push('b'); + out.push(quote as char); + for &byte in bytes { + match byte { + b'\\' => out.push_str("\\\\"), + b'\t' => out.push_str("\\t"), + b'\n' => out.push_str("\\n"), + b'\r' => out.push_str("\\r"), + byte if byte == quote => { + out.push('\\'); + out.push(byte as char); + } + 0x20..=0x7e => out.push(byte as char), + byte => { + let _ = write!(out, "\\x{byte:02x}"); + } + } + } + out.push(quote as char); +} + +fn is_printable(ch: char) -> bool { + let code = ch as u32; + nonprintable::NONPRINTABLE + .binary_search_by(|&(low, high)| { + if high < code { + std::cmp::Ordering::Less + } else if low > code { + std::cmp::Ordering::Greater + } else { + std::cmp::Ordering::Equal + } + }) + .is_err() +} diff --git a/litellm-rust/crates/python-compat/src/truthy.rs b/litellm-rust/crates/python-compat/src/truthy.rs new file mode 100644 index 00000000000..af747202572 --- /dev/null +++ b/litellm-rust/crates/python-compat/src/truthy.rs @@ -0,0 +1,18 @@ +use num_bigint::Sign; + +use crate::Value; + +/// `bool(value)` for plain data: `None`, `False`, zero, and empty containers are false. +pub fn truthy(value: &Value) -> bool { + match value { + Value::None => false, + Value::Bool(value) => *value, + Value::Int(value) => value.sign() != Sign::NoSign, + Value::Float(value) => *value != 0.0, + Value::Complex { re, im } => *re != 0.0 || *im != 0.0, + Value::Str(value) => !value.is_empty(), + Value::Bytes(value) => !value.is_empty(), + Value::Tuple(values) | Value::List(values) | Value::Set(values) => !values.is_empty(), + Value::Dict(entries) => !entries.is_empty(), + } +} diff --git a/litellm-rust/crates/python-compat/src/value.rs b/litellm-rust/crates/python-compat/src/value.rs new file mode 100644 index 00000000000..b1397190638 --- /dev/null +++ b/litellm-rust/crates/python-compat/src/value.rs @@ -0,0 +1,53 @@ +use num_bigint::BigInt; + +/// A Python value built only from literals: what `ast.literal_eval` can return. +#[derive(Clone, Debug, PartialEq)] +pub enum Value { + None, + Bool(bool), + Int(BigInt), + Float(f64), + Complex { + re: f64, + im: f64, + }, + Str(String), + Bytes(Vec), + Tuple(Vec), + List(Vec), + /// Insertion-ordered, with Python's key equality already applied. + Dict(Vec<(Value, Value)>), + /// Literal order, with Python's member equality already applied. + Set(Vec), +} + +impl Value { + /// Python's type name, as it appears in `TypeError` messages. + pub fn type_name(&self) -> &'static str { + match self { + Value::None => "NoneType", + Value::Bool(_) => "bool", + Value::Int(_) => "int", + Value::Float(_) => "float", + Value::Complex { .. } => "complex", + Value::Str(_) => "str", + Value::Bytes(_) => "bytes", + Value::Tuple(_) => "tuple", + Value::List(_) => "list", + Value::Dict(_) => "dict", + Value::Set(_) => "set", + } + } +} + +impl From for Value { + fn from(value: i64) -> Self { + Value::Int(value.into()) + } +} + +impl From<&str> for Value { + fn from(value: &str) -> Self { + Value::Str(value.to_owned()) + } +} diff --git a/litellm-rust/crates/python-compat/tests/fixtures.rs b/litellm-rust/crates/python-compat/tests/fixtures.rs new file mode 100644 index 00000000000..9c202fdc5f1 --- /dev/null +++ b/litellm-rust/crates/python-compat/tests/fixtures.rs @@ -0,0 +1,343 @@ +//! Replays `generated/values.json`, which CPython wrote with `scripts/generate_fixtures.py`. + +use std::{collections::BTreeMap, fs::File, io::Write}; + +use litellm_python_compat::{ + Value, json, + literal::literal_eval, + pickle, + repr::{repr, to_str}, + truthy::truthy, +}; +use rstest::{fixture, rstest}; +use serde::Deserialize; + +#[derive(Deserialize)] +struct Fixtures { + rows: Vec, + sources: Vec, +} + +/// `ast.literal_eval(source)`: the `repr` of its result, or the exception class it raised. +#[derive(Deserialize)] +struct Source { + name: String, + source: String, + repr: Option, + error: Option, +} + +#[derive(Deserialize)] +struct Row { + name: String, + source: String, + literal: bool, + plain: bool, + repr: String, + str: String, + truthy: bool, + json: Option, + json_error: Option, + pickle: Option>, + view: Option, +} + +/// Parsed once for the whole test binary. +#[fixture] +#[once] +fn fixtures() -> Fixtures { + serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/generated/values.json" + ))) + .expect("values.json matches the fixture schema") +} + +/// Accepted differences from CPython: `(fixture source, check prefix, reason)`. Each entry +/// must still differ, so a dependency fix that removes one fails the test until it is deleted. +const KNOWN: &[(&str, &str, &str)] = &[ + ( + "...", + "literal_eval source", + "`Ellipsis` is not part of the data model", + ), + ( + r"'\ud800'", + "literal_eval source", + "a Rust `String` cannot hold a lone surrogate", + ), + ( + r"'\N{BULLET}'", + "literal_eval source", + "`\\N{NAME}` needs the Unicode name table", + ), + ( + "nested_150", + "literal_eval", + "deeper than MAX_DEPTH: rejected for stack safety, where CPython still parses it", + ), + ( + "nested_150", + "pickle.loads", + "deeper than MAX_DEPTH: rejected for stack safety, where CPython has no limit", + ), + ( + "2**64", + "pickle.loads", + "serde-pickle's serde interface stops at i64", + ), + ( + "-(2**70)", + "pickle.loads", + "serde-pickle's serde interface stops at i64", + ), + ( + "2**64", + "pickle.dumps", + "serde-pickle's serde interface stops at i64", + ), + ( + "-(2**70)", + "pickle.dumps", + "serde-pickle's serde interface stops at i64", + ), + ( + "2**64", + "to_json", + "serde_json has no exact form for integers beyond u64", + ), + ( + "-(2**70)", + "to_json", + "serde_json has no exact form for integers beyond i64", + ), + ( + "b''", + "pickle.loads protocol 0", + "protocols 0-2 pickle `b''` as a `bytes()` call", + ), + ( + "b''", + "pickle.loads protocol 1", + "protocols 0-2 pickle `b''` as a `bytes()` call", + ), + ( + "b''", + "pickle.loads protocol 2", + "protocols 0-2 pickle `b''` as a `bytes()` call", + ), +]; + +/// Collects every mismatch so one run reports the whole divergence set. +#[derive(Default)] +struct Mismatches { + unexpected: Vec, + known_seen: Vec, + known_scope: Vec, +} + +impl Mismatches { + fn known(source: &str, what: &str) -> Option { + KNOWN + .iter() + .position(|(known, prefix, _)| *known == source && what.starts_with(prefix)) + } + + fn check(&mut self, row: &Row, what: &str, expected: &str, actual: &str) { + self.check_source(&row.name, what, expected, actual); + } + + /// `name` identifies the fixture row in reports and in [`KNOWN`]. + fn check_source(&mut self, name: &str, what: &str, expected: &str, actual: &str) { + let known = Self::known(name, what); + if let Some(index) = known { + self.known_scope.push(index); + } + if expected == actual { + return; + } + match known { + Some(index) => self.known_seen.push(index), + None => self.unexpected.push(format!( + "{name:?} [{what}]\n python: {expected}\n rust: {actual}" + )), + } + } + + fn finish(self) { + let resolved: Vec<_> = self + .known_scope + .iter() + .filter(|index| !self.known_seen.contains(index)) + .map(|&index| format!("{} [{}]", KNOWN[index].0, KNOWN[index].1)) + .collect(); + assert!( + self.unexpected.is_empty() && resolved.is_empty(), + "{} mismatches with CPython:\n{}\nknown divergences that now match (delete them \ + from KNOWN): {resolved:?}", + self.unexpected.len(), + self.unexpected.join("\n"), + ); + } +} + +fn check_value(mismatches: &mut Mismatches, row: &Row, value: &Value) { + mismatches.check(row, "repr", &row.repr, &repr(value)); + mismatches.check(row, "str", &row.str, &to_str(value)); + mismatches.check( + row, + "bool", + &row.truthy.to_string(), + &truthy(value).to_string(), + ); + let expected = row.json.clone().or_else(|| { + row.json_error + .clone() + .map(|error| format!("error: {error}")) + }); + let actual = match json::dumps(value) { + Ok(text) => text, + Err(error) => format!("error: {error}"), + }; + mismatches.check( + row, + "json.dumps", + expected.as_deref().unwrap_or(""), + &actual, + ); +} + +#[rstest] +fn literal_rows_match_python_repr_str_bool_and_json(fixtures: &Fixtures) { + let mut mismatches = Mismatches::default(); + for row in fixtures.rows.iter().filter(|row| row.literal) { + match literal_eval(&row.repr) { + Ok(value) => check_value(&mut mismatches, row, &value), + Err(error) => mismatches.check(row, "literal_eval", &row.repr, &error.to_string()), + } + } + mismatches.finish(); +} + +/// Errors compare by outcome only: CPython's exception class is not part of the contract. +#[rstest] +fn literal_eval_matches_python_on_source_texts(fixtures: &Fixtures) { + let mut mismatches = Mismatches::default(); + for case in &fixtures.sources { + let expected = match (&case.repr, &case.error) { + (Some(repr), None) => repr.clone(), + (None, Some(_)) => "an error".to_owned(), + _ => panic!("{:?}: a source records a repr or an error", case.source), + }; + let actual = match literal_eval(&case.source) { + Ok(value) => repr(&value), + Err(_) => "an error".to_owned(), + }; + mismatches.check_source(&case.name, "literal_eval source", &expected, &actual); + } + mismatches.finish(); +} + +#[rstest] +fn pickle_loads_matches_python_at_every_protocol(fixtures: &Fixtures) { + let mut mismatches = Mismatches::default(); + for row in &fixtures.rows { + let Some(pickles) = &row.pickle else { continue }; + for (protocol, data) in pickles { + let data = hex::decode(data).expect("fixture pickle is hex"); + let what = format!("pickle.loads protocol {protocol}"); + match (pickle::loads(&data), row.plain) { + (Ok(value), true) => { + let view = row.view.as_deref().expect("picklable rows have a view"); + mismatches.check(row, &what, view, &repr(&value)); + } + (Err(error), true) => { + mismatches.check(row, &what, "a value", &format!("error: {error}")) + } + (Ok(value), false) => { + mismatches.check(row, &what, "a class-reference error", &repr(&value)) + } + (Err(pickle_error), false) => assert!( + matches!(pickle_error, litellm_python_compat::Error::InvalidPickle(_)), + "{}: {pickle_error}", + row.source + ), + } + } + } + mismatches.finish(); +} + +/// Non-finite floats have no literal form; pickle is how Rust receives them. +#[rstest] +fn values_reached_only_through_pickle_match_python(fixtures: &Fixtures) { + let mut mismatches = Mismatches::default(); + for row in fixtures + .rows + .iter() + .filter(|row| !row.literal && row.plain && row.view.as_deref() == Some(&row.repr)) + { + let data = hex::decode(&row.pickle.as_ref().expect("plain rows pickle")["5"]) + .expect("fixture pickle is hex"); + let value = pickle::loads(&data).expect("plain pickle decodes"); + check_value(&mut mismatches, row, &value); + } + mismatches.finish(); +} + +/// Byte equality with CPython is not the contract: CPython adds memo opcodes and picks the +/// smallest integer opcode. `scripts/verify_rust_pickles.py` checks that CPython reads these +/// back; set `PYTHON_COMPAT_RUST_PICKLES` to a file path to export them. +#[rstest] +fn pickle_dumps_round_trips_every_plain_literal(fixtures: &Fixtures) { + // Truncate up front: the verifier must read this run's rows and nothing else. + let mut export = std::env::var_os("PYTHON_COMPAT_RUST_PICKLES") + .map(|path| File::create(path).expect("the export path is writable")); + let mut mismatches = Mismatches::default(); + for row in fixtures + .rows + .iter() + .filter(|row| row.literal && row.plain && row.view.as_deref() == Some(&row.repr)) + { + // Rows past `MAX_DEPTH` are covered by the literal test's own KNOWN entry. + let Ok(value) = literal_eval(&row.repr) else { + continue; + }; + let data = match pickle::dumps(&value) { + Ok(data) => data, + Err(error) => { + mismatches.check(row, "pickle.dumps", "a pickle", &format!("error: {error}")); + continue; + } + }; + let decoded = pickle::loads(&data).expect("rust pickle decodes"); + mismatches.check(row, "pickle round trip", &row.repr, &repr(&decoded)); + if let Some(file) = &mut export { + writeln!(file, "{}\t{}", hex::encode(&data), repr(&value)) + .expect("the export file is writable"); + } + } + mismatches.finish(); +} + +#[rstest] +fn to_json_matches_python_json_round_trip(fixtures: &Fixtures) { + let mut mismatches = Mismatches::default(); + for row in fixtures.rows.iter().filter(|row| row.literal) { + let Some(expected) = &row.json else { continue }; + let Ok(value) = literal_eval(&row.repr) else { + continue; + }; + let expected: serde_json::Value = + serde_json::from_str(expected).expect("python json.dumps output parses"); + match json::to_json(&value) { + Ok(actual) => { + mismatches.check(row, "to_json", &expected.to_string(), &actual.to_string()) + } + Err(error) => { + mismatches.check(row, "to_json", &expected.to_string(), &error.to_string()) + } + } + } + mismatches.finish(); +} diff --git a/litellm-rust/crates/python-compat/tests/limits.rs b/litellm-rust/crates/python-compat/tests/limits.rs new file mode 100644 index 00000000000..c4d931034c4 --- /dev/null +++ b/litellm-rust/crates/python-compat/tests/limits.rs @@ -0,0 +1,122 @@ +use std::time::{Duration, Instant}; + +use litellm_python_compat::{Error, MAX_DEPTH, Value, json, literal::literal_eval, pickle}; +use rstest::{fixture, rstest}; + +/// The bracket pair of one container shape, as `(open, close)`. +#[fixture] +fn shapes() -> [(&'static str, &'static str); 3] { + [("[", "]"), ("{'a': ", "}"), ("(", ",)")] +} + +fn nested_text(open: &str, close: &str, depth: usize) -> String { + format!("{}1{}", open.repeat(depth), close.repeat(depth)) +} + +fn nested_list(depth: usize) -> Value { + (0..depth).fold(Value::from(1), |value, _| Value::List(vec![value])) +} + +/// A protocol 3 pickle of `depth` nested lists around `1`: `EMPTY_LIST` per level, then +/// `BININT1 1`, then `APPEND` per level. Written by hand because `dumps` refuses the depth. +fn nested_list_pickle(depth: usize) -> Vec { + let mut data = vec![0x80, 3]; + data.extend(std::iter::repeat_n(b']', depth)); + data.extend([b'K', 1]); + data.extend(std::iter::repeat_n(b'a', depth)); + data.push(b'.'); + data +} + +#[rstest] +fn literal_eval_accepts_the_limit_and_rejects_past_it(shapes: [(&'static str, &'static str); 3]) { + for (open, close) in shapes { + assert!(literal_eval(&nested_text(open, close, MAX_DEPTH)).is_ok()); + assert!(matches!( + literal_eval(&nested_text(open, close, MAX_DEPTH + 1)), + Err(Error::TooDeep) + )); + } +} + +/// A backtracking parser (the `py_literal` grammar this replaced) doubles per nested level +/// and takes minutes here; the bound is loose enough to survive a slow debug build. +#[rstest] +fn literal_eval_stays_linear_in_depth(shapes: [(&'static str, &'static str); 3]) { + for (open, close) in shapes { + let text = nested_text(open, close, MAX_DEPTH); + let start = Instant::now(); + assert!(literal_eval(&text).is_ok()); + let elapsed = start.elapsed(); + assert!( + elapsed < Duration::from_millis(50), + "{open} nested {MAX_DEPTH} deep took {elapsed:?}" + ); + } +} + +#[rstest] +fn literal_eval_ignores_brackets_inside_strings() { + let text = format!("'{}'", "[".repeat(MAX_DEPTH + 1)); + assert!(matches!(literal_eval(&text), Ok(Value::Str(_)))); +} + +#[rstest] +fn pickle_nesting_is_bounded_in_both_directions() { + assert_eq!( + pickle::loads(&nested_list_pickle(MAX_DEPTH)).unwrap(), + nested_list(MAX_DEPTH) + ); + assert!(matches!( + pickle::loads(&nested_list_pickle(MAX_DEPTH + 1)), + Err(Error::InvalidPickle(_)) + )); + assert!(pickle::dumps(&nested_list(MAX_DEPTH)).is_ok()); + assert!(matches!( + pickle::dumps(&nested_list(MAX_DEPTH + 2)), + Err(Error::TooDeep) + )); +} + +#[rstest] +#[case("{1, 2}", "set")] +#[case("1+2j", "complex")] +fn pickle_dumps_refuses_types_it_would_change(#[case] source: &str, #[case] type_name: &str) { + let value = literal_eval(source).expect("source is a literal"); + assert!(matches!(pickle::dumps(&value), Err(Error::NotPicklable(name)) if name == type_name)); +} + +#[rstest] +#[case(b"\x80\x02c__builtin__\ncomplex\nq\x00.".to_vec(), "a class reference")] +#[case({ let mut data = pickle::dumps(&Value::from(1)).unwrap(); data.push(b'.'); data }, "trailing data")] +fn pickle_loads_rejects(#[case] data: Vec, #[case] what: &str) { + assert!( + matches!(pickle::loads(&data), Err(Error::InvalidPickle(_))), + "{what} must not decode" + ); +} + +#[rstest] +#[case(Value::Bytes(b"x".to_vec()), "Object of type bytes is not JSON serializable")] +#[case(Value::Set(vec![Value::from(1)]), "Object of type set is not JSON serializable")] +#[case(Value::Complex { re: 1.0, im: 2.0 }, "Object of type complex is not JSON serializable")] +#[case( + Value::Float(f64::NAN), + "Out of range float values are not JSON compliant" +)] +fn to_json_reports_what_python_json_dumps_would_reject( + #[case] value: Value, + #[case] message: &str, +) { + let error = json::to_json(&value).expect_err("value has no serde_json form"); + assert_eq!(error.to_string(), message); +} + +/// `json.dumps` writes the non-finite floats that `to_json` cannot represent. +#[rstest] +#[case(f64::NAN, "NaN")] +#[case(f64::INFINITY, "Infinity")] +#[case(f64::NEG_INFINITY, "-Infinity")] +fn json_dumps_writes_non_finite_floats(#[case] value: f64, #[case] text: &str) { + assert_eq!(json::dumps(&Value::Float(value)).unwrap(), text); +} From 2ef710e3d5a29351caa20a8d8841ba586fbbdccd Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:21:11 -0700 Subject: [PATCH 17/48] fix(langsmith): json.dumps with default=str so non-serializable metadata does not crash batch flush (#42424) * fix(langsmith): json.dumps with default=str so non-serializable metadata does not crash batch flush Serialize the runs/batch payload with json.dumps(default=str, allow_nan=False) and send it as content= with an explicit Content-Type, so datetime, Decimal and similar metadata values no longer raise TypeError and drop the batch. Forward content= on the AsyncHTTPHandler retry path so a retried batch re-sends the identical body Replaces #39133, which was cut from the retired staging branch and conflicts with main Co-authored-by: Damien Smrt Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(langsmith): drop test docstrings and replace monkeypatch with a client-injecting handler Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(langsmith): add live e2e for non-native metadata reaching LangSmith Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(langsmith): scope the e2e docstring to the values the test injects Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(http_handler): close injected retry clients Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): deselect the LangSmith live e2e on the stage-mirror stack Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Damien Smrt Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/e2e-stack/select_tests.py | 1 + litellm/integrations/langsmith.py | 5 +- litellm/llms/custom_httpx/http_handler.py | 3 + tests/e2e/coverage_registry/logging.yaml | 1 + .../test_langsmith_batch_serialization_e2e.py | 125 +++++++++++++++++ .../test_langsmith_unit_test.py | 8 +- .../integrations/test_langsmith_init.py | 127 +++++++++++++++++- .../llms/custom_httpx/test_http_handler.py | 44 ++++++ 8 files changed, 304 insertions(+), 10 deletions(-) create mode 100644 tests/e2e/logging/test_langsmith_batch_serialization_e2e.py diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py index 2386b184e54..492c52233cd 100644 --- a/.github/e2e-stack/select_tests.py +++ b/.github/e2e-stack/select_tests.py @@ -10,6 +10,7 @@ UNSUPPORTED: Final = re.compile( r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$" r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$" r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$" + r"|^tests/e2e/logging/test_langsmith_batch_serialization_e2e\.py$" ) HARNESS: Final = re.compile( r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$" diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 352fcdf90f3..991b0411ee8 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -1,6 +1,7 @@ #### What this does #### # On success, logs events to Langsmith import asyncio +import json import os import random import traceback @@ -415,7 +416,7 @@ class LangsmithLogger(CustomBatchLogger): langsmith_api_key: Final = credentials["LANGSMITH_API_KEY"] langsmith_tenant_id: Final = credentials.get("LANGSMITH_TENANT_ID") url: Final = self._add_endpoint_to_url(langsmith_api_base, "runs/batch") - headers: Final = {"x-api-key": langsmith_api_key} + headers: Final = {"x-api-key": langsmith_api_key, "Content-Type": "application/json"} if langsmith_tenant_id: headers["x-tenant-id"] = langsmith_tenant_id elements_to_log: Final = [queue_object["data"] for queue_object in queue_objects] @@ -426,7 +427,7 @@ class LangsmithLogger(CustomBatchLogger): verbose_logger.debug("[LANGSMITH MOCK] Mock mode enabled - API calls will be intercepted") response: Final = await self.async_httpx_client.post( url=url, - json={"post": elements_to_log}, + content=json.dumps({"post": elements_to_log}, default=str, allow_nan=False), headers=headers, ) response.raise_for_status() diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index fcc05e54bc5..8ea46a6b261 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -845,6 +845,7 @@ class AsyncHTTPHandler: params=params, headers=headers, stream=stream, + content=content, ) finally: await new_client.aclose() @@ -985,6 +986,7 @@ class AsyncHTTPHandler: params=params, headers=headers, stream=stream, + content=content, ) finally: await new_client.aclose() @@ -1051,6 +1053,7 @@ class AsyncHTTPHandler: params=params, headers=headers, stream=stream, + content=content, ) finally: await new_client.aclose() diff --git a/tests/e2e/coverage_registry/logging.yaml b/tests/e2e/coverage_registry/logging.yaml index 1f2f1d64711..7c83e4d3aea 100644 --- a/tests/e2e/coverage_registry/logging.yaml +++ b/tests/e2e/coverage_registry/logging.yaml @@ -13,6 +13,7 @@ - {id: logging.otel.failure.exports_metric, module: logging, tier: P0, event: failure, assertions: [exports_metric], exercised_on: [chat_completions, messages], source: "integrations/otel/logger.py", rationale: "Error spans for observability continuity"} - {id: logging.braintrust.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions, messages], source: "integrations/braintrust_logging.py", rationale: "Evals platform spend"} - {id: logging.langsmith.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions, messages], source: "integrations/langsmith.py", rationale: "LangChain ecosystem"} +- {id: logging.langsmith.success.serializes_non_native_metadata, module: logging, tier: P1, event: success, assertions: [serializes_non_native_metadata], exercised_on: [sdk], source: "integrations/langsmith.py", rationale: "datetime/Decimal/UUID metadata used to TypeError in json.dumps and drop the whole batch (LIT-8310)"} - {id: logging.arize.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions, embeddings], source: "integrations/arize/arize.py", rationale: "ML-ops observability"} - {id: logging.mlflow.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions], source: "integrations/mlflow.py", rationale: "Experiment tracking cost/run"} - {id: logging.opik.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions], source: "integrations/opik/opik.py", rationale: "Eval platform spend/case"} diff --git a/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py b/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py new file mode 100644 index 00000000000..874b2b6a045 --- /dev/null +++ b/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py @@ -0,0 +1,125 @@ +"""Live e2e: a LangSmith batch whose metadata holds non JSON-native Python values +(datetime, Decimal) must reach the real LangSmith API instead of dying in +json.dumps and dropping the whole batch. Only the SDK path can put such values +into the batch (the proxy JSON-decodes request metadata), so this test drives +litellm.acompletion in-process against the real OpenAI API with a LangsmithLogger +injected per request, flushes the batch, and reads the run back by id through +LangSmith's own API. Nothing is mocked. +""" + +from __future__ import annotations + +import asyncio +import datetime +import decimal +import os +import time +import uuid +from dataclasses import dataclass +from typing import Final + +import pytest +from e2e_config import CHEAP_OPENAI_MODEL, POLL_INTERVAL, POLL_TIMEOUT, unique_marker +from e2e_http import Headers, Success, get_external +from pydantic import BaseModel, ConfigDict, Field, JsonValue + +import litellm +from litellm.integrations.langsmith import LangsmithLogger + +pytestmark = pytest.mark.e2e + + +class LangsmithHeaders(Headers): + x_api_key: str = Field(serialization_alias="x-api-key") + + +class LangsmithRunExtra(BaseModel): + model_config = ConfigDict(extra="allow") + requester_metadata: dict[str, JsonValue] | None = None + + +class LangsmithRun(BaseModel): + id: str + session_name: str | None = None + extra: LangsmithRunExtra + + +@dataclass(frozen=True, slots=True) +class LangsmithCreds: + api_key: str + base_url: str + project: str + + +def load_langsmith_creds() -> LangsmithCreds: + api_key = os.getenv("LANGSMITH_API_KEY") + if not api_key: + pytest.fail("LangSmith e2e requires LANGSMITH_API_KEY; missing credentials is a hard failure, not a skip") + if os.getenv("LANGSMITH_MOCK"): + pytest.fail("LANGSMITH_MOCK is set; this e2e must hit the real LangSmith API") + return LangsmithCreds( + api_key=api_key, + base_url=(os.getenv("LANGSMITH_BASE_URL") or "https://api.smith.langchain.com").rstrip("/"), + project=os.getenv("LANGSMITH_PROJECT") or "litellm-e2e", + ) + + +def _fetch_run(creds: LangsmithCreds, run_id: uuid.UUID) -> LangsmithRun | None: + result = get_external( + f"{creds.base_url}/runs/{run_id}", + response_type=LangsmithRun, + headers=LangsmithHeaders(x_api_key=creds.api_key), + ) + match result: + case Success(data=run): + return run + case _: + return None + + +def _poll_run(creds: LangsmithCreds, run_id: uuid.UUID) -> LangsmithRun: + deadline: Final = time.monotonic() + POLL_TIMEOUT + while time.monotonic() < deadline: + run = _fetch_run(creds, run_id) + if run is not None: + return run + time.sleep(POLL_INTERVAL) + pytest.fail(f"LangSmith run {run_id} never appeared within {POLL_TIMEOUT}s; the batch flush dropped it") + + +class TestLangsmithBatchSerialization: + @pytest.mark.asyncio + @pytest.mark.covers("logging.langsmith.success.serializes_non_native_metadata") + async def test_non_json_native_metadata_reaches_langsmith(self) -> None: + creds: Final = load_langsmith_creds() + logger: Final = LangsmithLogger( + langsmith_api_key=creds.api_key, langsmith_project=creds.project, langsmith_base_url=creds.base_url + ) + assert not logger.is_mock_mode, "LangsmithLogger initialised in mock mode; this e2e needs the real API" + marker: Final = unique_marker() + run_id: Final = uuid.uuid4() + created_at: Final = datetime.datetime(2026, 1, 2, 3, 4, 5, tzinfo=datetime.timezone.utc) + spend: Final = decimal.Decimal("0.0042") + response: Final = await litellm.acompletion( + model=f"openai/{CHEAP_OPENAI_MODEL}", + messages=[{"role": "user", "content": f"Reply with the single word ok ({marker})"}], + max_completion_tokens=5, + callbacks=[logger], + metadata={"run_id": str(run_id), "metadata": {"marker": marker, "created_at": created_at, "spend": spend}}, + ) + assert isinstance(response, litellm.ModelResponse) and response.id, ( + "a non-streaming completion must return a ModelResponse before the batch flush is meaningful" + ) + enqueue_deadline: Final = time.monotonic() + POLL_TIMEOUT + while time.monotonic() < enqueue_deadline and len(logger.log_queue) == 0: + await asyncio.sleep(0.5) + assert len(logger.log_queue) == 1, ( + f"the completion must be queued for the LangSmith batch, got {len(logger.log_queue)} queued entries" + ) + await logger.async_send_batch() + run: Final = _poll_run(creds, run_id) + requester_metadata: Final = run.extra.requester_metadata + assert requester_metadata is not None, "the run must carry the caller metadata under extra.requester_metadata" + assert requester_metadata["marker"] == marker + assert requester_metadata["created_at"] == str(created_at) + assert requester_metadata["spend"] == str(spend) diff --git a/tests/logging_callback_tests/test_langsmith_unit_test.py b/tests/logging_callback_tests/test_langsmith_unit_test.py index 17cd63d8974..341f71f3b4a 100644 --- a/tests/logging_callback_tests/test_langsmith_unit_test.py +++ b/tests/logging_callback_tests/test_langsmith_unit_test.py @@ -356,10 +356,12 @@ async def test_langsmith_key_based_logging(): # tenant_id should not be in headers if not provided assert "x-tenant-id" not in call_args[1]["headers"] + assert call_args[1]["headers"]["Content-Type"] == "application/json" + # Verify the request body contains the expected data - request_body = call_args[1]["json"] + request_body = json.loads(call_args[1]["content"]) assert "post" in request_body - assert len(request_body["post"]) == 1 # Should contain one run + assert len(request_body["post"]) == 1 # EXPECTED BODY expected_body = { @@ -404,7 +406,7 @@ async def test_langsmith_key_based_logging(): } # Print both bodies for debugging - actual_body = call_args[1]["json"] + actual_body = json.loads(call_args[1]["content"]) print("\nExpected body:") print(json.dumps(expected_body, indent=2)) print("\nActual body:") diff --git a/tests/test_litellm/integrations/test_langsmith_init.py b/tests/test_litellm/integrations/test_langsmith_init.py index f56d2310e73..9c0650baee6 100644 --- a/tests/test_litellm/integrations/test_langsmith_init.py +++ b/tests/test_litellm/integrations/test_langsmith_init.py @@ -1,13 +1,17 @@ import asyncio +import json import os +from datetime import datetime, timezone +from decimal import Decimal from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest - import litellm from litellm.integrations.langsmith import LangsmithLogger +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.integrations.langsmith import LangsmithQueueObject @@ -219,6 +223,121 @@ class TestLangsmithLoggerInit: assert len(logger.log_queue) == 1 +class TestLangsmithBatchSerialization: + async def _logger(self, transport_handler, tenant_id=None): + logger = LangsmithLogger( + langsmith_api_key="test-key", + langsmith_project="test-project", + langsmith_base_url="https://api.smith.langchain.com", + langsmith_tenant_id=tenant_id, + ) + if logger._flush_task is not None: + logger._flush_task.cancel() + handler = AsyncHTTPHandler() + await handler.client.aclose() + handler.client = httpx.AsyncClient( + transport=httpx.MockTransport(transport_handler) + ) + logger.async_httpx_client = handler + return logger + + @staticmethod + def _capturing_transport(captured): + async def handle(request: httpx.Request) -> httpx.Response: + captured.append(request) + return httpx.Response(200, request=request, json={"ok": True}) + + return handle + + def _queue(self, logger, extra): + return [ + LangsmithQueueObject( + data={"id": "run-1", "name": "LLMRun", "extra": extra}, + credentials=logger.default_credentials, + ) + ] + + @pytest.mark.asyncio + async def test_datetime_and_decimal_metadata_reach_langsmith_as_strings(self): + captured: list[httpx.Request] = [] # mutable-ok: transport capture buffer + logger = await self._logger(self._capturing_transport(captured)) + logger.log_queue = self._queue( + logger, + { + "created_at": datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc), + "spend": Decimal("0.0042"), + }, + ) + + await logger.async_send_batch() + + assert len(captured) == 1, "batch was dropped instead of being sent" + body = json.loads(captured[0].content) + assert body["post"][0]["extra"] == { + "created_at": "2026-01-02 03:04:05+00:00", + "spend": "0.0042", + } + await logger.async_httpx_client.client.aclose() + + @pytest.mark.asyncio + async def test_nan_metadata_is_dropped_instead_of_shipping_invalid_json(self): + captured: list[httpx.Request] = [] # mutable-ok: transport capture buffer + logger = await self._logger(self._capturing_transport(captured)) + logger.log_queue = self._queue(logger, {"score": float("nan")}) + + await logger.async_send_batch() + + assert captured == [], ( + "nan metadata must abort the batch: a bare NaN token is invalid JSON and LangSmith rejects it" + ) + await logger.async_httpx_client.client.aclose() + + @pytest.mark.asyncio + async def test_batch_declares_json_content_type(self): + captured: list[httpx.Request] = [] # mutable-ok: transport capture buffer + logger = await self._logger(self._capturing_transport(captured)) + logger.log_queue = self._queue(logger, {"model": "gpt-4.1-mini"}) + + await logger.async_send_batch() + + assert captured[0].headers["content-type"] == "application/json", ( + "a content= body carries no implicit content type; LangSmith refuses it without this header" + ) + assert captured[0].url.path.endswith("/api/v1/runs/batch") + assert captured[0].headers["x-api-key"] == "test-key" + assert "x-tenant-id" not in captured[0].headers + await logger.async_httpx_client.client.aclose() + + @pytest.mark.asyncio + async def test_tenant_id_is_forwarded_on_the_batch_request(self): + captured: list[httpx.Request] = [] # mutable-ok: transport capture buffer + logger = await self._logger( + self._capturing_transport(captured), tenant_id="tenant-1" + ) + logger.log_queue = self._queue(logger, {"model": "gpt-4.1-mini"}) + + await logger.async_send_batch() + + assert captured[0].headers["x-tenant-id"] == "tenant-1" + await logger.async_httpx_client.client.aclose() + + @pytest.mark.asyncio + async def test_langsmith_error_response_does_not_propagate(self): + captured: list[httpx.Request] = [] # mutable-ok: transport capture buffer + + async def reject(request: httpx.Request) -> httpx.Response: + captured.append(request) + return httpx.Response(422, request=request, text="bad run") + + logger = await self._logger(reject) + logger.log_queue = self._queue(logger, {"model": "gpt-4.1-mini"}) + + await logger.async_send_batch() + + assert len(captured) == 1, "the batch never left the process" + await logger.async_httpx_client.client.aclose() + + class TestLangsmithPrepareLogData: """Regression test for #24001: _prepare_log_data must inject usage_metadata into outputs so LangSmith's Cost column is populated.""" @@ -544,12 +663,10 @@ async def test_events_appended_during_flush_are_not_dropped(): credentials=logger.default_credentials, data={"id": "late"} ) - async def fake_post( - url: str, json: dict[str, list[dict[str, str]]], headers: dict[str, str] - ) -> MagicMock: + async def fake_post(url: str, content: str, headers: dict[str, str]) -> MagicMock: if not sent_batches: logger.log_queue.append(late_event) - sent_batches.append(json["post"]) + sent_batches.append(json.loads(content)["post"]) response = MagicMock() response.status_code = 200 response.raise_for_status = MagicMock() diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index 33272a1a9e4..8358d15d30e 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -6,6 +6,8 @@ import pathlib import ssl import threading import weakref +from collections.abc import Callable, Mapping +from typing import Final from unittest.mock import MagicMock, patch import certifi @@ -23,6 +25,7 @@ from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, get_ssl_configuration, ) +from litellm.types.llms.custom_http import VerifyTypes @pytest.mark.asyncio @@ -1396,6 +1399,47 @@ async def test_finalizer_on_live_loop_disposes_foreign_loop_session_without_sche assert session.closed +class _RetryClientHandler(AsyncHTTPHandler): + def __init__(self, first: httpx.AsyncClient, retry: httpx.AsyncClient) -> None: + self._retry_client: Final = retry + super().__init__() + self.client = first + + def create_client( + self, + timeout: float | httpx.Timeout | None = None, + event_hooks: Mapping[str, list[Callable[..., object]]] | None = None, + ssl_verify: VerifyTypes | None = None, + shared_session: ClientSession | None = None, + ) -> httpx.AsyncClient: + return self._retry_client + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["post", "put", "patch", "delete"]) +async def test_connection_error_retry_forwards_content(method: str): + captured: list[bytes] = [] # mutable-ok: async closure capture buffer + + async def raise_connection_error(request: httpx.Request) -> httpx.Response: + raise httpx.RemoteProtocolError("connection dropped", request=request) + + async def capture_and_succeed(request: httpx.Request) -> httpx.Response: + captured.append(request.content) + return httpx.Response(200, request=request) + + first: Final = httpx.AsyncClient(transport=httpx.MockTransport(raise_connection_error)) + retry: Final = httpx.AsyncClient(transport=httpx.MockTransport(capture_and_succeed)) + async with first, retry: + handler: Final = _RetryClientHandler(first=first, retry=retry) + + body = b'{"post": ["run1"]}' + await getattr(handler, method)("https://api.example.com/runs/batch", content=body) + + assert captured == [body], "the retried request must carry the same content= body" + await handler.close() + + + @pytest.fixture def forward_proxy_server(): """Plain HTTP forward proxy that records the absolute URIs it is asked to fetch.""" From 569ccaece9328646535e5ab6ac163e530665c771 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:21:20 -0700 Subject: [PATCH 18/48] fix(proxy): make the lazy OpenAPI snapshot byte-identical on every Python version (#42519) Python 3.13+ strips the common indentation of docstrings at compile time and 3.12 keeps it, and the 429 error description in ERROR_RESPONSES came straight from RateLimitError.__doc__, so regenerating litellm/proxy/_lazy_openapi_snapshot.json on a 3.13+ venv produced a one-line diff that the check-ui-api-types job (Python 3.12) rejected. Run the docstring through inspect.cleandoc before it lands in the spec, regenerate the snapshot once, and pin the behavior with a test Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/_lazy_openapi_snapshot.json | 2 +- litellm/proxy/common_utils/swagger_utils.py | 7 ++++++- .../proxy/common_utils/test_swagger_utils.py | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/proxy/common_utils/test_swagger_utils.py diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 4dbca14b917..87f28624d58 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -19919,7 +19919,7 @@ } } }, - "description": "\n Unified rate-limit error.\n\n Every rate-limit condition surfaced by litellm \u2014 whether it originated from\n an upstream LLM provider, a vendor batch endpoint, or one of litellm's own\n proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\n max-iterations, etc.) \u2014 is raised as an instance of this class.\n\n The :attr:`category` attribute lets callers distinguish the source. See\n :class:`RateLimitErrorCategory` for the available values.\n " + "description": "Unified rate-limit error.\n\nEvery rate-limit condition surfaced by litellm \u2014 whether it originated from\nan upstream LLM provider, a vendor batch endpoint, or one of litellm's own\nproxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\nmax-iterations, etc.) \u2014 is raised as an instance of this class.\n\nThe :attr:`category` attribute lets callers distinguish the source. See\n:class:`RateLimitErrorCategory` for the available values." }, "500": { "content": { diff --git a/litellm/proxy/common_utils/swagger_utils.py b/litellm/proxy/common_utils/swagger_utils.py index 2609a98a997..83480bf161d 100644 --- a/litellm/proxy/common_utils/swagger_utils.py +++ b/litellm/proxy/common_utils/swagger_utils.py @@ -1,3 +1,4 @@ +import inspect from typing import Any, Final from pydantic import BaseModel, Field @@ -31,11 +32,15 @@ def get_status_code(exception): return 500 # Internal Server Error as default +def _error_description(exception: type[Exception]) -> str: + return inspect.cleandoc(exception.__doc__) if exception.__doc__ else exception.__name__ + + # Create error responses ERROR_RESPONSES: Final = { get_status_code(exception): { "model": ErrorResponse, - "description": exception.__doc__ or exception.__name__, + "description": _error_description(exception), } for exception in LITELLM_EXCEPTION_TYPES } diff --git a/tests/test_litellm/proxy/common_utils/test_swagger_utils.py b/tests/test_litellm/proxy/common_utils/test_swagger_utils.py new file mode 100644 index 00000000000..659d2ad2941 --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_swagger_utils.py @@ -0,0 +1,18 @@ +import inspect + +from litellm.exceptions import RateLimitError +from litellm.proxy.common_utils.swagger_utils import ERROR_RESPONSES, _error_description + + +class _ChildWithoutDoc(RateLimitError): + pass + + +def test_error_response_descriptions_carry_no_docstring_indentation(): + assert ERROR_RESPONSES[429]["description"] == inspect.cleandoc(RateLimitError.__doc__ or "") + for response in ERROR_RESPONSES.values(): + assert response["description"] == inspect.cleandoc(response["description"]) + + +def test_error_description_falls_back_to_the_class_name_without_an_own_docstring(): + assert _error_description(_ChildWithoutDoc) == "_ChildWithoutDoc" From 97a6c27bee7d9c3f12f122b9769d85f01ebda159 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 19:21:32 +0000 Subject: [PATCH 19/48] test(rust_bridge): drop route dispatch assertions, test the bridge directly (#42536) Co-authored-by: ryan Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../spend_tracking/test_budget_reservation.py | 251 +----------------- .../proxy/spend_tracking/test_input_tokens.py | 164 +----------- .../rust_bridge/test_token_counter.py | 173 +++--------- .../rust_bridge/test_tokenizer.py | 157 +++-------- 4 files changed, 68 insertions(+), 677 deletions(-) diff --git a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py index 13488106df4..9df8e6f4d67 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py @@ -1,10 +1,8 @@ from __future__ import annotations -import json import math from datetime import datetime, timedelta, timezone -from types import MappingProxyType -from typing import Final, cast +from typing import Final import pytest @@ -24,16 +22,12 @@ from litellm.proxy.common_utils.user_api_key_cache import ( ) from litellm.proxy.spend_tracking.budget_reservation import ( _get_team_member_budget_counter, - count_request_input_tokens, estimate_request_max_cost, release_unbound_budget_reservation, reserve_budget_for_request, ) from litellm.proxy.utils import ProxyLogging from litellm.router import Router -from litellm.rust_bridge import bindings, configuration -from litellm.rust_bridge import token_counter as rust_token_counter -from litellm.rust_bridge import tokenizer as tokenizer_dispatch from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo TOKEN_COUNTING_ROUTES: Final = ( @@ -222,249 +216,6 @@ def test_deployment_pricing_update_invalidates_cached_estimate() -> None: assert math.isclose(after, before * 1000) -ANTHROPIC_TOKENIZER_MODEL: Final = "claude-sonnet-4-5-20250929" -CL100K_MODEL: Final = "gpt-4" -O200K_MODEL: Final = "gpt-4o" -RUST_COUNTED_BODY: Final = {"model": ANTHROPIC_TOKENIZER_MODEL, "max_tokens": 16, "messages": ANTHROPIC_MESSAGES} -RUST_INPUT_TOKENS: Final = 4_321 -RUST_INPUT_TOKENS_BY_TOKENIZER: Final = MappingProxyType( - {"anthropic": RUST_INPUT_TOKENS, "cl100k_base": 1_234, "o200k_base": 2_345} -) - - -class _FakeDeclined(Exception): - pass - - -class _FakeUpstream(Exception): - pass - - -class _FakeTokenizer: - """Stands in for one shared native `Tokenizer`; only its name identifies it.""" - - def __init__(self, name: str, json: str | None = None) -> None: - self.name = name - self.json = json - - -def _fake_native_tokenizers(monkeypatch: pytest.MonkeyPatch, anthropic_json: str | None = None) -> None: - """Point the counter's tokenizer lookups at fakes; the codec path keeps falling back to Python.""" - fakes: Final = {name: _FakeTokenizer(name) for name in ("cl100k_base", "o200k_base")} - anthropic: Final = _FakeTokenizer("anthropic", anthropic_json) - monkeypatch.setattr(tokenizer_dispatch, "native_encoding", fakes.__getitem__) - monkeypatch.setattr(tokenizer_dispatch, "native_anthropic", lambda: anthropic) - - -class _FakeNative: - RustBridgeDeclined = _FakeDeclined - RustUpstreamError = _FakeUpstream - - -class _RecordingCounter: - """Stands in for one native counter; records `(tokenizer, body)` on the shared factory.""" - - def __init__(self, factory: _RecordingFactory, tokenizer: rust_token_counter.RustTokenizer) -> None: - self.factory = factory - self.tokenizer = tokenizer - - async def acount_request(self, body: bytes) -> object: - self.factory.calls.append((self.tokenizer, body)) - return {"model": "", "input_tokens": RUST_INPUT_TOKENS_BY_TOKENIZER[self.tokenizer]} - - -class _RecordingFactory: - """Stands in for the native `TokenCounter` class, built over a loaded `Tokenizer`.""" - - def __init__(self) -> None: - self.calls: list[tuple[rust_token_counter.RustTokenizer, bytes]] = [] - - def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _RecordingCounter: - return _RecordingCounter(self, cast(rust_token_counter.RustTokenizer, tokenizer.name)) - - -class _DecliningCounter: - async def acount_request(self, body: bytes) -> object: - raise _FakeDeclined("unsupported content block") - - -class _DecliningFactory: - def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _DecliningCounter: - return _DecliningCounter() - - -@pytest.fixture -def rust_counter(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeNative()) - _fake_native_tokenizers(monkeypatch) - rust_token_counter._counter.cache_clear() - configuration.reset_rust_configuration() - yield - rust_token_counter.TOKEN_COUNTER.reset() - rust_token_counter._counter.cache_clear() - configuration.reset_rust_configuration() - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("route", "request_body"), - ( - ("/v1/messages", RUST_COUNTED_BODY), - ("/v1/chat/completions", {"model": ANTHROPIC_TOKENIZER_MODEL, "messages": ANTHROPIC_MESSAGES}), - ("/v1/completions", {"model": ANTHROPIC_TOKENIZER_MODEL, "prompt": "hi"}), - ("/v1/responses", {"model": ANTHROPIC_TOKENIZER_MODEL, "input": "hi"}), - ("/v1/embeddings", {"model": ANTHROPIC_TOKENIZER_MODEL, "input": ["hi"]}), - ("/v1/rerank", {"model": ANTHROPIC_TOKENIZER_MODEL, "query": "hi", "documents": ["a"]}), - ), -) -async def test_rust_count_replaces_python_tokenizing_on_every_llm_route( - rust_counter: None, route: str, request_body: dict -) -> None: - factory: Final = _RecordingFactory() - litellm.rust(True) - rust_token_counter.TOKEN_COUNTER.override(factory) - raw_body: Final = json.dumps(request_body).encode() - - counts: Final = await count_request_input_tokens( - request_body=request_body, route=route, llm_router=None, raw_body=raw_body - ) - - assert dict(counts) == {ANTHROPIC_TOKENIZER_MODEL: RUST_INPUT_TOKENS} - assert factory.calls == [("anthropic", raw_body)] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("model", (CL100K_MODEL, "azure/gpt-35-turbo", "gemini/gemini-2.5-pro", "my-router-alias")) -async def test_tiktoken_cl100k_models_are_counted_by_rust(rust_counter: None, model: str) -> None: - factory: Final = _RecordingFactory() - litellm.rust(True) - rust_token_counter.TOKEN_COUNTER.override(factory) - body: Final = {"model": model, "messages": ANTHROPIC_MESSAGES} - raw_body: Final = json.dumps(body).encode() - - counts: Final = await count_request_input_tokens( - request_body=body, route="/v1/chat/completions", llm_router=None, raw_body=raw_body - ) - - assert dict(counts) == {model: RUST_INPUT_TOKENS_BY_TOKENIZER["cl100k_base"]} - assert factory.calls == [("cl100k_base", raw_body)] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("model", (O200K_MODEL, "gpt-5", "o3", "gpt-4.1", "chatgpt-4o-latest")) -async def test_tiktoken_o200k_models_are_counted_by_rust(rust_counter: None, model: str) -> None: - factory: Final = _RecordingFactory() - litellm.rust(True) - rust_token_counter.TOKEN_COUNTER.override(factory) - body: Final = {"model": model, "messages": ANTHROPIC_MESSAGES} - raw_body: Final = json.dumps(body).encode() - - counts: Final = await count_request_input_tokens( - request_body=body, route="/v1/chat/completions", llm_router=None, raw_body=raw_body - ) - - assert dict(counts) == {model: RUST_INPUT_TOKENS_BY_TOKENIZER["o200k_base"]} - assert factory.calls == [("o200k_base", raw_body)] - - -@pytest.mark.asyncio -async def test_multi_model_request_counts_once_per_tokenizer_and_python_for_the_rest(rust_counter: None) -> None: - factory: Final = _RecordingFactory() - litellm.rust(True) - rust_token_counter.TOKEN_COUNTER.override(factory) - models: Final = ( - CL100K_MODEL, - ANTHROPIC_TOKENIZER_MODEL, - "gemini/gemini-2.5-pro", - O200K_MODEL, - "gpt-5", - "replicate/meta/llama-2-70b-chat", - ) - body: Final = {"model": list(models), "messages": ANTHROPIC_MESSAGES} - raw_body: Final = json.dumps(body).encode() - python_counts: Final = await count_request_input_tokens( - request_body=body, route="/v1/chat/completions", llm_router=None - ) - - counts: Final = await count_request_input_tokens( - request_body=body, route="/v1/chat/completions", llm_router=None, raw_body=raw_body - ) - - assert factory.calls == [("cl100k_base", raw_body), ("anthropic", raw_body), ("o200k_base", raw_body)] - assert dict(counts) == { - CL100K_MODEL: RUST_INPUT_TOKENS_BY_TOKENIZER["cl100k_base"], - "gemini/gemini-2.5-pro": RUST_INPUT_TOKENS_BY_TOKENIZER["cl100k_base"], - ANTHROPIC_TOKENIZER_MODEL: RUST_INPUT_TOKENS, - O200K_MODEL: RUST_INPUT_TOKENS_BY_TOKENIZER["o200k_base"], - "gpt-5": RUST_INPUT_TOKENS_BY_TOKENIZER["o200k_base"], - "replicate/meta/llama-2-70b-chat": python_counts["replicate/meta/llama-2-70b-chat"], - } - assert counts["replicate/meta/llama-2-70b-chat"] not in RUST_INPUT_TOKENS_BY_TOKENIZER.values() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("model", (ANTHROPIC_TOKENIZER_MODEL, CL100K_MODEL, O200K_MODEL)) -async def test_rust_decline_falls_back_to_python_count(rust_counter: None, model: str) -> None: - litellm.rust(True) - rust_token_counter.TOKEN_COUNTER.override(_DecliningFactory()) - body: Final = {**RUST_COUNTED_BODY, "model": model} - python_counts: Final = await count_request_input_tokens(request_body=body, route="/v1/messages", llm_router=None) - - counts: Final = await count_request_input_tokens( - request_body=body, - route="/v1/messages", - llm_router=None, - raw_body=json.dumps(body).encode(), - ) - - assert dict(counts) == dict(python_counts) - assert counts[model] not in RUST_INPUT_TOKENS_BY_TOKENIZER.values() - - -@pytest.mark.asyncio -async def test_disabled_rust_never_sees_the_raw_body(rust_counter: None) -> None: - factory: Final = _RecordingFactory() - litellm.rust(False) - rust_token_counter.TOKEN_COUNTER.override(factory) - body: Final = {"model": [ANTHROPIC_TOKENIZER_MODEL, CL100K_MODEL, O200K_MODEL], "messages": ANTHROPIC_MESSAGES} - - counts: Final = await count_request_input_tokens( - request_body=body, - route="/v1/chat/completions", - llm_router=None, - raw_body=json.dumps(body).encode(), - ) - - assert factory.calls == [] - assert set(counts) == {ANTHROPIC_TOKENIZER_MODEL, CL100K_MODEL, O200K_MODEL} - assert not set(counts.values()) & set(RUST_INPUT_TOKENS_BY_TOKENIZER.values()) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("model", ("replicate/meta/llama-2-70b-chat", "meta-llama/Llama-3-8b", "text-davinci-003")) -async def test_models_without_a_rust_tokenizer_stay_in_python( - rust_counter: None, monkeypatch: pytest.MonkeyPatch, model: str -) -> None: - monkeypatch.setattr( - litellm, "open_ai_chat_completion_models", litellm.open_ai_chat_completion_models | {"text-davinci-003"} - ) - factory: Final = _RecordingFactory() - litellm.rust(True) - rust_token_counter.TOKEN_COUNTER.override(factory) - body: Final = {"model": model, "messages": ANTHROPIC_MESSAGES} - python_counts: Final = await count_request_input_tokens( - request_body=body, route="/v1/chat/completions", llm_router=None - ) - - counts: Final = await count_request_input_tokens( - request_body=body, route="/v1/chat/completions", llm_router=None, raw_body=json.dumps(body).encode() - ) - - assert factory.calls == [] - assert dict(counts) == dict(python_counts) - assert counts[model] not in RUST_INPUT_TOKENS_BY_TOKENIZER.values() - - @pytest.mark.asyncio @pytest.mark.parametrize( "expiry_offset, expected_max_budget", diff --git a/tests/test_litellm/proxy/spend_tracking/test_input_tokens.py b/tests/test_litellm/proxy/spend_tracking/test_input_tokens.py index 49bbe148386..a1da6c79cd7 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_input_tokens.py +++ b/tests/test_litellm/proxy/spend_tracking/test_input_tokens.py @@ -2,180 +2,18 @@ from __future__ import annotations -import json from types import MappingProxyType -from typing import Final, cast +from typing import Final import pytest -import litellm from litellm.proxy.spend_tracking.input_tokens import ( TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, count_input_tokens, count_input_tokens_for_model, ) -from litellm.rust_bridge import bindings, configuration, token_counter -from litellm.rust_bridge import tokenizer as tokenizer_dispatch -from litellm.rust_bridge.token_counter import RustTokenizer -ANTHROPIC_MODEL: Final = "claude-sonnet-4-5-20250929" CL100K_MODEL: Final = "gpt-4" -O200K_MODEL: Final = "gpt-4o" -PYTHON_ONLY_MODEL: Final = "replicate/meta/llama-2-70b-chat" -MESSAGES: Final = [{"role": "user", "content": "hello"}] -RUST_TOKENS: Final = 777 - - -class _FakeDeclined(Exception): - pass - - -class _FakeUpstream(Exception): - pass - - -class _FakeTokenizer: - """Stands in for one shared native `Tokenizer`; only its name identifies it.""" - - def __init__(self, name: str, json: str | None = None) -> None: - self.name = name - self.json = json - - -def _fake_native_tokenizers(monkeypatch: pytest.MonkeyPatch, anthropic_json: str | None = None) -> None: - """Point the counter's tokenizer lookups at fakes; the codec path keeps falling back to Python.""" - fakes: Final = {name: _FakeTokenizer(name) for name in ("cl100k_base", "o200k_base")} - anthropic: Final = _FakeTokenizer("anthropic", anthropic_json) - monkeypatch.setattr(tokenizer_dispatch, "native_encoding", fakes.__getitem__) - monkeypatch.setattr(tokenizer_dispatch, "native_anthropic", lambda: anthropic) - - -class _FakeNative: - RustBridgeDeclined = _FakeDeclined - RustUpstreamError = _FakeUpstream - - -class _RecordingCounter: - def __init__(self, factory: _RecordingFactory, tokenizer: RustTokenizer) -> None: - self.factory = factory - self.tokenizer = tokenizer - - async def acount_request(self, body: bytes) -> object: - self.factory.calls.append((self.tokenizer, body)) - return {"model": "", "input_tokens": RUST_TOKENS} - - -class _RecordingFactory: - def __init__(self) -> None: - self.calls: list[tuple[RustTokenizer, bytes]] = [] - - def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _RecordingCounter: - return _RecordingCounter(self, cast(RustTokenizer, tokenizer.name)) - - -class _DecliningCounter: - async def acount_request(self, body: bytes) -> object: - raise _FakeDeclined("unsupported request shape") - - -class _DecliningFactory: - def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _DecliningCounter: - return _DecliningCounter() - - -@pytest.fixture(autouse=True) -def _reset_bridge(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeNative()) - _fake_native_tokenizers(monkeypatch) - token_counter.TOKEN_COUNTER.reset() - token_counter._counter.cache_clear() - configuration.reset_rust_configuration() - yield - token_counter.TOKEN_COUNTER.reset() - token_counter._counter.cache_clear() - configuration.reset_rust_configuration() - - -def _body(model: object) -> tuple[dict[str, object], bytes]: - body: Final = {"model": model, "messages": MESSAGES} - return body, json.dumps(body).encode() - - -@pytest.mark.asyncio -async def test_models_sharing_a_tokenizer_are_counted_once_and_merged() -> None: - factory: Final = _RecordingFactory() - litellm.rust(True) - token_counter.TOKEN_COUNTER.override(factory) - request_body, raw_body = _body([ANTHROPIC_MODEL, CL100K_MODEL, O200K_MODEL, "gpt-5", PYTHON_ONLY_MODEL]) - - counts: Final = await count_input_tokens( - request_body=request_body, - raw_body=raw_body, - models=(ANTHROPIC_MODEL, CL100K_MODEL, O200K_MODEL, "gpt-5", PYTHON_ONLY_MODEL), - ) - - assert factory.calls == [("anthropic", raw_body), ("cl100k_base", raw_body), ("o200k_base", raw_body)] - assert dict(counts) == { - ANTHROPIC_MODEL: RUST_TOKENS, - CL100K_MODEL: RUST_TOKENS, - O200K_MODEL: RUST_TOKENS, - "gpt-5": RUST_TOKENS, - PYTHON_ONLY_MODEL: count_input_tokens_for_model(request_body=request_body, model=PYTHON_ONLY_MODEL), - } - - -@pytest.mark.asyncio -async def test_rust_disabled_counts_everything_in_python() -> None: - factory: Final = _RecordingFactory() - litellm.rust(False) - token_counter.TOKEN_COUNTER.override(factory) - request_body, raw_body = _body([ANTHROPIC_MODEL, CL100K_MODEL]) - - counts: Final = await count_input_tokens( - request_body=request_body, raw_body=raw_body, models=(ANTHROPIC_MODEL, CL100K_MODEL) - ) - - assert factory.calls == [] - assert dict(counts) == { - model: count_input_tokens_for_model(request_body=request_body, model=model) - for model in (ANTHROPIC_MODEL, CL100K_MODEL) - } - - -@pytest.mark.asyncio -async def test_missing_raw_body_counts_in_python() -> None: - factory: Final = _RecordingFactory() - litellm.rust(True) - token_counter.TOKEN_COUNTER.override(factory) - request_body, _ = _body(ANTHROPIC_MODEL) - - counts: Final = await count_input_tokens(request_body=request_body, raw_body=None, models=(ANTHROPIC_MODEL,)) - - assert factory.calls == [] - assert counts[ANTHROPIC_MODEL] == count_input_tokens_for_model(request_body=request_body, model=ANTHROPIC_MODEL) - - -@pytest.mark.asyncio -async def test_missing_binding_counts_in_python() -> None: - litellm.rust(True) - token_counter.TOKEN_COUNTER.override(None) - request_body, raw_body = _body(ANTHROPIC_MODEL) - - counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw_body, models=(ANTHROPIC_MODEL,)) - - assert counts[ANTHROPIC_MODEL] == count_input_tokens_for_model(request_body=request_body, model=ANTHROPIC_MODEL) - - -@pytest.mark.asyncio -async def test_declined_request_counts_in_python() -> None: - litellm.rust(True) - token_counter.TOKEN_COUNTER.override(_DecliningFactory()) - request_body, raw_body = _body(ANTHROPIC_MODEL) - - counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw_body, models=(ANTHROPIC_MODEL,)) - - assert counts[ANTHROPIC_MODEL] == count_input_tokens_for_model(request_body=request_body, model=ANTHROPIC_MODEL) - assert counts[ANTHROPIC_MODEL] != RUST_TOKENS @pytest.mark.asyncio diff --git a/tests/test_litellm/rust_bridge/test_token_counter.py b/tests/test_litellm/rust_bridge/test_token_counter.py index 3da291c898d..101fda629a7 100644 --- a/tests/test_litellm/rust_bridge/test_token_counter.py +++ b/tests/test_litellm/rust_bridge/test_token_counter.py @@ -1,8 +1,7 @@ -"""Tests for the Rust input token counter bridge. +"""Tests for the Rust input token counter bridge, called directly rather than through the route catalog. -The native factory is dependency-injected through ``TOKEN_COUNTER.override`` -so the fallback cases run without the compiled extension present. The parity -cases need the extension and are skipped when it is not built. +The factory is passed into ``native_count`` so the caching cases run without the compiled extension +present. The parity cases need the extension and are skipped when it is not built. """ from __future__ import annotations @@ -16,8 +15,7 @@ import pytest import litellm from litellm.constants import TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS from litellm.litellm_core_utils.token_counter import openai_tokenizer_encoding -from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model -from litellm.rust_bridge import bindings, configuration +from litellm.proxy.spend_tracking.input_tokens import count_input_tokens_for_model from litellm.rust_bridge import token_counter as bridge from litellm.rust_bridge import tokenizer as tokenizer_dispatch from litellm.rust_bridge._native import Tokenizer @@ -38,14 +36,6 @@ def _counted(body: dict[str, object], model: str) -> tuple[bytes, dict[str, obje return raw, json.loads(raw) -class _FakeDeclined(Exception): - pass - - -class _FakeUpstream(Exception): - pass - - class _FakeTokenizer: """Stands in for one shared native `Tokenizer`; only its name identifies it.""" @@ -54,29 +44,6 @@ class _FakeTokenizer: self.json = json -def _fake_native_tokenizers(monkeypatch: pytest.MonkeyPatch, anthropic_json: str | None = None) -> None: - """Point the counter's tokenizer lookups at fakes while the bridge is faked; the codec path - keeps falling back to Python. Parity tests that restore the real extension get the real - lookups back.""" - fakes: Final = {name: _FakeTokenizer(name) for name in ("cl100k_base", "o200k_base")} - anthropic: Final = _FakeTokenizer("anthropic", anthropic_json) - real_encoding: Final = tokenizer_dispatch.native_encoding - real_anthropic: Final = tokenizer_dispatch.native_anthropic - - def faked() -> bool: - return isinstance(bindings.get_native_bridge(), _FakeNative) - - monkeypatch.setattr( - tokenizer_dispatch, "native_encoding", lambda name: fakes[name] if faked() else real_encoding(name) - ) - monkeypatch.setattr(tokenizer_dispatch, "native_anthropic", lambda: anthropic if faked() else real_anthropic()) - - -class _FakeNative: - RustBridgeDeclined = _FakeDeclined - RustUpstreamError = _FakeUpstream - - class _RecordingCounter: def __init__(self, tokenizer: _FakeTokenizer, fast: bool) -> None: self.tokenizer = tokenizer @@ -100,57 +67,25 @@ class _RecordingFactory: return counter -class _RaisingCounter: - def __init__(self, error: Exception) -> None: - self.error = error - - async def acount_request(self, body: bytes) -> object: - raise self.error - - -class _RaisingFactory: - """Every counter it builds, for either tokenizer, raises `error` on count.""" - - def __init__(self, error: Exception) -> None: - self.error = error - - def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _RaisingCounter: - return _RaisingCounter(self.error) - - @pytest.fixture(autouse=True) -def _reset_bridge(monkeypatch: pytest.MonkeyPatch): - bridge.TOKEN_COUNTER.reset() +def _reset_counters(): bridge._counter.cache_clear() - configuration.reset_rust_configuration() - monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeNative()) - _fake_native_tokenizers(monkeypatch, anthropic_json=claude_json_str) yield - bridge.TOKEN_COUNTER.reset() bridge._counter.cache_clear() - configuration.reset_rust_configuration() + + +@pytest.fixture +def fake_tokenizers(monkeypatch: pytest.MonkeyPatch) -> None: + """Point the counter's tokenizer lookups at fakes so a recording factory sees which one it was built over.""" + fakes: Final = {name: _FakeTokenizer(name) for name in ("cl100k_base", "o200k_base")} + anthropic: Final = _FakeTokenizer("anthropic", claude_json_str) + monkeypatch.setattr(tokenizer_dispatch, "native_encoding", fakes.__getitem__) + monkeypatch.setattr(tokenizer_dispatch, "native_anthropic", lambda: anthropic) @pytest.mark.asyncio -@pytest.mark.parametrize("tokenizer", TOKENIZERS) -async def test_disabled_bridge_never_constructs_a_counter(tokenizer: bridge.RustTokenizer) -> None: +async def test_native_count_returns_typed_count_and_reuses_one_counter(fake_tokenizers: None) -> None: factory: Final = _RecordingFactory() - litellm.rust(False) - bridge.TOKEN_COUNTER.override(factory) - model: Final = MODEL_BY_TOKENIZER[tokenizer] - raw, request_body = _counted({"messages": [{"role": "user", "content": "hello"}]}, model) - - counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw, models=(model,)) - - assert counts[model] == count_input_tokens_for_model(request_body=request_body, model=model) - assert factory.counters == [] - - -@pytest.mark.asyncio -async def test_enabled_bridge_returns_typed_count_and_reuses_one_counter() -> None: - factory: Final = _RecordingFactory() - litellm.rust(True) - bridge.TOKEN_COUNTER.override(factory) first: Final = await bridge.native_count(factory, "anthropic", BODY) second: Final = await bridge.native_count(factory, "anthropic", BODY) @@ -166,10 +101,10 @@ async def test_enabled_bridge_returns_typed_count_and_reuses_one_counter() -> No @pytest.mark.asyncio @pytest.mark.parametrize("tokenizer", ("cl100k_base", "o200k_base")) -async def test_tiktoken_counter_is_built_over_the_shared_encoding_once(tokenizer: bridge.RustTokenizer) -> None: +async def test_tiktoken_counter_is_built_over_the_shared_encoding_once( + fake_tokenizers: None, tokenizer: bridge.RustTokenizer +) -> None: factory: Final = _RecordingFactory() - litellm.rust(True) - bridge.TOKEN_COUNTER.override(factory) first: Final = await bridge.native_count(factory, tokenizer, BODY) second: Final = await bridge.native_count(factory, tokenizer, BODY) @@ -183,10 +118,8 @@ async def test_tiktoken_counter_is_built_over_the_shared_encoding_once(tokenizer @pytest.mark.asyncio -async def test_each_tokenizer_gets_its_own_cached_counter() -> None: +async def test_each_tokenizer_gets_its_own_cached_counter(fake_tokenizers: None) -> None: factory: Final = _RecordingFactory() - litellm.rust(True) - bridge.TOKEN_COUNTER.override(factory) await bridge.native_count(factory, "anthropic", BODY) await bridge.native_count(factory, "cl100k_base", BODY) @@ -198,43 +131,6 @@ async def test_each_tokenizer_gets_its_own_cached_counter() -> None: assert [len(counter.bodies) for counter in factory.counters] == [2, 1, 2] -@pytest.mark.asyncio -async def test_missing_native_module_falls_back(monkeypatch: pytest.MonkeyPatch) -> None: - litellm.rust(True) - monkeypatch.setattr(bindings, "get_native_bridge", lambda: None) - raw, request_body = _counted({"messages": [{"role": "user", "content": "hello"}]}, MODEL) - - counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw, models=(MODEL,)) - - assert counts[MODEL] == count_input_tokens_for_model(request_body=request_body, model=MODEL) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("tokenizer", TOKENIZERS) -async def test_declined_request_falls_back(tokenizer: bridge.RustTokenizer) -> None: - litellm.rust(True) - bridge.TOKEN_COUNTER.override(_RaisingFactory(_FakeDeclined("request has no messages"))) - model: Final = MODEL_BY_TOKENIZER[tokenizer] - raw, request_body = _counted({"messages": [{"role": "user", "content": "hello"}]}, model) - - counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw, models=(model,)) - - assert counts[model] == count_input_tokens_for_model(request_body=request_body, model=model) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("tokenizer", TOKENIZERS) -async def test_runtime_failure_falls_back(tokenizer: bridge.RustTokenizer) -> None: - litellm.rust(True) - bridge.TOKEN_COUNTER.override(_RaisingFactory(RuntimeError("encode failed"))) - model: Final = MODEL_BY_TOKENIZER[tokenizer] - raw, request_body = _counted({"messages": [{"role": "user", "content": "hello"}]}, model) - - counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw, models=(model,)) - - assert counts[model] == count_input_tokens_for_model(request_body=request_body, model=model) - - @pytest.mark.parametrize( ("model", "expected"), ( @@ -416,39 +312,34 @@ PARITY_MODELS: Final[tuple[tuple[str, bridge.RustTokenizer], ...]] = ( @pytest.mark.parametrize(("model", "tokenizer"), PARITY_MODELS) @pytest.mark.parametrize("request_body", PARITY_REQUESTS) async def test_native_count_matches_python_budget_counter( - monkeypatch: pytest.MonkeyPatch, request_body: dict[str, object], model: str, tokenizer: bridge.RustTokenizer + request_body: dict[str, object], model: str, tokenizer: bridge.RustTokenizer ) -> None: native: Final = pytest.importorskip("litellm.rust_bridge._native") - monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) - litellm.rust(True) body: Final = json.dumps(request_body).replace(MODEL, model) + parsed: Final = json.loads(body) - request_body_parsed: Final = json.loads(body) - counts: Final = await count_input_tokens(request_body=request_body_parsed, raw_body=body.encode(), models=(model,)) - python_count: Final = count_input_tokens_for_model(request_body=request_body_parsed, model=model) + counted: Final = await bridge.native_count(native.TokenCounter, tokenizer, body.encode()) - assert counts[model] == python_count + assert counted.input_tokens == count_input_tokens_for_model(request_body=parsed, model=model) @pytest.mark.asyncio @pytest.mark.parametrize(("model", "tokenizer"), ((CL100K_MODEL, "cl100k_base"), (O200K_MODEL, "o200k_base"))) async def test_tiktoken_counts_long_text_exactly_where_python_chunks( - monkeypatch: pytest.MonkeyPatch, model: str, tokenizer: bridge.RustTokenizer + model: str, tokenizer: bridge.RustTokenizer ) -> None: """Python encodes tiktoken text in fixed-size chunks (drift of up to one token per chunk boundary); Rust does not.""" native: Final = pytest.importorskip("litellm.rust_bridge._native") - monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) - litellm.rust(True) text: Final = "x " * 20_000 body: Final = {"model": model, "messages": [{"role": "user", "content": text}]} encoding: Final = Tokenizer.from_tiktoken(tokenizer) exact: Final = 3 + encoding.count("user") + encoding.count(text) + 3 chunks: Final = -(-len(text) // TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS) - counts: Final = await count_input_tokens(request_body=body, raw_body=json.dumps(body).encode(), models=(model,)) + counted: Final = await bridge.native_count(native.TokenCounter, tokenizer, json.dumps(body).encode()) python_count: Final = count_input_tokens_for_model(request_body=body, model=model) - assert counts[model] == exact + assert counted.input_tokens == exact assert python_count is not None assert exact < python_count <= exact + chunks @@ -470,14 +361,10 @@ DECLINED_REQUESTS: Final[tuple[dict[str, object], ...]] = ( @pytest.mark.parametrize("tokenizer", TOKENIZERS) @pytest.mark.parametrize("request_body", DECLINED_REQUESTS) async def test_native_declines_shapes_python_prices_differently( - monkeypatch: pytest.MonkeyPatch, request_body: dict[str, object], tokenizer: bridge.RustTokenizer + request_body: dict[str, object], tokenizer: bridge.RustTokenizer ) -> None: native: Final = pytest.importorskip("litellm.rust_bridge._native") - monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) - litellm.rust(True) - model: Final = MODEL_BY_TOKENIZER[tokenizer] - raw, parsed = _counted(request_body, model) + raw, _ = _counted(request_body, MODEL_BY_TOKENIZER[tokenizer]) - counts: Final = await count_input_tokens(request_body=parsed, raw_body=raw, models=(model,)) - - assert counts.get(model) == count_input_tokens_for_model(request_body=parsed, model=model) + with pytest.raises(native.RustBridgeDeclined): + await bridge.native_count(native.TokenCounter, tokenizer, raw) diff --git a/tests/test_litellm/rust_bridge/test_tokenizer.py b/tests/test_litellm/rust_bridge/test_tokenizer.py index 0de7ad50b1e..188aa81093f 100644 --- a/tests/test_litellm/rust_bridge/test_tokenizer.py +++ b/tests/test_litellm/rust_bridge/test_tokenizer.py @@ -1,134 +1,49 @@ -from collections.abc import Generator from typing import Final import pytest import tiktoken from tokenizers import Tokenizer -import litellm from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEncoding -from litellm.rust_bridge import configuration, tokenizer -from litellm.utils import _select_tokenizer +from litellm.rust_bridge import tokenizer +from litellm.utils import claude_json_str from tests.test_litellm.litellm_core_utils.test_decode_special_tokens import TOKENIZER_JSON - -@pytest.fixture(autouse=True) -def isolated_configuration(monkeypatch: pytest.MonkeyPatch) -> Generator[None]: - monkeypatch.delenv("LITELLM_RUST", raising=False) - configuration.reset_rust_configuration() - yield - tokenizer.TOKENIZER.reset() - configuration.reset_rust_configuration() +TEXTS: Final = ("hello <|endoftext|> world", "café 漢字 🙂", " def f():\n return 1\n", "hello again") -@pytest.mark.parametrize("environment", (None, "0", "1")) -@pytest.mark.parametrize("process", (None, False, True)) -def test_tokenizer_factories_follow_rollout( - monkeypatch: pytest.MonkeyPatch, environment: str | None, process: bool | None -) -> None: - configuration.rust(process) - if environment is not None: - monkeypatch.setenv("LITELLM_RUST", environment) - enabled: Final = environment == "1" if environment is not None else process is True - encoding: Final = tokenizer.get_encoding("cl100k_base") - custom: Final = litellm.create_tokenizer(TOKENIZER_JSON) +@pytest.mark.parametrize("name", ("cl100k_base", "o200k_base")) +@pytest.mark.parametrize("text", TEXTS) +def test_native_encoding_matches_tiktoken(name: str, text: str) -> None: + native: Final = tokenizer.native_encoding(name) + if native is None: + pytest.skip("native extension is not built") + encoding: Final = OpenAIEncoding.wrap(native) + reference: Final = tiktoken.get_encoding(name) + + ids: Final = encoding.encode(text, disallowed_special=()) + assert ids == reference.encode(text, disallowed_special=()) + assert encoding.decode(ids) == reference.decode(ids) + + +@pytest.mark.parametrize("text", TEXTS) +def test_native_anthropic_tokenizer_matches_python(text: str) -> None: + native: Final = tokenizer.native_anthropic() + if native is None: + pytest.skip("native extension is not built") + reference: Final = Tokenizer.from_str(claude_json_str) + + ids: Final = HuggingFaceTokenizer(native).encode(text).ids + assert ids == reference.encode(text).ids + assert HuggingFaceTokenizer(native).decode(ids) == reference.decode(ids) + + +def test_native_custom_tokenizer_matches_python() -> None: + factory: Final = tokenizer.TOKENIZER.load() + if factory is None: + pytest.skip("native extension is not built") + native: Final = HuggingFaceTokenizer(factory.from_json(TOKENIZER_JSON)) reference: Final = Tokenizer.from_str(TOKENIZER_JSON) - assert isinstance(encoding, OpenAIEncoding if enabled else tiktoken.Encoding) - assert isinstance(custom["tokenizer"], HuggingFaceTokenizer if enabled else Tokenizer) - assert encoding.encode("café 漢字 🙂") == tiktoken.get_encoding(encoding.name).encode("café 漢字 🙂") - assert litellm.encode(text="Hello World", custom_tokenizer=custom) == reference.encode("Hello World").ids - assert litellm.token_counter(text="Hello World", custom_tokenizer=custom) == len(reference.encode("Hello World")) - - -def test_missing_native_binding_keeps_python_tokenizer_api() -> None: - configuration.rust(True) - tokenizer.TOKENIZER.override(None) - encoding: Final = tokenizer.get_encoding("cl100k_base") - custom: Final = litellm.create_tokenizer(TOKENIZER_JSON)["tokenizer"] - - assert isinstance(encoding, tiktoken.Encoding) - assert isinstance(custom, Tokenizer) - custom.enable_padding(pad_id=0, pad_token="[UNK]") - assert [item.ids for item in custom.encode_batch(["Hello", "Hello World"])] == [[3, 1, 0], [3, 1, 2]] - - -def test_cached_selection_follows_backend_changes(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(litellm, "disable_hf_tokenizer_download", True) - configuration.rust(True) - native: Final = _select_tokenizer("dispatch-fixture")["tokenizer"] - configuration.rust(False) - python: Final = _select_tokenizer("dispatch-fixture")["tokenizer"] - - assert isinstance(native, OpenAIEncoding) - assert isinstance(python, tiktoken.Encoding) - assert native.encode("hello") == python.encode("hello") - - -def test_declined_native_factory_falls_back_before_tokenizing() -> None: - from litellm.rust_bridge._native import RustBridgeDeclined - - class UnavailableTokenizer: - @staticmethod - def from_json(json: str) -> None: - raise RustBridgeDeclined("huggingface feature is disabled") - - configuration.rust(True) - binding: Final = tokenizer._as_factory(UnavailableTokenizer) - tokenizer.TOKENIZER.override(binding) - custom: Final = litellm.create_tokenizer(TOKENIZER_JSON) - - assert isinstance(custom["tokenizer"], Tokenizer) - assert ( - litellm.decode(tokens=litellm.encode(text="Hello World", custom_tokenizer=custom), custom_tokenizer=custom) - == "Hello World" - ) - - -@pytest.mark.parametrize( - ("model", "text"), - ( - ("gpt-4o", "hello <|endoftext|> world"), - ("gpt-3.5-turbo", "café 漢字 🙂"), - ("text-davinci-003", " def f():\n return 1\n"), - ("tokenizer-parity-fixture", "hello again"), - ), -) -def test_public_token_api_is_identical_across_backends(monkeypatch: pytest.MonkeyPatch, model: str, text: str) -> None: - """`litellm.token_counter`, `encode` and `decode` return the same values whichever backend - the catalog picks; only the object types differ.""" - monkeypatch.setattr(litellm, "anthropic_models", {*litellm.anthropic_models, "tokenizer-parity-fixture"}) - messages: Final = [{"role": "user", "content": text}, {"role": "assistant", "content": "ok"}] - - def observe() -> tuple[int, int, list[int], str]: - ids: Final = litellm.encode(model=model, text=text) - return ( - litellm.token_counter(model=model, text=text), - litellm.token_counter(model=model, messages=messages), - ids, - litellm.decode(model=model, tokens=ids), - ) - - configuration.rust(False) - python: Final = observe() - configuration.rust(True) - rust: Final = observe() - - assert rust == python - - -def test_cached_huggingface_tokenizers_follow_backend_changes(monkeypatch: pytest.MonkeyPatch) -> None: - from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer as RustHuggingFaceTokenizer - from litellm.utils import _load_huggingface_tokenizer - - monkeypatch.setattr(litellm, "anthropic_models", {*litellm.anthropic_models, "tokenizer-cache-fixture"}) - _load_huggingface_tokenizer.cache_clear() - configuration.rust(True) - native: Final = _select_tokenizer("tokenizer-cache-fixture")["tokenizer"] - configuration.rust(False) - python: Final = _select_tokenizer("tokenizer-cache-fixture")["tokenizer"] - configuration.rust(True) - - assert isinstance(native, RustHuggingFaceTokenizer) - assert isinstance(python, Tokenizer) - assert _select_tokenizer("tokenizer-cache-fixture")["tokenizer"] is native + assert native.encode("Hello World").ids == reference.encode("Hello World").ids + assert native.decode(reference.encode("Hello World").ids) == reference.decode(reference.encode("Hello World").ids) From b6d4133e41df345bb3894b6beb1802863e611a0a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:35:27 -0700 Subject: [PATCH 20/48] fix(websearch_interception): keep intercepted searches under the parent request's session and trace (#41711) * fix(websearch_interception): propagate parent session/trace ids into intercepted searches Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(websearch_interception): let parent correlation win over configured search params and type test params Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): bill an intercepted web search under the parent request session Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): drop unrelated reformatting from the websearch session harness change Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci(e2e): keep the websearch interception session suite out of the stage-mirror gate The stage-mirror stack runs no websearch_interception callback or search tool, so the suite is deselected there and the changed-tests gate fails on a file that executed nothing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci(e2e): run the websearch interception session suite on the stage-mirror stack Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * docs(e2e): leave CONTRIBUTING.md untouched Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: shivam Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../websearch_interception/handler.py | 93 ++++++++++- tests/e2e/AGENTS.md | 3 +- .../coverage_registry/quota_management.yaml | 1 + tests/e2e/e2e_http.py | 1 + tests/e2e/gateway/stage_mirror_ci_config.yml | 11 +- tests/e2e/models.py | 10 ++ tests/e2e/proxy_client.py | 39 ++++- ...test_websearch_interception_session_e2e.py | 102 ++++++++++++ .../test_websearch_interception_handler.py | 156 ++++++++++++++++++ 9 files changed, 406 insertions(+), 10 deletions(-) create mode 100644 tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 6a4c67c7db1..54578323fa4 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -10,6 +10,8 @@ import asyncio import math import uuid from collections.abc import AsyncIterator, Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, cast from typing_extensions import Never, ReadOnly @@ -196,6 +198,46 @@ class _AcompletionNamedParams(TypedDict, total=False): _NO_ACREATE_NAMED: Final[_AcreateNamedParams] = {} _NO_ASEARCH_NAMED: Final[_AsearchNamedParams] = {} + + +def _as_str_mapping(value: object) -> Mapping[str, object] | None: + return value if isinstance(value, Mapping) else None # pyright: ignore[reportUnknownVariableType] # str-keyed request metadata is not narrowable from object + + +@dataclass(frozen=True, slots=True) +class _ParentRequestCorrelation: + """Correlation ids of the LLM request that triggered an intercepted search, so the search's + own spend log row and traces land under the same session/trace instead of a fresh one.""" + + session_id: str | None + trace_id: str | None + parent_request_id: str | None + parent_otel_span: object | None + + def as_search_metadata(self) -> Mapping[str, object]: + return MappingProxyType( + { + key: value + for key, value in ( + ("session_id", self.session_id), + ("trace_id", self.trace_id), + ("parent_request_id", self.parent_request_id), + ("litellm_parent_otel_span", self.parent_otel_span), + ) + if value is not None + } + ) + + def as_search_kwargs(self) -> Mapping[str, str]: + return MappingProxyType( + { + key: value + for key, value in (("litellm_session_id", self.session_id), ("litellm_trace_id", self.trace_id)) + if value is not None + } + ) + + _NO_ACOMPLETION_NAMED: Final[_AcompletionNamedParams] = {} @@ -1527,15 +1569,17 @@ class WebSearchInterceptionLogger(CustomLogger): "WebSearchInterception: Executing search for '%s' using provider '%s'", query, search_provider ) user_api_key_auth: Final = self._get_user_api_key_auth_from_kwargs(kwargs) + parent_correlation: Final = self._get_parent_request_correlation(kwargs, user_api_key_auth) search_metadata: Final = ( None if user_api_key_auth is None else self._build_search_request_metadata( user_api_key_auth=user_api_key_auth, search_tool_name=search_tool_name, + parent_correlation=parent_correlation, ) ) - search_kwargs: Final = { + configured_search_kwargs: Final = { key: value for key, value in search_litellm_params.items() if key != "search_provider" and value is not None @@ -1549,8 +1593,11 @@ class WebSearchInterceptionLogger(CustomLogger): if rich_queries: query_arg = rich_queries rich_objective = rich.get("objective") - if rich_objective and "objective" not in search_kwargs: - search_kwargs["objective"] = rich_objective + if rich_objective and "objective" not in configured_search_kwargs: + configured_search_kwargs["objective"] = rich_objective + search_kwargs: Final = MappingProxyType( + {**configured_search_kwargs, **parent_correlation.as_search_kwargs()} + ) result: Final = ( await litellm.asearch( query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs @@ -1624,6 +1671,7 @@ class WebSearchInterceptionLogger(CustomLogger): def _build_search_request_metadata( user_api_key_auth: "UserAPIKeyAuth", search_tool_name: str | None, + parent_correlation: _ParentRequestCorrelation, ) -> Mapping[str, object]: """ Spend-tracking metadata for the intercepted search, so its provider cost is logged @@ -1637,11 +1685,50 @@ class WebSearchInterceptionLogger(CustomLogger): ) return { # mutable-ok: litellm's metadata channel is a plain dict its logging path reads and enriches **user_api_key_metadata, + **parent_correlation.as_search_metadata(), "model_group": search_tool_name, "user_api_key": user_api_key_auth.api_key, "user_api_key_auth": user_api_key_auth, } + @staticmethod + def _get_parent_request_correlation( + kwargs: Mapping[str, object] | None, + user_api_key_auth: "UserAPIKeyAuth | None", + ) -> _ParentRequestCorrelation: + """Read the originating request's ids from the hook kwargs, which are either the raw call + kwargs (metadata/litellm_metadata at top level) or a logging payload (under litellm_params).""" + if not kwargs: + return _ParentRequestCorrelation(None, None, None, None) + litellm_params: Final = _as_str_mapping(kwargs.get("litellm_params")) + scopes: Final[tuple[Mapping[str, object], ...]] = ( + (kwargs,) if litellm_params is None else (kwargs, litellm_params) + ) + metadatas: Final[tuple[Mapping[str, object], ...]] = tuple( + metadata + for scope in scopes + for metadata_key in ("metadata", "litellm_metadata") + if (metadata := _as_str_mapping(scope.get(metadata_key))) is not None + ) + + def first_str(scope_key: str | None, metadata_key: str | None) -> str | None: + candidates: Final[tuple[object, ...]] = ( + *(scope.get(scope_key) for scope in scopes if scope_key is not None), + *(metadata.get(metadata_key) for metadata in metadatas if metadata_key is not None), + ) + return next((value for value in candidates if isinstance(value, str) and value), None) + + parent_otel_span: Final[object | None] = next( + (span for metadata in metadatas if (span := metadata.get("litellm_parent_otel_span")) is not None), + None if user_api_key_auth is None else user_api_key_auth.parent_otel_span, + ) + return _ParentRequestCorrelation( + session_id=first_str("litellm_session_id", "session_id"), + trace_id=first_str("litellm_trace_id", "trace_id"), + parent_request_id=first_str("litellm_call_id", None), + parent_otel_span=parent_otel_span, + ) + @staticmethod def _selected_search_tool_name(search_tool: Mapping[str, object] | None) -> str | None: if search_tool is None: diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index b00b7dfac95..6ea75b806d0 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -210,6 +210,7 @@ quota_management... chat_completions | stream | messages_bridge | embeddings | cache_hit | key_rollup | concurrent_burst | tags | end_user | per_model | failure | spend_calculate | pagination | key_attribution + | websearch_interception assertion : blocks_over_limit | resets_after_window | headers_report_remaining | picks_under_tpm | blocks_then_resets | resets_windows_independently | alerts_without_blocking | isolates_per_model | isolates_per_member | isolates_per_group | enforced_across_keys @@ -217,7 +218,7 @@ quota_management... | matches_sum_of_logs | loses_no_spend | attributes_spend | writes_own_rows | writes_failure_row | returns_cost | keeps_total | joins_key | reports_alias_and_email | health_rows_keep_service_account | retrieve_batch_cost_joins_retrieving_key - | poller_batch_cost_joins_creating_key + | poller_batch_cost_joins_creating_key | bills_under_request_session e.g. quota_management.ratelimit.rpm.blocks_over_limit exercised_on=[chat_completions, messages] quota_management.budget.key.blocks_over_limit exercised_on=[chat_completions] ``` diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index ad0914d455b..f6b38243896 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -58,6 +58,7 @@ - {id: quota_management.spend_tracking.service_tier.bills_tier_rates, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier, assertions: [bills_tier_rates], exercised_on: [chat_completions], source: "cost_calculator.py", rationale: "A priority service_tier call bills input, output, and reasoning at the deployment's *_priority rates and records the tier on the row (#35923, #35925)"} - {id: quota_management.spend_tracking.cost_headers.additive_components, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_headers, assertions: [additive_components], exercised_on: [chat_completions], source: "proxy/common_request_processing.py", rationale: "The x-litellm-response-cost-* component headers sum to the total, input covers only fresh tokens, and reasoning stays a subset of output (#36965)"} - {id: quota_management.spend_tracking.passthrough_stream.injects_usage_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: passthrough_stream, assertions: [injects_usage_cost], exercised_on: [openai_passthrough], source: "proxy/pass_through_endpoints/streaming_handler.py", rationale: "With include_cost_in_streaming_usage on, the /openai passthrough's final streaming usage frame carries the proxy-computed cost (#36503). Uncovered: the flag is only settable in litellm_settings, and the shared e2e stack does not turn it on yet"} +- {id: quota_management.spend_tracking.websearch_interception.bills_under_request_session, module: quota_management, tier: P1, behavior: spend_tracking, variant: websearch_interception, assertions: [bills_under_request_session], exercised_on: [messages], source: "integrations/websearch_interception/handler.py", fail_before_fix: proven, rationale: "A web_search server tool the proxy intercepts into litellm.asearch writes its own asearch spend row, and that row carries the parent request's session_id so the session view counts the search and its cost next to the turn that triggered it (LIT-8063)"} - {id: quota_management.spend_tracking.key_attribution.joins_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [joins_key], exercised_on: [chat_completions, messages, responses, embeddings, batches, files, google_native, rust_control_plane], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Every spend row a virtual key writes across chat, queued chat, messages, responses, embeddings, the Gemini passthrough, file upload, batch create, and a replayed callback log carries api_key equal to the key's token hash and the key alias, the join the usage APIs depend on; a re-hashed token shows up as an unattributed key-hash-* row (#39568, #39572)"} - {id: quota_management.spend_tracking.key_attribution.reports_alias_and_email, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [reports_alias_and_email], exercised_on: [chat_completions, messages, responses, embeddings, batches, files, google_native, rust_control_plane], source: "proxy/management_endpoints/internal_user_endpoints.py", rationale: "/spend/logs?api_key= returns every one of the key's rows with its alias and /user/daily/activity aggregates them under the key's token with key_alias and user_email; /spend/logs carries no email field, so the email is asserted on daily activity only"} - {id: quota_management.spend_tracking.key_attribution.health_rows_keep_service_account, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [health_rows_keep_service_account], exercised_on: [chat_completions], source: "proxy/health_check.py", rationale: "A /health probe's spend row stays keyed by the literal litellm-internal-health-check service account rather than a hash of it, so health spend never appears as an unattributed key"} diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index 97f1e1671f8..1b62a1dbd8c 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -49,6 +49,7 @@ class AnthropicHeaders(AuthHeaders): on its own internal calls.""" anthropic_version: str = Field(default="2023-06-01", alias="anthropic-version") + x_litellm_session_id: str | None = Field(default=None, serialization_alias="x-litellm-session-id") class PartialBody(BaseModel): diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml index 2d02fedddae..03167692727 100644 --- a/tests/e2e/gateway/stage_mirror_ci_config.yml +++ b/tests/e2e/gateway/stage_mirror_ci_config.yml @@ -32,7 +32,10 @@ litellm_settings: - host: 127.0.0.1 port: 6379 ssl: true - callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel"] + callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel", "websearch_interception"] + websearch_interception_params: + enabled_providers: ["bedrock"] + search_tool_name: e2e-search require_auth_for_metrics_endpoint: false router_settings: @@ -65,6 +68,12 @@ model_list: model: openai/text-embedding-3-small api_key: os.environ/OPENAI_API_KEY +search_tools: + - search_tool_name: e2e-search + litellm_params: + search_provider: perplexity + api_key: os.environ/PERPLEXITY_API_KEY + files_settings: - custom_llm_provider: openai api_key: os.environ/OPENAI_API_KEY diff --git a/tests/e2e/models.py b/tests/e2e/models.py index f9c17314477..3309479edaf 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -949,6 +949,7 @@ class SpendLogRow(BaseModel): completion_tokens: int | None = None total_tokens: int | None = None request_tags: list[str] | None = None + session_id: str | None = None metadata: SpendLogMetadata | None = None proxy_server_request: JsonValue = None response: JsonValue = None @@ -985,6 +986,15 @@ class SpendLogsPageParams(BaseModel): api_key: str | None = None +class SessionSpendLogsParams(BaseModel): + """Query for /spend/logs/session/ui, the session view the Admin UI logs page + opens: every row whose session_id equals the given one, newest first.""" + + session_id: str + page: int = 1 + page_size: int = 100 + + class SpendLogsPage(BaseModel): data: list[SpendLogRow] = [] total: int diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index f1981e0aa5e..66df1c4c106 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -87,6 +87,7 @@ from models import ( RouterSettingsResponse, SearchToolCreateBody, SearchToolCreateResponse, + SessionSpendLogsParams, SpendLogRow, SpendLogs, SpendLogsPage, @@ -1004,19 +1005,26 @@ class ProxyClient: response_type=CountTokensResponse, ) - def messages(self, key: str, body: AnthropicMessagesBody) -> Result[AnthropicMessagesResponse]: + def messages( + self, key: str, body: AnthropicMessagesBody, *, session_id: str | None = None + ) -> Result[AnthropicMessagesResponse]: """POST /v1/messages (Anthropic-native). The response is either the Anthropic-shape passthrough (`content`) or the OpenAI-normalized shape - (`choices`); AnthropicMessagesResponse models both.""" + (`choices`); AnthropicMessagesResponse models both. `session_id` goes out + as the `x-litellm-session-id` header, the way Claude Code sends it through + ANTHROPIC_CUSTOM_HEADERS, so every spend row the call produces shares it.""" return self.transport.post( "/v1/messages", - headers=self._anthropic_headers(key), + headers=self._anthropic_headers(key, session_id=session_id), json=body, response_type=AnthropicMessagesResponse, ) - def _anthropic_headers(self, key: str) -> AnthropicHeaders: - return AnthropicHeaders(authorization=self.transport.bearer(key).authorization) + def _anthropic_headers(self, key: str, *, session_id: str | None = None) -> AnthropicHeaders: + return AnthropicHeaders( + authorization=self.transport.bearer(key).authorization, + x_litellm_session_id=session_id, + ) # ---- spend read-back ------------------------------------------------ @@ -1060,6 +1068,27 @@ class ProxyClient: ) -> list[SpendLogRow]: return self._poll(lambda: self.spend_logs(SpendLogsParams(api_key=key)), min_rows, predicate) + def session_spend_logs(self, session_id: str) -> list[SpendLogRow]: + """GET /spend/logs/session/ui, the per-session view the Admin UI logs page + opens when a session id is clicked.""" + return unwrap( + self.transport.get( + "/spend/logs/session/ui", + headers=self.management_headers(), + params=SessionSpendLogsParams(session_id=session_id), + response_type=SpendLogsPage, + ) + ).data + + def poll_logs_for_session( + self, + session_id: str, + *, + min_rows: int = 1, + predicate: RowsPredicate | None = None, + ) -> list[SpendLogRow]: + return self._poll(lambda: self.session_spend_logs(session_id), min_rows, predicate) + def poll_logs_for_request_id( self, request_id: str, diff --git a/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py b/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py new file mode 100644 index 00000000000..89d0beec414 --- /dev/null +++ b/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py @@ -0,0 +1,102 @@ +"""Intercepted web searches are billed under the LLM request's session. + +The websearch_interception callback turns an Anthropic ``web_search`` server tool +into a ``litellm.asearch`` call against a configured search tool, so each search is +its own spend row (call_type ``asearch``) next to the ``anthropic_messages`` row for +the turn that asked for it. Claude Code and the Admin UI group spend by +``session_id``, so the search row has to carry the same session as the turn that +triggered it; before the fix it landed under a session of its own and the session +view under-counted both requests and spend (LIT-8063). + +Needs a proxy booted with the callback and a real search backend, which +``gateway/stage_mirror_ci_config.yml`` carries as the ``e2e-search`` Perplexity tool. +""" + +from typing import Final + +import pytest +from e2e_config import unique_marker +from e2e_http import unwrap +from lifecycle import ResourceManager +from models import ( + AnthropicMessagesBody, + AnthropicWebSearchTool, + ChatMessage, + LiteLLMParamsBody, + SpendLogRow, +) +from proxy_client import ProxyClient + +pytestmark = pytest.mark.e2e + +BEDROCK_INVOKE_BACKEND: Final = "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0" +SEARCH_CALL_TYPE: Final = "asearch" + + +def _has_search_row(rows: list[SpendLogRow]) -> bool: + return any(row.call_type == SEARCH_CALL_TYPE for row in rows) + + +class TestWebSearchInterceptionSession: + @pytest.mark.covers( + "quota_management.spend_tracking.websearch_interception.bills_under_request_session", + exercised_on=("messages",), + ) + def test_intercepted_search_is_billed_under_the_request_session( + self, proxy: ProxyClient, resources: ResourceManager + ) -> None: + """One /v1/messages turn that runs an intercepted web search must produce an + ``asearch`` spend row in the same session as its ``anthropic_messages`` row, + billed separately and with its own request id.""" + marker: Final = unique_marker() + model: Final = f"e2e-websearch-session-{marker}" + model_id: Final = proxy.create_model( + model, LiteLLMParamsBody(model=BEDROCK_INVOKE_BACKEND, aws_region_name="us-east-1") + ) + resources.defer(lambda: proxy.delete_model(model_id)) + key: Final = resources.key(models=[model]) + session_id: Final = f"e2e-websearch-session-{marker}" + + response: Final = unwrap( + proxy.messages( + key, + AnthropicMessagesBody( + model=model, + max_tokens=512, + tools=[AnthropicWebSearchTool(type="web_search_20250305", name="web_search", max_uses=1)], + messages=[ + ChatMessage( + role="user", + content=f"Use web search to find one recent news headline about Anthropic ({marker}).", + ) + ], + ), + session_id=session_id, + ) + ) + block_types: Final = tuple(block.type for block in response.content or ()) + assert "web_search_tool_result" in block_types, ( + f"precondition: the turn never ran an intercepted search, so there is no search row to attribute. " + f"blocks={block_types}" + ) + + rows: Final = proxy.poll_logs_for_session(session_id, min_rows=2, predicate=_has_search_row) + by_call_type: Final = {row.call_type or "" for row in rows} + assert SEARCH_CALL_TYPE in by_call_type, ( + f"session {session_id} has no {SEARCH_CALL_TYPE} row, so the intercepted search was billed under a " + f"different session and the session view misses its cost. call_types={sorted(by_call_type)} " + f"rows={[(row.call_type, row.request_id, row.spend) for row in rows]}" + ) + search_rows: Final = tuple(row for row in rows if row.call_type == SEARCH_CALL_TYPE) + turn_rows: Final = tuple(row for row in rows if row.call_type != SEARCH_CALL_TYPE) + assert turn_rows, f"session {session_id} carries only search rows: {rows!r}" + assert all(row.session_id == session_id for row in rows), ( + f"session view returned rows outside {session_id}: {[row.session_id for row in rows]}" + ) + assert all((row.spend or 0.0) > 0 for row in search_rows), ( + f"an intercepted search must stay a separately billed row: {[row.spend for row in search_rows]}" + ) + assert {row.request_id for row in search_rows}.isdisjoint({row.request_id for row in turn_rows}), ( + "a search row reused its parent turn's request_id instead of keeping its own: " + f"{[(row.call_type, row.request_id) for row in rows]}" + ) diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py index f39f41a6d12..d6450d0f1de 100644 --- a/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py @@ -287,6 +287,162 @@ async def test_execute_search_attributes_spend_to_the_calling_key(monkeypatch): ) +def _perplexity_router() -> MagicMock: + router = MagicMock() + router.search_tools = [ + { + "search_tool_name": "perplexity-sonar-pro", + "litellm_params": {"search_provider": "perplexity", "api_key": "fake-key"}, + } + ] + return router + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "parent_kwargs", + [ + pytest.param( + { + "litellm_call_id": "parent-call-1", + "litellm_trace_id": "trace-abc", + "litellm_session_id": "session-abc", + "metadata": { + "user_api_key_auth": UserAPIKeyAuth(api_key="hashed-sk-1234"), + "session_id": "session-abc", + }, + }, + id="chat-completions-call-kwargs", + ), + pytest.param( + { + "litellm_call_id": "parent-call-1", + "litellm_metadata": { + "user_api_key_auth": UserAPIKeyAuth(api_key="hashed-sk-1234"), + "session_id": "session-abc", + "trace_id": "trace-abc", + }, + }, + id="anthropic-messages-litellm-metadata", + ), + pytest.param( + { + "litellm_params": { + "litellm_call_id": "parent-call-1", + "litellm_trace_id": "trace-abc", + "metadata": {"user_api_key_auth": UserAPIKeyAuth(api_key="hashed-sk-1234")}, + "litellm_metadata": {"session_id": "session-abc"}, + } + }, + id="logging-payload-with-both-metadata-keys", + ), + ], +) +async def test_execute_search_inherits_parent_request_session_and_trace( + monkeypatch: pytest.MonkeyPatch, parent_kwargs: dict[str, object] +): + """The intercepted asearch is billed as its own call but must land in the parent request's + session and trace, otherwise every search shows up as a separate one-call session in SpendLogs.""" + import litellm + from litellm.proxy import proxy_server + from litellm.proxy.spend_tracking.spend_tracking_utils import _get_session_id_for_spend_log + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="perplexity-sonar-pro") + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + monkeypatch.setattr(proxy_server, "llm_router", _perplexity_router()) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + + await logger._execute_search("what is litellm", kwargs=parent_kwargs) + + forwarded = mock_asearch.await_args.kwargs + assert forwarded["litellm_session_id"] == "session-abc" + assert forwarded["litellm_trace_id"] == "trace-abc" + assert forwarded["litellm_metadata"]["session_id"] == "session-abc" + assert forwarded["litellm_metadata"]["trace_id"] == "trace-abc" + assert forwarded["litellm_metadata"]["parent_request_id"] == "parent-call-1" + assert forwarded["litellm_metadata"]["user_api_key"] == "hashed-sk-1234" + assert forwarded["litellm_metadata"]["model_group"] == "perplexity-sonar-pro" + assert "litellm_call_id" not in forwarded + assert ( + _get_session_id_for_spend_log( + kwargs={"litellm_trace_id": forwarded["litellm_trace_id"]}, + metadata=forwarded["litellm_metadata"], + standard_logging_payload=None, + omit_when_missing=True, + ) + == "session-abc" + ) + + +@pytest.mark.asyncio +async def test_execute_search_forwards_parent_otel_span_from_key_auth(monkeypatch: pytest.MonkeyPatch): + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="perplexity-sonar-pro") + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + monkeypatch.setattr(proxy_server, "llm_router", _perplexity_router()) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + parent_span = object() + + await logger._execute_search( + "what is litellm", + kwargs={"metadata": {"user_api_key_auth": UserAPIKeyAuth(api_key="sk", parent_otel_span=parent_span)}}, + ) + + assert mock_asearch.await_args.kwargs["litellm_metadata"]["litellm_parent_otel_span"] is parent_span + + +@pytest.mark.asyncio +async def test_execute_search_without_parent_session_does_not_invent_one(monkeypatch: pytest.MonkeyPatch): + """A parent request with no session/trace must not stamp empty correlation keys on the search.""" + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="perplexity-sonar-pro") + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + monkeypatch.setattr(proxy_server, "llm_router", _perplexity_router()) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + + await logger._execute_search( + "what is litellm", + kwargs={"litellm_params": {"metadata": {"user_api_key_auth": UserAPIKeyAuth(api_key="sk"), "prompt": "x"}}}, + ) + + forwarded = mock_asearch.await_args.kwargs + assert "litellm_session_id" not in forwarded + assert "litellm_trace_id" not in forwarded + assert not {"session_id", "trace_id", "parent_request_id", "prompt"} & forwarded["litellm_metadata"].keys() + + +@pytest.mark.asyncio +async def test_concurrent_searches_keep_their_own_parent_session(monkeypatch: pytest.MonkeyPatch): + import asyncio + + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="perplexity-sonar-pro") + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + monkeypatch.setattr(proxy_server, "llm_router", _perplexity_router()) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + + await asyncio.gather( + *( + logger._execute_search( + f"query {i}", + kwargs={"metadata": {"user_api_key_auth": UserAPIKeyAuth(api_key="sk"), "session_id": f"session-{i}"}}, + ) + for i in range(5) + ) + ) + + seen = { + call.kwargs["query"]: call.kwargs["litellm_metadata"]["session_id"] for call in mock_asearch.await_args_list + } + assert seen == {f"query {i}": f"session-{i}" for i in range(5)} + + @pytest.mark.asyncio async def test_execute_search_without_proxy_auth_context_stays_sdk_only(monkeypatch): """SDK callers have no key to attribute the search to, so no proxy metadata is invented.""" From 75163278988aa0b6fc073767d5652426327f4480 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:42:41 -0700 Subject: [PATCH 21/48] test(e2e): assert a cooldown reaches a sibling replica within the 1s Redis read interval (#42422) * test(e2e): assert a cooldown reaches a sibling replica within the 1s Redis read interval * test(e2e): skip the sibling replica cooldown cell when one gateway URL is named * test(e2e): collapse repeated gateway addresses so the sibling cooldown cell skips instead of erroring --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/e2e/coverage_registry/reliability.yaml | 1 + tests/e2e/e2e_config.py | 2 +- tests/e2e/router/reliability_support.py | 32 ++++- .../router/test_reliability_cooldowns_e2e.py | 112 +++++++++++++++--- tests/e2e/test_proxy_client.py | 4 + 5 files changed, 133 insertions(+), 18 deletions(-) diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 334780eda53..9f88f2478c0 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -9,6 +9,7 @@ - {id: reliability.retry.auth.succeeds_within_retries, module: reliability, tier: P1, behavior: retry, variant: auth, assertions: [succeeds_within_retries], exercised_on: [chat_completions], source: "get_retry_from_policy.py:42", rationale: "Transient auth glitch retry"} - {id: reliability.retry.context_window.succeeds_within_retries, module: reliability, tier: P1, behavior: retry, variant: context_window, assertions: [succeeds_within_retries], exercised_on: [chat_completions], source: "get_retry_from_policy.py:51", fail_before_fix: proven, rationale: "A context-window 400 under BadRequestErrorRetries retries onto a sibling deployment in the same model group, instead of coming straight back as the 400 the deployment that just refused it returned"} - {id: reliability.cooldown.5xx.trips_then_recovers, module: reliability, tier: P0, behavior: cooldown, variant: "5xx", assertions: [trips_then_recovers], exercised_on: [chat_completions], source: "cooldown_handlers.py:40", rationale: "Deployment cools after repeated 5xx, recovers after cooldown_time"} +- {id: reliability.cooldown.sibling_replica.serves_backup_within_read_interval, module: reliability, tier: P1, behavior: cooldown, variant: sibling_replica, assertions: [serves_backup_within_read_interval], exercised_on: [chat_completions], source: "cooldown_cache.py:44", fail_before_fix: proven, rationale: "A bench taken on one gateway reaches a sibling that already holds the key's read timer within the 1s Redis read interval plus margin, so its next call lands on the backup"} - {id: reliability.cooldown.429.trips_then_recovers, module: reliability, tier: P0, behavior: cooldown, variant: "429", assertions: [trips_then_recovers], exercised_on: [chat_completions], source: "cooldown_handlers.py:69", rationale: "Cools on 429, avoids hammering exhausted provider"} - {id: reliability.cooldown.auth.trips_then_recovers, module: reliability, tier: P1, behavior: cooldown, variant: auth, assertions: [trips_then_recovers], exercised_on: [chat_completions], source: "cooldown_handlers.py:74", rationale: "Cools on 401 auth error"} - {id: reliability.cooldown.timeout.trips_then_recovers, module: reliability, tier: P1, behavior: cooldown, variant: timeout, assertions: [trips_then_recovers], exercised_on: [chat_completions], source: "cooldown_handlers.py:77", rationale: "Cools on 408 timeout"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 311c944eeb4..71f01834788 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -34,7 +34,7 @@ CONTROL_PLANE_BASE_URL = os.environ.get( def parse_replica_urls(raw: str, fallback: str) -> tuple[str, ...]: - urls: Final = tuple(url.strip().rstrip("/") for url in raw.split(",") if url.strip()) + urls: Final = tuple(dict.fromkeys(url.strip().rstrip("/") for url in raw.split(",") if url.strip())) return urls or (fallback,) diff --git a/tests/e2e/router/reliability_support.py b/tests/e2e/router/reliability_support.py index 3d5b76f6408..4388e7f11bc 100644 --- a/tests/e2e/router/reliability_support.py +++ b/tests/e2e/router/reliability_support.py @@ -22,6 +22,7 @@ from pydantic import ValidationError from proxy_client import ProxyClient from e2e_config import CHEAP_OPENAI_MODEL, PROXY_BASE_URL, unique_marker from e2e_http import NetworkError, StreamHead, StreamingResponse +from transport import Transport from models import ( CacheControl, ChatMessage, @@ -274,9 +275,36 @@ def chat_turns_override( ) -> StreamingResponse: """POST /chat/completions with an optional per-request router_settings_override, returning the raw outcome so tests read status, body, and reliability headers.""" - return proxy.transport.send( + return chat_turns_override_via( + proxy.transport, key, model, turns, override=override, stream=stream, cache=cache, max_tokens=max_tokens + ) + + +def chat_override_via( + transport: Transport, + key: str, + model: str, + content: str, + override: RouterSettingsOverride | None = None, +) -> StreamingResponse: + """`chat_override` aimed at one replica's transport (from `proxy.replicas`) instead of + the client's default, for cells that must know which gateway took the call.""" + return chat_turns_override_via(transport, key, model, [ChatMessage(role="user", content=content)], override=override) + + +def chat_turns_override_via( + transport: Transport, + key: str, + model: str, + turns: Sequence[ChatMessage], + override: RouterSettingsOverride | None = None, + stream: bool = False, + cache: dict[str, bool] | None = {"no-cache": True}, + max_tokens: int = 512, +) -> StreamingResponse: + return transport.send( "/chat/completions", - headers=proxy.transport.bearer(key), + headers=transport.bearer(key), json=ReliabilityChatBody( model=model, messages=turns, diff --git a/tests/e2e/router/test_reliability_cooldowns_e2e.py b/tests/e2e/router/test_reliability_cooldowns_e2e.py index 769971e1533..ce3bce32880 100644 --- a/tests/e2e/router/test_reliability_cooldowns_e2e.py +++ b/tests/e2e/router/test_reliability_cooldowns_e2e.py @@ -6,21 +6,33 @@ way (a 500, a 429, a 401, or a timeout) holding all of the group's shuffle weigh with an `allowed_fails_policy` of zero for that error class and a short `cooldown_time`, plus a healthy backup at weight 0. The first call, retries off, surfaces the failure to the customer as-is and benches the deployment. The proxy -records the bench off the request path, and a sibling replica only sees it on -its next read of the cooldown keys from Redis, which the cooldown cache does at -most every 1s (DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS). So for +writes the bench to Redis before it answers that failure, and a sibling replica +only sees it on its next read of the cooldown keys from Redis, which the +cooldown cache does at most once per COOLDOWN_REDIS_READ_INTERVAL_SECONDS per +key (DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS, 1s). So for REPLICA_PROPAGATION_SECONDS after the trip, a window kept far wider than that -so this cell asserts the trip and the recovery rather than how fast siblings -catch up, every answer has to be either the deployment's own failure or a 200 -from the backup, which the proxy names in x-litellm-model-id, and at least one -replica has to have served from the backup by then. From then until shortly -before the cooldown can lapse, every call has to land on the backup whichever -replica takes it. Then the test polls until the weighted shuffle opens on the -failing deployment again and the same failure comes back (or, for the 429 pair, -its own 200 once the key's rpm window has reset): that is the recovery, since a -benched deployment is one the router will try again, not one it forgot. Its -deadline counts from the last failure a stale replica caused, because every -failure re-arms the cooldown. +so the trip-then-recover cells assert the trip and the recovery rather than how +fast siblings catch up, every answer has to be either the deployment's own +failure or a 200 from the backup, which the proxy names in x-litellm-model-id, +and at least one replica has to have served from the backup by then. From then +until shortly before the cooldown can lapse, every call has to land on the +backup whichever replica takes it. Then the test polls until the weighted +shuffle opens on the failing deployment again and the same failure comes back +(or, for the 429 pair, its own 200 once the key's rpm window has reset): that +is the recovery, since a benched deployment is one the router will try again, +not one it forgot. Its deadline counts from the last failure a stale replica +caused, because every failure re-arms the cooldown. + +The sibling cell is the one that asserts the speed. It addresses two gateways +from PROXY_REPLICA_URLS by name, warms the second with a healthy call so its +router has already read the failing deployment's cooldown key from Redis and +started the read interval on it, trips the deployment through the first, waits +the interval plus a margin, and then sends the second replica exactly one call, +which has to come back from the backup. One call, because a poll that reached +the failing deployment through the second replica would bench it there too and +hide whether the first replica's bench ever travelled. A stack addressed only +through its load balancer cannot pin which replica takes a call, so the cell is +skipped at collection unless LITELLM_PROXY_REPLICA_URLS names at least two. The failures are the same real ones the retry tests use: a 1ms deadline and a bogus key on the real backend, and this proxy standing in as the upstream for @@ -37,7 +49,7 @@ from dataclasses import dataclass import pytest from complexity_router_client import ComplexityRouterClient -from e2e_config import CHEAP_OPENAI_MODEL, unique_marker +from e2e_config import CHEAP_OPENAI_MODEL, PROXY_REPLICA_URLS, unique_marker from e2e_http import StreamingResponse from lifecycle import ResourceManager from models import KeyGenerateBody, RouterSettingsOverride @@ -45,6 +57,7 @@ from reliability_support import ( COOLDOWN_SECONDS, REPLICA_PROPAGATION_SECONDS, chat_override, + chat_override_via, create_always_5xx_deployment, create_always_rate_limited_deployment, create_always_timing_out_deployment, @@ -54,12 +67,15 @@ from reliability_support import ( model_id_of, spend_only_request_of, ) +from transport import Transport pytestmark = pytest.mark.e2e RECOVERY_GRACE_SECONDS = 10 PROPAGATION_POLL_SECONDS = 0.25 BENCH_MARGIN_SECONDS = 4.0 +COOLDOWN_REDIS_READ_INTERVAL_SECONDS = 1.0 +SIBLING_READ_MARGIN_SECONDS = 1.0 def _call_without_retries(client: ComplexityRouterClient, key: str, group: str) -> StreamingResponse: @@ -68,6 +84,31 @@ def _call_without_retries(client: ComplexityRouterClient, key: str, group: str) ) +def _call_replica_without_retries(transport: Transport, key: str, group: str) -> StreamingResponse: + return chat_override_via( + transport, key, group, f"say hi {unique_marker()}", override=RouterSettingsOverride(num_retries=0) + ) + + +@dataclass(frozen=True, slots=True) +class _Replica: + url: str + transport: Transport + + +def _two_replicas(client: ComplexityRouterClient) -> tuple[_Replica, _Replica]: + first, second, *_ = (_Replica(url, transport) for url, transport in client.proxy.replicas.items()) + return first, second + + +def _warm_cooldown_reads(replica: _Replica, key: str) -> None: + warmed = chat_override_via(replica.transport, key, CHEAP_OPENAI_MODEL, f"say hi {unique_marker()}") + assert warmed.status_code == 200, ( + f"{replica.url} should have answered a healthy {CHEAP_OPENAI_MODEL} call before the trip, got " + f"{warmed.status_code}: {warmed.body[:300]}" + ) + + def _assert_served_by_backup(resp: StreamingResponse, backup: str, when: str) -> None: assert resp.status_code == 200, ( f"{when} the group should have served from the backup, got {resp.status_code}: {resp.body[:300]}" @@ -176,6 +217,47 @@ class TestReliabilityCooldowns: _assert_trips_then_recovers(client, scoped_key, group, failing, backup, failure_status=500) + @pytest.mark.covers("reliability.cooldown.sibling_replica.serves_backup_within_read_interval") + @pytest.mark.skipif( + len(PROXY_REPLICA_URLS) < 2, + reason=( + "this cell trips a deployment through one gateway and reads the bench from another, so " + f"LITELLM_PROXY_REPLICA_URLS has to name at least two distinct gateways, got {PROXY_REPLICA_URLS}" + ), + ) + def test_sibling_replica_serves_backup_within_redis_read_interval( + self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str + ) -> None: + tripping, sibling = _two_replicas(client) + + upstream = f"reliability-cooldown-sibling-upstream-{unique_marker()}" + upstream_id = create_bad_base_deployment(client.proxy, upstream) + resources.defer(lambda: client.proxy.delete_model(upstream_id)) + + group = f"reliability-cooldown-sibling-{unique_marker()}" + failing = create_always_5xx_deployment( + client.proxy, group, upstream, scoped_key, cooldown_time=COOLDOWN_SECONDS + ) + resources.defer(lambda: client.proxy.delete_model(failing)) + backup = create_zero_weight_backup_deployment(client.proxy, group) + resources.defer(lambda: client.proxy.delete_model(backup)) + + _warm_cooldown_reads(sibling, scoped_key) + + tripped = _call_replica_without_retries(tripping.transport, scoped_key, group) + assert tripped.status_code == 500, ( + f"the first call through {tripping.url} should have surfaced the deployment's own 500, got " + f"{tripped.status_code}: {tripped.body[:300]}" + ) + tripped_at = time.monotonic() + + time.sleep(COOLDOWN_REDIS_READ_INTERVAL_SECONDS + SIBLING_READ_MARGIN_SECONDS) + _assert_served_by_backup( + _call_replica_without_retries(sibling.transport, scoped_key, group), + backup, + f"{time.monotonic() - tripped_at:.1f}s after {tripping.url} benched {failing}, on {sibling.url}", + ) + @pytest.mark.covers("reliability.cooldown.429.trips_then_recovers") def test_429_trips_cooldown_then_recovers( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str diff --git a/tests/e2e/test_proxy_client.py b/tests/e2e/test_proxy_client.py index 0c4aed5bd65..a8f07ed6dd7 100644 --- a/tests/e2e/test_proxy_client.py +++ b/tests/e2e/test_proxy_client.py @@ -338,6 +338,10 @@ class TestParseReplicaUrls: def test_falls_back_to_the_data_plane_address_when_unset(self) -> None: assert parse_replica_urls("", "http://lb") == ("http://lb",) + def test_collapses_repeated_gateway_addresses_to_one_replica(self) -> None: + raw: Final = "http://127.0.0.1:4010,http://127.0.0.1:4010/,http://127.0.0.1:4011,http://127.0.0.1:4010" + assert parse_replica_urls(raw, "http://lb") == ("http://127.0.0.1:4010", "http://127.0.0.1:4011") + def _answers(answers: Iterable[str]) -> ReplicaRead[str]: it: Final = iter(answers) From cb2f22533c1f778322b09578269238028f2c3862 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:43:30 -0700 Subject: [PATCH 22/48] feat(proxy): opt-in include_guardrail_response returns guardrail_information in the response (#42327) * feat(proxy): opt-in include_guardrail_response returns guardrail_information in the response Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): read include_guardrail_response from the request metadata bucket the router did not reseed Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(proxy): format common request processing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): redact matched content in guardrail_information and stop mutating cached responses Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): traverse guardrail diagnostics iteratively Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): annotate guardrail traversal cast Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): reuse core redaction helper for guardrail_information Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(proxy): justify response rebind when attaching guardrail information Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/core_helpers.py | 14 +- litellm/proxy/common_request_processing.py | 44 +++++- litellm/proxy/litellm_pre_call_utils.py | 8 ++ tests/e2e/coverage_registry/guardrail.yaml | 1 + tests/e2e/guardrails/guardrails_client.py | 2 + ...test_guardrail_information_response_e2e.py | 118 ++++++++++++++++ tests/e2e/models.py | 10 ++ .../litellm_core_utils/test_core_helpers.py | 22 +++ .../proxy/test_common_request_processing.py | 128 ++++++++++++++++++ .../proxy/test_litellm_pre_call_utils.py | 40 ++++++ 10 files changed, 379 insertions(+), 8 deletions(-) create mode 100644 tests/e2e/guardrails/test_guardrail_information_response_e2e.py diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 5bcde688521..d7fbe9f7e09 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -3,7 +3,7 @@ import copy import logging import re -from collections.abc import Iterable, Mapping +from collections.abc import Collection, Iterable, Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol @@ -709,10 +709,11 @@ def filter_internal_params(data: dict, additional_internal_params: set | None = def redact_nested_match_and_regex_keys( payload: dict | list[Any] | str | None, + keys: Collection[str] = ("match", "regex"), ) -> dict | list[Any] | str | None: """ - Deep-copy `payload` and replace every `match` / `regex` string field with - "[REDACTED]" anywhere in nested dict/list structures. + Deep-copy `payload` and replace every configured string field with "[REDACTED]" + anywhere in nested dict/list structures. Used for guardrail spend/compliance logging so raw spans are not persisted. """ @@ -734,10 +735,9 @@ def redact_nested_match_and_regex_keys( continue seen.add(node_id) if isinstance(node, dict): - if "match" in node: - node["match"] = "[REDACTED]" - if "regex" in node: - node["regex"] = "[REDACTED]" + for key in keys: + if key in node: + node[key] = "[REDACTED]" stack.extend(node.values()) elif isinstance(node, list): stack.extend(node) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 6a7b6970bb8..c40090233be 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -26,7 +26,7 @@ import httpx import orjson from fastapi import HTTPException, Request, status from fastapi.responses import JSONResponse, Response, StreamingResponse -from pydantic import TypeAdapter, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError from starlette.types import Receive, Scope, Send import litellm @@ -56,6 +56,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_or_create_metadata_bucket, independent_snapshot, is_expected_client_error, + redact_nested_match_and_regex_keys, ) from litellm.litellm_core_utils.dd_tracing import NullTracer, tracer from litellm.litellm_core_utils.get_supported_openai_params import ( @@ -1401,6 +1402,42 @@ def _override_openai_response_model( ) +_METADATA_BUCKET_KEYS: Final = ("metadata", "litellm_metadata") +_RESPONSE_REDACTED_KEYS: Final = ("keyword", "snippet", "match", "regex") + + +def _request_metadata_buckets(request_data: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + return tuple(bucket for key in _METADATA_BUCKET_KEYS if isinstance(bucket := request_data.get(key), Mapping)) + + +def include_guardrail_response_requested(request_data: Mapping[str, object]) -> bool: + return any(bucket.get("include_guardrail_response") is True for bucket in _request_metadata_buckets(request_data)) + + +def attach_guardrail_information(response: object, request_data: Mapping[str, object]) -> object: + recorded: Final[Sequence[object]] = next( + ( + entries + for bucket in _request_metadata_buckets(request_data) + if isinstance( + entries := bucket.get("standard_logging_guardrail_information"), + list, + ) + ), + (), + ) + guardrail_information: Final = [ # mutable-ok: response list contract + redact_nested_match_and_regex_keys(entry, keys=_RESPONSE_REDACTED_KEYS) + for entry in recorded + if isinstance(entry, dict) + ] + if isinstance(response, dict): + return response | MappingProxyType({"guardrail_information": guardrail_information}) + if isinstance(response, BaseModel) and response.model_config.get("extra") == "allow": + return response.model_copy(update=MappingProxyType({"guardrail_information": guardrail_information})) + return response + + class CostBreakdownHeaderValues(NamedTuple): original_cost: float | None = None discount_amount: float | None = None @@ -2898,6 +2935,11 @@ class ProxyBaseLLMRequestProcessing: if isinstance(response, dict): response.pop("_hidden_params", None) + if include_guardrail_response_requested(self.data): + response = attach_guardrail_information( # rebind-ok: response tail rebinds the copied response + response=response, request_data=self.data + ) + # Call response headers hook for non-streaming success callback_headers = await proxy_logging_obj.post_call_response_headers_hook( data=self.data, diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 44d45dcd687..4d5b91be034 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -3116,7 +3116,15 @@ async def move_guardrails_to_metadata( - If guardrails not set on API key, then checks request metadata - Adds guardrails from policies attached to key/team metadata - Adds guardrails from policy engine based on team/key/model context + - Moves include_guardrail_response into request metadata before provider dispatch """ + if "include_guardrail_response" in data: + data[_metadata_variable_name][ + "include_guardrail_response" + ] = ( # rebind-ok: pre-call hooks mutate the shared request dict in place + data.pop("include_guardrail_response") is True + ) + # Early-out: skip all guardrails processing when nothing is configured key_metadata: Final = user_api_key_dict.metadata team_metadata: Final = user_api_key_dict.team_metadata diff --git a/tests/e2e/coverage_registry/guardrail.yaml b/tests/e2e/coverage_registry/guardrail.yaml index 02b5a921add..f49568c883b 100644 --- a/tests/e2e/coverage_registry/guardrail.yaml +++ b/tests/e2e/coverage_registry/guardrail.yaml @@ -10,6 +10,7 @@ - {id: guardrail.litellm_content_filter.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "test_team_disable_global_guardrail_e2e.py", rationale: "Local content-filter default-on blocks banned keyword pre-call"} - {id: guardrail.litellm_content_filter.pre_call.blocks_video, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [videos], source: "test_key_guardrail_video_e2e.py", fail_before_fix: proven, rationale: "A content-filter guardrail attached to a key (metadata.guardrails) blocks a banned prompt on POST /v1/videos before the provider is called; before the fix the route's call type was unknown to the unified guardrail hook and the prompt went to the provider unscanned (LIT-6685)"} - {id: guardrail.litellm_content_filter.pre_call.allows, module: guardrail, tier: P0, hook_point: pre_call, assertions: [allows], exercised_on: [chat_completions], source: "test_team_disable_global_guardrail_e2e.py", rationale: "Team disable_global_guardrails bypasses default-on content filter"} +- {id: guardrail.litellm_content_filter.pre_call.returns_guardrail_information, module: guardrail, tier: P0, hook_point: pre_call, assertions: [allows], exercised_on: [chat_completions], source: "guardrails/test_guardrail_information_response_e2e.py", rationale: "Opt-in chat responses expose successful guardrail execution details"} - {id: guardrail.litellm_content_filter.apply_endpoint.blocks, module: guardrail, tier: P0, hook_point: apply_endpoint, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_endpoints.py:apply_guardrail", rationale: "POST /guardrails/apply_guardrail blocks banned content for customers that call the apply surface directly"} - {id: guardrail.litellm_content_filter.apply_endpoint.allows, module: guardrail, tier: P0, hook_point: apply_endpoint, assertions: [allows], exercised_on: [chat_completions], source: "guardrail_endpoints.py:apply_guardrail", rationale: "POST /guardrails/apply_guardrail returns clean text for allowed input"} - {id: guardrail.bedrock.during.blocks, module: guardrail, tier: P0, hook_point: during, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "During-call moderation for streaming"} diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py index 007e9392616..60f875ccb7d 100644 --- a/tests/e2e/guardrails/guardrails_client.py +++ b/tests/e2e/guardrails/guardrails_client.py @@ -291,6 +291,7 @@ class GuardrailsClient: text: str, *, guardrails: list[str] | None = None, + include_guardrail_response: bool | None = None, max_tokens: int = 16, tools: list[ChatTool] | None = None, ) -> Result[ChatResponse]: @@ -306,6 +307,7 @@ class GuardrailsClient: messages=[ChatMessage(role="user", content=text)], max_tokens=max_tokens, guardrails=guardrails, + include_guardrail_response=include_guardrail_response, tools=tools, ), ) diff --git a/tests/e2e/guardrails/test_guardrail_information_response_e2e.py b/tests/e2e/guardrails/test_guardrail_information_response_e2e.py new file mode 100644 index 00000000000..9701ea35819 --- /dev/null +++ b/tests/e2e/guardrails/test_guardrail_information_response_e2e.py @@ -0,0 +1,118 @@ +"""Live e2e: an opted-in chat response includes the guardrail execution details.""" + +from __future__ import annotations + +import time +from typing import Final + +import pytest + +from e2e_config import unique_marker +from e2e_http import unwrap +from guardrails_client import ( + BlockedWordBody, + ContentFilterParamsBody, + GuardrailsClient, +) +from lifecycle import ResourceManager +from models import ChatResponse, GuardrailInformationEntry + +pytestmark = pytest.mark.e2e + +GUARDRAIL_PROPAGATION_DEADLINE_SECONDS: Final = 40.0 +GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS: Final = 5.0 + + +def _register_content_filter(client: GuardrailsClient, resources: ResourceManager, *, name: str) -> None: + guardrail_id = client.register( + name, + ContentFilterParamsBody( + mode="pre_call", + default_on=False, + blocked_words=[BlockedWordBody(keyword=f"never-match-{unique_marker()}", action="MASK")], + ), + ) + resources.defer(lambda: client.delete_guardrail(guardrail_id)) + + +def _opted_in_entries( + client: GuardrailsClient, + key: str, + model: str, + name: str, +) -> tuple[ChatResponse, tuple[GuardrailInformationEntry, ...]]: + response = unwrap( + client.chat( + key, + model, + "Reply with the single word OK.", + guardrails=[name], + include_guardrail_response=True, + max_tokens=16, + ) + ) + entries = tuple(entry for entry in response.guardrail_information or () if entry.guardrail_name == name) + return response, entries + + +class TestGuardrailInformationResponse: + @pytest.mark.covers( + "guardrail.litellm_content_filter.pre_call.returns_guardrail_information", + exercised_on=["chat_completions"], + ) + def test_flag_returns_guardrail_information_for_the_guardrail_that_ran( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"e2e-guardrail-information-{unique_marker()}" + _register_content_filter(client, resources, name=name) + model = client.create_backend_model( + resources, + prefix="e2e-guardrail-info-backend", + backend="openai/gpt-4.1-mini", + api_key="os.environ/OPENAI_API_KEY", + ) + deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS + + while True: + response, entries = _opted_in_entries(client, scoped_key, model, name) + if len(entries) == 1: + entry = entries[0] + assert entry.guardrail_status == "success", ( + f"guardrail information should report a successful run, got {entry!r}; response: {response}" + ) + assert entry.duration is not None and entry.duration >= 0, ( + f"guardrail information should report a non-negative duration; response: {response}" + ) + return + if time.monotonic() >= deadline: + pytest.fail( + f"guardrail information did not report exactly one successful {name!r} entry within " + f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; response: {response}" + ) + time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS) + + def test_without_flag_response_has_no_guardrail_information( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"e2e-guardrail-information-default-{unique_marker()}" + _register_content_filter(client, resources, name=name) + model = client.create_backend_model( + resources, + prefix="e2e-guardrail-info-backend", + backend="openai/gpt-4.1-mini", + api_key="os.environ/OPENAI_API_KEY", + ) + + response = unwrap( + client.chat( + scoped_key, + model, + "Reply with the single word OK.", + guardrails=[name], + max_tokens=16, + ) + ) + + assert "guardrail_information" not in response.model_fields_set, ( + f"guardrail information must remain absent without include_guardrail_response, got {response}" + ) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 3309479edaf..74cad3ad8df 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -334,6 +334,7 @@ class ChatBody(BaseModel): tools: Sequence[ChatTool | McpChatTool] | None = None tool_choice: str | None = None guardrails: list[str] | None = None + include_guardrail_response: bool | None = None response_format: dict[str, object] | None = None chat_template_kwargs: dict[str, bool] | None = None cache: dict[str, bool] | None = {"no-cache": True} @@ -448,6 +449,14 @@ class Usage(BaseModel): completion_tokens_details: CompletionTokensDetails | None = None +class GuardrailInformationEntry(BaseModel): + guardrail_name: str + guardrail_status: str + guardrail_mode: object | None = None + guardrail_response: object | None = None + duration: float | None = None + + class ChatResponse(BaseModel): id: str | None = None object: str | None = None @@ -455,6 +464,7 @@ class ChatResponse(BaseModel): choices: list[ChatChoice] = [] usage: Usage | None = None service_tier: str | None = None + guardrail_information: list[GuardrailInformationEntry] | None = None # ---------- anthropic /v1/messages + count_tokens ---------- diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py index bca61a0e76f..6eeea271127 100644 --- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py +++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py @@ -322,6 +322,28 @@ class TestRedactNestedMatchAndRegexKeys: assert redact_nested_match_and_regex_keys(None) is None assert redact_nested_match_and_regex_keys("plain") == "plain" + def test_redacts_custom_keys_without_changing_default_keys(self): + payload = { + "keyword": "secret-keyword", + "snippet": "secret-snippet", + "match": "secret-match", + "regex": "secret-regex", + "nested": [{"keyword": "nested-keyword", "match": "nested-match"}], + } + + custom_keys = redact_nested_match_and_regex_keys(payload, keys=("keyword", "snippet")) + default_keys = redact_nested_match_and_regex_keys(payload) + + assert custom_keys["keyword"] == "[REDACTED]" + assert custom_keys["snippet"] == "[REDACTED]" + assert custom_keys["nested"][0]["keyword"] == "[REDACTED]" + assert custom_keys["match"] == "secret-match" + assert custom_keys["regex"] == "secret-regex" + assert default_keys["match"] == "[REDACTED]" + assert default_keys["regex"] == "[REDACTED]" + assert default_keys["keyword"] == "secret-keyword" + assert default_keys["snippet"] == "secret-snippet" + @pytest.mark.parametrize( "value, expected", diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 360122cde25..5b9cd761dda 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -35,10 +35,12 @@ from litellm.proxy.common_request_processing import ( _buffer_first_chunk_honoring_disconnect, _cancel_llm_call_on_client_disconnect, _ClientDisconnectedBeforeFirstChunk, + attach_guardrail_information, _extract_error_from_sse_chunk, _get_cost_breakdown_from_logging_obj, CostBreakdownHeaderValues, _has_attribute_error_in_chain, + include_guardrail_response_requested, _is_azure_model_router_request, open_sse_before_first_byte, resolve_litellm_call_id, @@ -61,6 +63,132 @@ from litellm.proxy.utils import ProxyLogging from litellm.router import Router +def test_attach_guardrail_information_copies_recorded_entries_onto_model_response(): + recorded = [ + {"guardrail_name": "first", "guardrail_status": "success"}, + {"guardrail_name": "second", "guardrail_status": "success"}, + ] + response = litellm.ModelResponse() + + result = attach_guardrail_information( + response=response, + request_data={"metadata": {"standard_logging_guardrail_information": recorded}}, + ) + + assert isinstance(result, litellm.ModelResponse) + assert result.model_dump()["guardrail_information"] == recorded + assert "guardrail_information" not in response.model_dump() + + +def test_attach_guardrail_information_reports_empty_list_when_nothing_ran(): + response = litellm.ModelResponse() + + result = attach_guardrail_information(response=response, request_data={}) + + assert isinstance(result, litellm.ModelResponse) + assert result.model_dump()["guardrail_information"] == [] + assert "guardrail_information" not in response.model_dump() + + +def test_attach_guardrail_information_sets_key_on_dict_response(): + recorded = [{"guardrail_name": "first", "guardrail_status": "success"}] + response = {"id": "x"} + + result = attach_guardrail_information( + response=response, + request_data={"metadata": {"standard_logging_guardrail_information": recorded}}, + ) + + assert isinstance(result, dict) + assert result == {"id": "x", "guardrail_information": recorded} + assert response == {"id": "x"} + + +def test_attach_guardrail_information_redacts_matched_content(): + recorded = [ + { + "guardrail_name": "cf", + "guardrail_status": "success", + "guardrail_response": [ + {"type": "blocked_word", "keyword": "secret-word", "action": "MASK"} + ], + "match_details": [{"snippet": "secret-word", "detection_method": "keyword"}], + } + ] + + result = attach_guardrail_information( + response={"id": "x"}, + request_data={"metadata": {"standard_logging_guardrail_information": recorded}}, + ) + + assert isinstance(result, dict) + guardrail_information = result["guardrail_information"] + assert isinstance(guardrail_information, list) + assert guardrail_information[0]["guardrail_response"][0]["keyword"] == "[REDACTED]" + assert guardrail_information[0]["match_details"][0]["snippet"] == "[REDACTED]" + assert guardrail_information[0]["match_details"][0]["detection_method"] == "keyword" + assert "secret-word" not in json.dumps(result) + + +def test_attach_guardrail_information_leaves_cached_dict_response_untouched(): + recorded = [{"guardrail_name": "cf", "guardrail_status": "success"}] + cached = {"id": "x", "content": []} + + result = attach_guardrail_information( + response=cached, + request_data={ + "metadata": { + "include_guardrail_response": True, + "standard_logging_guardrail_information": recorded, + } + }, + ) + + assert "guardrail_information" not in cached + assert result is not cached + assert isinstance(result, dict) + assert result["guardrail_information"] == recorded + + original = litellm.ModelResponse() + copied = attach_guardrail_information( + response=original, + request_data={"metadata": {"standard_logging_guardrail_information": recorded}}, + ) + + assert "guardrail_information" not in original.model_dump() + assert isinstance(copied, litellm.ModelResponse) + assert copied.model_dump()["guardrail_information"] == recorded + + +def test_include_guardrail_response_requested_reads_flag_from_metadata_when_router_seeded_litellm_metadata(): + recorded = [ + {"guardrail_name": "first", "guardrail_status": "success"}, + {"guardrail_name": "second", "guardrail_status": "success"}, + ] + request_data = { + "metadata": { + "include_guardrail_response": True, + "standard_logging_guardrail_information": recorded, + }, + "litellm_metadata": {}, + } + + assert include_guardrail_response_requested(request_data) is True + + response = litellm.ModelResponse() + result = attach_guardrail_information(response=response, request_data=request_data) + + assert isinstance(result, litellm.ModelResponse) + assert result.model_dump()["guardrail_information"] == recorded + + +def test_include_guardrail_response_requested_is_false_without_exact_true(): + assert include_guardrail_response_requested( + {"metadata": {"include_guardrail_response": "true"}, "litellm_metadata": {}} + ) is False + assert include_guardrail_response_requested({}) is False + + class TestProxyBaseLLMRequestProcessing: @pytest.mark.asyncio async def test_base_passthrough_process_llm_request_preserves_litellm_headers_for_non_streaming_response( diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 9257a2dd23d..b7b7b5d1942 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -34,6 +34,7 @@ from litellm.proxy.litellm_pre_call_utils import ( add_provider_specific_headers_to_request, check_if_token_is_service_account, clean_headers, + move_guardrails_to_metadata, ) from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY @@ -5187,6 +5188,45 @@ def test_clean_headers_strips_x_api_key_when_byok_enabled_but_x_api_key_was_auth # --------------------------------------------------------------------------- +@pytest.mark.asyncio +async def test_move_guardrails_to_metadata_moves_include_guardrail_response_before_the_no_guardrail_early_out(): + policy_registry = MagicMock() + policy_registry.is_initialized.return_value = False + user_api_key_dict = UserAPIKeyAuth(api_key="test-key") + + true_data = { + "model": "gpt-4.1-mini", + "messages": [{"role": "user", "content": "Hello"}], + "metadata": {}, + "include_guardrail_response": True, + } + with patch("litellm.proxy.policy_engine.policy_registry.get_policy_registry", return_value=policy_registry): + await move_guardrails_to_metadata( + data=true_data, + _metadata_variable_name="metadata", + user_api_key_dict=user_api_key_dict, + ) + + assert "include_guardrail_response" not in true_data + assert true_data["metadata"]["include_guardrail_response"] is True + + string_data = { + "model": "gpt-4.1-mini", + "messages": [{"role": "user", "content": "Hello"}], + "metadata": {}, + "include_guardrail_response": "true", + } + with patch("litellm.proxy.policy_engine.policy_registry.get_policy_registry", return_value=policy_registry): + await move_guardrails_to_metadata( + data=string_data, + _metadata_variable_name="metadata", + user_api_key_dict=user_api_key_dict, + ) + + assert "include_guardrail_response" not in string_data + assert string_data["metadata"]["include_guardrail_response"] is False + + @pytest.mark.asyncio async def test_team_guardrail_merges_with_global_policy(): """ From 1373322e0c9c6c0ecb4d25ac88aae7b758ac712a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:44:03 -0700 Subject: [PATCH 23/48] feat(logs): add span type filter to request logs (#42491) * feat(logs): add span type filter to request logs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logs): look up span type sql conditions from a mapping Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Mubashir Osmani Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../spend_management_endpoints.py | 33 +++++++ .../test_spend_management_endpoints.py | 98 +++++++++++++++++++ .../src/components/networking.tsx | 1 + .../view_logs/RequestLogsFilters.test.tsx | 33 +++++++ .../view_logs/RequestLogsFilters.tsx | 29 ++++++ .../components/view_logs/RequestLogsTable.tsx | 11 ++- .../src/components/view_logs/constants.ts | 7 ++ .../view_logs/log_filter_logic.test.tsx | 2 + .../components/view_logs/log_filter_logic.tsx | 3 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 + 10 files changed, 220 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index a319535f725..b5980f9b224 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -69,6 +69,18 @@ _SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)" _SESSION_GROUP_KEY_SQL: Final = f"{_SESSION_KEY_EXPR}, api_key" _MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')" _AGENT_CALL_TYPE_SQL: Final = "'asend_message'" +_BATCH_CALL_TYPES_SQL: Final = "('acreate_batch', 'create_batch', 'aretrieve_batch', 'retrieve_batch')" +_SPAN_TYPE_SQL_CONDITIONS: Final[Mapping[str, str]] = MappingProxyType( + { + "mcp": f"call_type IN {_MCP_CALL_TYPES_SQL}", + "agent": f"call_type = {_AGENT_CALL_TYPE_SQL}", + "batch": f"call_type IN {_BATCH_CALL_TYPES_SQL}", + "llm": ( + f"(call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL} " + f"AND call_type NOT IN {_BATCH_CALL_TYPES_SQL})" + ), + } +) _SPEND_LOG_LIST_COLUMNS: Final = """ request_id, call_type, api_key, spend, total_tokens, prompt_tokens, completion_tokens, "startTime", "endTime", @@ -2410,6 +2422,10 @@ async def ui_view_spend_logs( default=None, description="Filter logs by cache state: 'hit' or 'miss'. Miss includes legacy rows with a null/unknown cache state", ), + span_type: str | None = fastapi.Query( + default=None, + description="Filter logs by span type: llm, agent, mcp, or batch", + ), model: str | None = fastapi.Query(default=None, description="Filter logs by model"), model_id: str | None = fastapi.Query( default=None, @@ -2512,6 +2528,13 @@ async def ui_view_spend_logs( param="cache_hit_filter", code=status.HTTP_400_BAD_REQUEST, ) + if isinstance(span_type, str) and span_type not in _SPAN_TYPE_SQL_CONDITIONS: + raise ProxyException( + message=f"Invalid span_type: {span_type}. Must be one of: llm, agent, mcp, batch", + type="bad_request", + param="span_type", + code=status.HTTP_400_BAD_REQUEST, + ) try: is_admin_view: Final = _is_admin_view_safe(user_api_key_dict=user_api_key_dict) @@ -2776,6 +2799,10 @@ async def ui_view_spend_logs( elif cache_hit_filter == "miss": sql_conditions.append("(cache_hit IS NULL OR LOWER(cache_hit) != 'true')") + span_type_condition: Final = _span_type_sql_condition(span_type) + if span_type_condition is not None: + sql_conditions.append(span_type_condition) + if exclude_internal_health_checks: sql_conditions.append(f"api_key NOT IN (${p}, ${p + 1})") sql_params.extend(_INTERNAL_HEALTH_CHECK_API_KEYS) @@ -4673,6 +4700,12 @@ def _build_status_filter_condition(status_filter: str | None) -> Mapping[str, ob return {"status": {"equals": status_filter}} +def _span_type_sql_condition(span_type: str | None) -> str | None: + if span_type is None: + return None + return _SPAN_TYPE_SQL_CONDITIONS.get(span_type) + + def _is_admin_view_safe(user_api_key_dict: UserAPIKeyAuth) -> bool: """ Safely determine if the current user has admin view permissions. diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 9de6679472e..16cbd109b24 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -139,6 +139,15 @@ def _reconstruct_ui_where_from_sql(sql_query, params): where["cache_hit"] = "hit" elif cond == "(cache_hit IS NULL OR LOWER(cache_hit) != 'true')": where["cache_hit"] = "miss" + elif "call_type" in cond: + if "call_type NOT IN" in cond: + where["span_type"] = "llm" + elif "call_mcp_tool" in cond: + where["span_type"] = "mcp" + elif "call_type = 'asend_message'" in cond: + where["span_type"] = "agent" + elif "acreate_batch" in cond: + where["span_type"] = "batch" elif sess: where["session_id"] = {"contains": str(params[int(sess.group(1)) - 1]).strip("%")} elif status: @@ -3418,6 +3427,95 @@ async def test_ui_view_spend_logs_with_cache_hit_filter(client, monkeypatch): app.dependency_overrides.pop(ps.user_api_key_auth, None) +@pytest.mark.asyncio +async def test_ui_view_spend_logs_with_span_type_filter(client, monkeypatch): + base = { + "api_key": "sk-test-key", + "user": "test_user_1", + "team_id": "team1", + "spend": 0.05, + "startTime": datetime.datetime.now(timezone.utc).isoformat(), + "model": "gpt-4", + "status": "success", + } + mock_spend_logs = [ + {**base, "id": "log1", "request_id": "req-llm", "call_type": "acompletion"}, + {**base, "id": "log2", "request_id": "req-agent", "call_type": "asend_message"}, + {**base, "id": "log3", "request_id": "req-mcp", "call_type": "call_mcp_tool"}, + {**base, "id": "log4", "request_id": "req-batch", "call_type": "aretrieve_batch"}, + ] + + call_types_by_span = { + "llm": lambda ct: ct not in {"call_mcp_tool", "list_mcp_tools", "asend_message"} + and ct not in {"acreate_batch", "create_batch", "aretrieve_batch", "retrieve_batch"}, + "agent": lambda ct: ct == "asend_message", + "mcp": lambda ct: ct in {"call_mcp_tool", "list_mcp_tools"}, + "batch": lambda ct: ct + in {"acreate_batch", "create_batch", "aretrieve_batch", "retrieve_batch"}, + } + + def filter_by_span_type(where): + span_type = where.get("span_type") + if span_type is None: + return mock_spend_logs + return [log for log in mock_spend_logs if call_types_by_span[span_type](log["call_type"])] + + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + make_ui_spend_logs_mock_prisma(mock_spend_logs, filter_by_span_type), + ) + + start_date, end_date = _default_date_range() + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN + ) + try: + for span_type, expected_ids in [ + ("batch", ["req-batch"]), + ("llm", ["req-llm"]), + ("mcp", ["req-mcp"]), + ("agent", ["req-agent"]), + ]: + response = client.get( + "/spend/logs/ui", + params={ + "span_type": span_type, + "start_date": start_date, + "end_date": end_date, + }, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200, response.text + data = response.json() + assert data["total"] == len(expected_ids) + assert [row["request_id"] for row in data["data"]] == expected_ids + + response = client.get( + "/spend/logs/ui", + params={ + "start_date": start_date, + "end_date": end_date, + }, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200 + assert response.json()["total"] == 4 + + response = client.get( + "/spend/logs/ui", + params={ + "span_type": "invalid", + "start_date": start_date, + "end_date": end_date, + }, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 400 + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + @pytest.mark.asyncio async def test_ui_view_spend_logs_with_model(client, monkeypatch): mock_spend_logs = [ diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 82399792674..d358d23408c 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2003,6 +2003,7 @@ interface UiSpendLogsParams { end_user?: string; status_filter?: string; cache_hit_filter?: string; + span_type?: string; /** Filter by model name (e.g. "gpt-4") */ model?: string; /** Filter by model ID (litellm model deployment id) */ diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx index 8d1847e0121..a2da80a3f7b 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx @@ -84,6 +84,7 @@ describe("RequestLogsFilters", () => { for (const label of [ "Team ID", + "Span Type", "Status", "Cache", "Key Alias", @@ -286,6 +287,38 @@ describe("RequestLogsFilters", () => { expect(await screen.findByText(label)).toBeInTheDocument(); }); + it.each([ + ["", "All Types"], + ["llm", "LLM"], + ["agent", "Agent"], + ["mcp", "MCP"], + ["batch", "Batch"], + ])("shows the human label on the Span Type trigger for %s", async (spanType, label) => { + renderFilters(spanType === "" ? {} : { [LOG_FILTER_IDS.SPAN_TYPE]: spanType }); + + expect(await screen.findByText(label)).toBeInTheDocument(); + }); + + it("selecting Batch sets the span_type filter", async () => { + const user = userEvent.setup(); + const { set } = renderFilters(); + + await user.click(await screen.findByText("All Types")); + await user.click(await screen.findByRole("option", { name: "Batch" })); + + expect(set).toHaveBeenCalledWith(LOG_FILTER_IDS.SPAN_TYPE, "batch"); + }); + + it("selecting All Types clears the span_type filter", async () => { + const user = userEvent.setup(); + const { set } = renderFilters({ [LOG_FILTER_IDS.SPAN_TYPE]: "batch" }); + + await user.click(await screen.findByText("Batch")); + await user.click(await screen.findByRole("option", { name: "All Types" })); + + expect(set).toHaveBeenCalledWith(LOG_FILTER_IDS.SPAN_TYPE, undefined); + }); + it.each([ ["Cache Hit", "hit"], ["Cache Miss", "miss"], diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.tsx index e97552838da..5059f117944 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.tsx @@ -37,6 +37,14 @@ const CACHE_FILTER_ITEMS = [ { value: "hit", label: "Cache Hit" }, { value: "miss", label: "Cache Miss" }, ] as const; + +const SPAN_TYPE_FILTER_ITEMS = [ + { value: ALL_VALUE, label: "All Types" }, + { value: "llm", label: "LLM" }, + { value: "agent", label: "Agent" }, + { value: "mcp", label: "MCP" }, + { value: "batch", label: "Batch" }, +] as const; const PAGE_SIZE = 50; const SEARCH_INPUT_REASONS: ReadonlySet = new Set(["input-change", "input-clear", "clear-press"]); @@ -328,6 +336,27 @@ export function RequestLogsFilters({ get, set, teams, logsWindow }: RequestLogsF teams={teams} /> + + + + } @@ -135,10 +143,22 @@ const RoutingGroupModal: React.FC = ({ control={form.control} name="models" label="Models" - description="Models from your model list that this group routes between. A model can only be in one group." + description={ + selectedStrategy === "priority" + ? "Models from your model list that this group routes between. Models can belong to multiple priority groups." + : "Models from your model list that this group routes between. A model can belong to one non-priority group." + } > {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( - + { + onChange(models); + form.setValue("model_priorities", prioritiesForModels(models, form.getValues("model_priorities"))); + }} + > }> {(selected: string[]) => ( @@ -176,7 +196,11 @@ const RoutingGroupModal: React.FC = ({ control={form.control} name="routing_strategy" label="Routing Strategy" - description={strategyDescriptions[selectedStrategy]} + description={ + selectedStrategy === "priority" + ? "Lower priorities are tried first. Models with the same priority share traffic. Applies only when calling this group." + : strategyDescriptions[selectedStrategy] + } > {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( + onChange( + value.map((item, itemIndex) => + itemIndex === index ? { ...item, priority: event.target.value } : item, + ), + ) + } + /> + {!selectedModels.includes(entry.model) && ( + + )} + + ))} + + )} + + )} + {STRATEGIES_WITH_ARGS.has(selectedStrategy) && ( = ({ )}

- Models not claimed by an explicit group fall through to the proxy's top-level routing strategy. + {selectedStrategy === "priority" + ? "Direct requests to a member model keep their existing routing behavior." + : "Models outside non-priority groups use the proxy's top-level routing strategy."}

diff --git a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupUsagePanel.tsx b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupUsagePanel.tsx index fafea569012..85dd67099d6 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupUsagePanel.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupUsagePanel.tsx @@ -14,7 +14,8 @@ interface RoutingGroupUsagePanelProps { baseUrl: string; } -const exampleModel = (group: RoutingGroup): string => group.models[0] ?? ""; +const exampleModel = (group: RoutingGroup): string => + group.routing_strategy === "priority" ? group.group_name : group.models[0] ?? ""; const buildCurlSnippet = (group: RoutingGroup, baseUrl: string): string => `curl -X POST '${baseUrl}/v1/chat/completions' \\ @@ -69,8 +70,17 @@ export function RoutingGroupUsagePanel({ group, baseUrl }: RoutingGroupUsagePane How routing works for this group

- Callers request any model in the group by name; LiteLLM picks a deployment behind the scenes using the{" "} - {formatStrategyLabel(group.routing_strategy)} strategy. + {group.routing_strategy === "priority" ? ( + <> + Request {group.group_name} to try eligible models in + priority order. Direct requests to a member model keep their existing routing behavior. + + ) : ( + <> + Callers request any model in the group by name; LiteLLM picks a deployment behind the scenes using the{" "} + {formatStrategyLabel(group.routing_strategy)} strategy. + + )}

diff --git a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.test.tsx b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.integration.test.tsx similarity index 86% rename from ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.test.tsx rename to ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.integration.test.tsx index 6f14b76e2fd..6664ecab623 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.test.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.integration.test.tsx @@ -113,6 +113,22 @@ describe("RoutingGroupsTable", () => { expect(panel?.textContent).toContain("gpt-4o"); }); + it("uses the callable group name in every priority routing example", async () => { + const user = userEvent.setup(); + render(); + expect(within(rowFor("prod-group")).getByText("Priority")).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "prod-group" })); + + expect(screen.getByRole("tabpanel")).toHaveTextContent('"model": "prod-group"'); + await user.click(screen.getByRole("tab", { name: "Python (OpenAI SDK)" })); + expect(screen.getByRole("tabpanel")).toHaveTextContent('model="prod-group"'); + await user.click(screen.getByRole("tab", { name: "JavaScript (OpenAI SDK)" })); + expect(screen.getByRole("tabpanel")).toHaveTextContent('model: "prod-group"'); + expect( + screen.getByText(/Direct requests to a member model keep their existing routing behavior/), + ).toBeInTheDocument(); + }); + it("should expand only the clicked group", async () => { const user = userEvent.setup(); render(); diff --git a/ui/litellm-dashboard/src/components/routing_groups/index.tsx b/ui/litellm-dashboard/src/components/routing_groups/index.tsx index 17329d0b572..ae2d62ae20f 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/index.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/index.tsx @@ -46,6 +46,7 @@ const RoutingGroups: React.FC = () => { const availableStrategies = useMemo(() => { if (data?.availableStrategies?.length) return data.availableStrategies; + if (routerFields?.routing_group_strategies?.length) return routerFields.routing_group_strategies; const fromFields = routerFields?.fields?.find((f) => f.field_name === "routing_strategy")?.options; return fromFields ?? []; }, [data?.availableStrategies, routerFields]); @@ -178,8 +179,17 @@ const RoutingGroups: React.FC = () => { Delete routing group?

- Models in {deletingGroup?.group_name} will fall back to the - proxy's top-level routing strategy. This cannot be undone. + {deletingGroup?.routing_strategy === "priority" ? ( + <> + Calls to {deletingGroup.group_name} will stop working. Direct + requests to its member models keep their existing routing behavior. This cannot be undone. + + ) : ( + <> + Models in {deletingGroup?.group_name} will fall back to the + proxy's top-level routing strategy. This cannot be undone. + + )}

{showValidationErrors && error &&

{error}

} diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 9b4f1980f62..13df40464f0 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -414,8 +414,8 @@ export const getReminderMarkersError = (pairs: ReminderMarkerPair[] | undefined) for (const [index, pair] of (pairs ?? []).entries()) { const open = pair.open.trim().toLowerCase(); const close = pair.close.trim().toLowerCase(); - if (!open || !close) return `Reminder marker pair ${index + 1} needs both an opening and a closing delimiter`; - if (open === close) return `Reminder marker pair ${index + 1} must use different opening and closing delimiters`; + if (!open || !close) return `Tag pair ${index + 1} needs both an opening and a closing tag`; + if (open === close) return `Tag pair ${index + 1} must use different opening and closing tags`; } return null; }; diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx index aa6cf92ceb9..6d66152cee5 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx @@ -377,9 +377,9 @@ describe("EditAutoRouterModal advanced field round trips", () => { expect(screen.getByRole("switch", { name: "Route housekeeping calls to the cheapest tier" })).not.toBeChecked(); expect(screen.getByRole("combobox", { name: "e.g., conversation title" })).toHaveValue(""); - await user.click(screen.getByText("Advanced: Reminder Markers")); - expect(screen.getByLabelText("Opening delimiter")).toHaveValue(""); - expect(screen.getByLabelText("Closing delimiter")).toHaveValue(""); + await user.click(screen.getByText("Advanced: Ignore Custom Tags")); + expect(screen.getByLabelText("Opening tag")).toHaveValue(""); + expect(screen.getByLabelText("Closing tag")).toHaveValue(""); await user.click(screen.getByText("Advanced: Response Format")); const maxTokensSwitch = screen.getByRole("switch", { name: "Cap max_tokens at the tier model's output ceiling" }); From 2727359a9ef5b6e058c668ba1477fd13c49df136 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:24:49 -0700 Subject: [PATCH 32/48] chore(prices): sync Baseten prices: 12 models, 9 new [enrichment failed: Baseten, 15 held] (#42558) baseten/deepseek-ai/DeepSeek-V4.1-Flash: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_reasoning, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost baseten/moonshotai/Kimi-K2.6: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_reasoning, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost baseten/moonshotai/Kimi-K2.7-Code: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_reasoning, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost baseten/moonshotai/Kimi-K3: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_reasoning, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost baseten/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_reasoning, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost baseten/openai/gpt-oss-120b: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_reasoning, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, cache_read_input_token_cost baseten/thinkingmachines/inkling: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_reasoning, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost baseten/thinkingmachines/inkling-small: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_reasoning, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost baseten/zai-org/GLM-4.7: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_reasoning, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, cache_read_input_token_cost baseten/zai-org/GLM-5.2: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_reasoning, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost baseten/zai-org/GLM-5.3: supports_reasoning baseten/zai-org/GLM-5.3-Flash: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_reasoning, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 182 +++++++++++++++++- model_prices_and_context_window.json | 182 +++++++++++++++++- 2 files changed, 358 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0e80d4732c8..a2863f6b0dc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -31722,10 +31722,21 @@ "output_cost_per_token": 3.15e-06 }, "baseten/zai-org/GLM-4.7": { + "cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 6e-07, "litellm_provider": "baseten", + "max_input_tokens": 200000, + "max_output_tokens": 200000, + "max_tokens": 200000, "mode": "chat", - "output_cost_per_token": 2.2e-06 + "output_cost_per_token": 2.2e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false }, "baseten/zai-org/GLM-4.6": { "input_cost_per_token": 6e-07, @@ -31752,10 +31763,21 @@ "output_cost_per_token": 2.5e-06 }, "baseten/openai/gpt-oss-120b": { + "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-07, "litellm_provider": "baseten", + "max_input_tokens": 128072, + "max_output_tokens": 128072, + "max_tokens": 128072, "mode": "chat", - "output_cost_per_token": 5e-07 + "output_cost_per_token": 5e-07, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false }, "baseten/deepseek-ai/DeepSeek-V3.1": { "input_cost_per_token": 5e-07, @@ -68322,7 +68344,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://www.baseten.co/pricing/", + "source": "https://inference.baseten.co/v1/models", "supported_modalities": [ "text", "image" @@ -68332,6 +68354,7 @@ ], "supports_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true @@ -77489,5 +77512,158 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": false + }, + "baseten/deepseek-ai/DeepSeek-V4.1-Flash": { + "cache_read_input_token_cost": 7e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/moonshotai/Kimi-K2.6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "baseten", + "max_input_tokens": 262000, + "max_output_tokens": 262000, + "max_tokens": 262000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/moonshotai/Kimi-K2.7-Code": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "baseten", + "max_input_tokens": 262000, + "max_output_tokens": 262000, + "max_tokens": 262000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/moonshotai/Kimi-K3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "baseten", + "max_input_tokens": 202800, + "max_output_tokens": 202800, + "max_tokens": 202800, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "baseten/thinkingmachines/inkling": { + "cache_read_input_token_cost": 1.7e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4.05e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/thinkingmachines/inkling-small": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 5e-07, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/zai-org/GLM-5.2": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/zai-org/GLM-5.3-Flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0e80d4732c8..a2863f6b0dc 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -31722,10 +31722,21 @@ "output_cost_per_token": 3.15e-06 }, "baseten/zai-org/GLM-4.7": { + "cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 6e-07, "litellm_provider": "baseten", + "max_input_tokens": 200000, + "max_output_tokens": 200000, + "max_tokens": 200000, "mode": "chat", - "output_cost_per_token": 2.2e-06 + "output_cost_per_token": 2.2e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false }, "baseten/zai-org/GLM-4.6": { "input_cost_per_token": 6e-07, @@ -31752,10 +31763,21 @@ "output_cost_per_token": 2.5e-06 }, "baseten/openai/gpt-oss-120b": { + "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-07, "litellm_provider": "baseten", + "max_input_tokens": 128072, + "max_output_tokens": 128072, + "max_tokens": 128072, "mode": "chat", - "output_cost_per_token": 5e-07 + "output_cost_per_token": 5e-07, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false }, "baseten/deepseek-ai/DeepSeek-V3.1": { "input_cost_per_token": 5e-07, @@ -68322,7 +68344,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://www.baseten.co/pricing/", + "source": "https://inference.baseten.co/v1/models", "supported_modalities": [ "text", "image" @@ -68332,6 +68354,7 @@ ], "supports_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true @@ -77489,5 +77512,158 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": false + }, + "baseten/deepseek-ai/DeepSeek-V4.1-Flash": { + "cache_read_input_token_cost": 7e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/moonshotai/Kimi-K2.6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "baseten", + "max_input_tokens": 262000, + "max_output_tokens": 262000, + "max_tokens": 262000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/moonshotai/Kimi-K2.7-Code": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "baseten", + "max_input_tokens": 262000, + "max_output_tokens": 262000, + "max_tokens": 262000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/moonshotai/Kimi-K3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "baseten", + "max_input_tokens": 202800, + "max_output_tokens": 202800, + "max_tokens": 202800, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "baseten/thinkingmachines/inkling": { + "cache_read_input_token_cost": 1.7e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4.05e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/thinkingmachines/inkling-small": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 5e-07, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/zai-org/GLM-5.2": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/zai-org/GLM-5.3-Flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } From 666f6b01b4c4dc474bbb4d9d129c127cfada6961 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 22 Sep 2026 13:28:31 -0700 Subject: [PATCH 33/48] fix: enforce disable_custom_api_keys from general_settings (#42437) * fix: enforce disable_custom_api_keys from general_settings The gate in _check_custom_key_allowed read the persisted UI settings row through get_ui_settings_cached, which had two consequences. A config-file general_settings.disable_custom_api_keys was never enforced, because the gate only ever looked at the stored ui_settings row. POST /key/generate with a custom key value returned 200 even with the flag set to true in config.yaml. The read went through a DualCache with a 600s TTL, which is per worker without Redis, and only the worker that served PATCH /update/ui_settings refreshed it. A gate flipped through the UI was then a coin flip across workers for up to ten minutes. Both go away by routing the flag the way the other runtime UI flags are already routed. Adding it to _RUNTIME_GENERAL_SETTINGS_FLAGS and to the settings rules' _UI_SETTINGS_FIELDS makes SettingsStore resolve it from the ui_settings row with the config file winning, and every pod re-reads it on its own settings sync rather than holding a private cached copy. The two lists have to stay in step: a flag in one and not the other resolves against the wrong stored row and silently never reaches a reader, so there is a test for that invariant. Writes to the ui_settings table did not publish on the config-sync channel, so other pods only discovered a change on their next periodic reload. Adding litellm_uisettings to _CONFIG_SYNCED_TABLE_NAMES puts it on the same pubsub path model and SSO config writes already use, which cuts cross-pod propagation from tens of seconds to a few. The value reaching the gate is run through coerce_bool first. Resolution hands back the raw YAML value, so a quoted "true" in config.yaml is a str and the old `is True` check let custom keys straight through. * test: assert both directions of the coerced config value * test: assert the runtime flags read back instead of inspecting the registry --- .../proxy/common_utils/config_sync_pubsub.py | 1 + .../proxy/config_resolvers/settings_rules.py | 1 + .../key_management_endpoints.py | 9 +- .../proxy_setting_endpoints.py | 1 + .../common_utils/test_config_sync_pubsub.py | 28 +++ .../test_key_management_endpoints.py | 188 +++++++++++------- .../test_proxy_setting_endpoints.py | 26 +++ 7 files changed, 181 insertions(+), 73 deletions(-) diff --git a/litellm/proxy/common_utils/config_sync_pubsub.py b/litellm/proxy/common_utils/config_sync_pubsub.py index 6d781babe63..b20c0d9c9a5 100644 --- a/litellm/proxy/common_utils/config_sync_pubsub.py +++ b/litellm/proxy/common_utils/config_sync_pubsub.py @@ -55,6 +55,7 @@ _CONFIG_SYNCED_TABLE_NAMES: Final[frozenset[str]] = frozenset( "litellm_ssoconfig", "litellm_cacheconfig", "litellm_configoverrides", + "litellm_uisettings", } ) diff --git a/litellm/proxy/config_resolvers/settings_rules.py b/litellm/proxy/config_resolvers/settings_rules.py index f346dd6198d..1f0adfc5248 100644 --- a/litellm/proxy/config_resolvers/settings_rules.py +++ b/litellm/proxy/config_resolvers/settings_rules.py @@ -48,6 +48,7 @@ _UI_SETTINGS_FIELDS: Final[tuple[str, ...]] = ( "allow_agents_for_team_admins", "disable_vector_stores_for_internal_users", "allow_vector_stores_for_team_admins", + "disable_custom_api_keys", "disable_key_generate_for_org_admin", "team_admin_editable_team_fields", ) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index fbc7cf18003..801c45501ce 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -118,9 +118,6 @@ from litellm.proxy.management_helpers.team_member_permission_checks import ( from litellm.proxy.management_helpers.utils import management_endpoint_wrapper from litellm.proxy.spend_tracking.budget_reservation import get_budget_window_start from litellm.proxy.spend_tracking.spend_tracking_utils import _is_master_key -from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( - get_ui_settings_cached, -) from litellm.proxy.utils import ( PrismaClient, ProxyLogging, @@ -487,8 +484,10 @@ async def _check_custom_key_allowed(custom_key_value: str | None) -> None: if custom_key_value is None: return - ui_settings: Final = await get_ui_settings_cached() - if ui_settings.get("disable_custom_api_keys", False) is True: + from litellm.proxy.config_resolvers.settings_rules import coerce_bool + from litellm.proxy.proxy_server import general_settings + + if coerce_bool(general_settings.get("disable_custom_api_keys", False)) is True: verbose_proxy_logger.warning("Custom API key rejected: disable_custom_api_keys is enabled") raise HTTPException( status_code=403, diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index f37b82f3b54..d520177965c 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -407,6 +407,7 @@ _RUNTIME_GENERAL_SETTINGS_FLAGS: Final = [ "allow_agents_for_team_admins", "disable_vector_stores_for_internal_users", "allow_vector_stores_for_team_admins", + "disable_custom_api_keys", "disable_key_generate_for_org_admin", TEAM_ADMIN_EDITABLE_TEAM_FIELDS_SETTING, ] diff --git a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py index 2a0ebf9492a..0f64ef2b4ca 100644 --- a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py +++ b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py @@ -47,6 +47,7 @@ _EXPECTED_CONFIG_SYNCED_TABLE_NAMES = frozenset( "litellm_proxymodeltable", "litellm_searchtoolstable", "litellm_ssoconfig", + "litellm_uisettings", } ) @@ -702,6 +703,33 @@ async def test_model_repository_write_publishes_via_live_coordination_cache() -> assert json.loads(message) == {"object_type": "litellm_proxymodeltable"} +async def test_ui_settings_write_publishes_via_live_coordination_cache() -> None: + from litellm.proxy import proxy_server + from litellm.proxy.proxy_server import _set_redis_usage_cache + from litellm.repositories.table_repositories import UISettingsRepository + + client = _RecordingRedisClient() + prisma_client = MagicMock() + prisma_client.db.litellm_uisettings.upsert = AsyncMock(return_value={"id": "ui_settings"}) + table = UISettingsRepository(prisma_client).table + assert isinstance(table, _PublishOnWriteActions) + + previous_cache = proxy_server.redis_usage_cache + _set_redis_usage_cache(_FakeRedisCache(client)) + try: + await table.upsert( + where={"id": "ui_settings"}, + data={"create": {"id": "ui_settings"}, "update": {"ui_settings": "{}"}}, + ) + finally: + _set_redis_usage_cache(previous_cache) + + assert len(client.published) == 1 + channel, message = client.published[0] + assert channel == CONFIG_SYNC_CHANNEL + assert json.loads(message) == {"object_type": "litellm_uisettings"} + + async def _publish_calls_for_invalidated_param(param_name: str) -> List[Tuple[str, str]]: from litellm.proxy import proxy_server from litellm.proxy.proxy_server import _set_redis_usage_cache diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 8eaa4901c59..c539a10e1e2 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -609,10 +609,7 @@ async def test_generate_key_debug_log_never_contains_raw_token(monkeypatch, capl mock_prisma_client.insert_data = AsyncMock(side_effect=_insert_data_side_effect) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) from litellm.proxy._types import GenerateKeyRequest, LitellmUserRoles from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth @@ -1494,18 +1491,12 @@ async def test_list_keys_full_object_returns_lifetime_total_spend(): @pytest.mark.asyncio async def test_get_new_token_with_valid_key(monkeypatch): """Test get_new_token function when provided with a valid key that starts with 'sk-'""" - from unittest.mock import AsyncMock - from litellm.proxy._types import RegenerateKeyRequest from litellm.proxy.management_endpoints.key_management_endpoints import ( get_new_token, ) - # Mock get_ui_settings_cached to return setting disabled (custom keys allowed) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) # Test with valid new_key data = RegenerateKeyRequest(new_key="sk-test1234567890abc") @@ -1517,8 +1508,6 @@ async def test_get_new_token_with_valid_key(monkeypatch): @pytest.mark.asyncio async def test_get_new_token_with_invalid_key(monkeypatch): """Test get_new_token function when provided with an invalid key that doesn't start with 'sk-'""" - from unittest.mock import AsyncMock - from fastapi import HTTPException from litellm.proxy._types import RegenerateKeyRequest @@ -1526,11 +1515,7 @@ async def test_get_new_token_with_invalid_key(monkeypatch): get_new_token, ) - # Mock get_ui_settings_cached to return setting disabled (custom keys allowed) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) # Test with invalid new_key (doesn't start with 'sk-') data = RegenerateKeyRequest(new_key="invalid-key-123") @@ -1546,8 +1531,6 @@ async def test_get_new_token_with_invalid_key(monkeypatch): async def test_get_new_token_rejects_short_new_key(monkeypatch): """Regression test for LIT-4355: a short custom key like sk-99 must be rejected, otherwise the stored key_name (sk-...{last 4 chars}) reveals the entire key.""" - from unittest.mock import AsyncMock - from fastapi import HTTPException from litellm.proxy._types import RegenerateKeyRequest @@ -1555,10 +1538,7 @@ async def test_get_new_token_rejects_short_new_key(monkeypatch): get_new_token, ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) data = RegenerateKeyRequest(new_key="sk-99") @@ -1588,10 +1568,7 @@ async def test_generate_key_fn_rejects_short_custom_key(monkeypatch, short_key): ) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) assert len(short_key) < 16 @@ -1628,10 +1605,7 @@ async def test_generate_key_fn_accepts_custom_key_at_minimum_length(monkeypatch) ) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) custom_key = "sk-abcdefghijklm" assert len(custom_key) == 16 @@ -1649,18 +1623,13 @@ async def test_generate_key_fn_accepts_custom_key_at_minimum_length(monkeypatch) @pytest.mark.asyncio async def test_check_custom_key_allowed_when_disabled(monkeypatch): """_check_custom_key_allowed raises 403 when disable_custom_api_keys is true.""" - from unittest.mock import AsyncMock - from fastapi import HTTPException from litellm.proxy.management_endpoints.key_management_endpoints import ( _check_custom_key_allowed, ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={"disable_custom_api_keys": True}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"disable_custom_api_keys": True}) with pytest.raises(HTTPException) as exc_info: await _check_custom_key_allowed("sk-custom-key-123") @@ -1672,16 +1641,11 @@ async def test_check_custom_key_allowed_when_disabled(monkeypatch): @pytest.mark.asyncio async def test_check_custom_key_allowed_when_enabled(monkeypatch): """_check_custom_key_allowed does nothing when disable_custom_api_keys is false.""" - from unittest.mock import AsyncMock - from litellm.proxy.management_endpoints.key_management_endpoints import ( _check_custom_key_allowed, ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={"disable_custom_api_keys": False}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"disable_custom_api_keys": False}) # Should not raise await _check_custom_key_allowed("sk-custom-key-123") @@ -1690,35 +1654,133 @@ async def test_check_custom_key_allowed_when_enabled(monkeypatch): @pytest.mark.asyncio async def test_check_custom_key_allowed_when_unset(monkeypatch): """_check_custom_key_allowed does nothing when setting is not present.""" - from unittest.mock import AsyncMock - from litellm.proxy.management_endpoints.key_management_endpoints import ( _check_custom_key_allowed, ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) # Should not raise await _check_custom_key_allowed("sk-custom-key-123") @pytest.mark.asyncio -async def test_check_custom_key_allowed_none_key_always_passes(monkeypatch): - """_check_custom_key_allowed does nothing when key is None, even if setting is on.""" - from unittest.mock import AsyncMock +async def test_check_custom_key_allowed_honours_the_config_file(monkeypatch): + """A config-file general_settings.disable_custom_api_keys is enforced with no stored UI row.""" + from fastapi import HTTPException + from litellm.proxy.config_resolvers import SettingsStore from litellm.proxy.management_endpoints.key_management_endpoints import ( _check_custom_key_allowed, ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={"disable_custom_api_keys": True}), + general_settings = SettingsStore("general_settings") + general_settings.load_yaml({"disable_custom_api_keys": True}) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", general_settings) + + with pytest.raises(HTTPException) as exc_info: + await _check_custom_key_allowed("sk-custom-key-123456") + + assert exc_info.value.status_code == 403 + + +@pytest.mark.parametrize( + ("config_value", "blocked"), + [ + (True, True), + ("true", True), + ("True", True), + (1, True), + (False, False), + ("false", False), + ("False", False), + (0, False), + ], +) +@pytest.mark.asyncio +async def test_check_custom_key_allowed_coerces_a_non_bool_config_value(monkeypatch, config_value, blocked): + """A YAML value that is not a bare bool, such as a quoted "true", still decides the gate.""" + from fastapi import HTTPException + + from litellm.proxy.config_resolvers import SettingsStore + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _check_custom_key_allowed, ) + general_settings = SettingsStore("general_settings") + general_settings.load_yaml({"disable_custom_api_keys": config_value}) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", general_settings) + + rejected = False + try: + await _check_custom_key_allowed("sk-custom-key-123456") + except HTTPException as e: + rejected = e.status_code == 403 + + assert rejected is blocked + + +@pytest.mark.asyncio +async def test_check_custom_key_allowed_config_file_beats_the_stored_ui_row(monkeypatch): + """The config file owns the flag, so a stored UI row saying false cannot re-open custom keys.""" + from fastapi import HTTPException + + from litellm.proxy.config_resolvers import SettingsStore + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _check_custom_key_allowed, + ) + from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( + apply_runtime_general_settings_flags, + ) + + general_settings = SettingsStore("general_settings") + general_settings.load_yaml({"disable_custom_api_keys": True}) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", general_settings) + + apply_runtime_general_settings_flags({"disable_custom_api_keys": False}) + + with pytest.raises(HTTPException) as exc_info: + await _check_custom_key_allowed("sk-custom-key-123456") + + assert exc_info.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_check_custom_key_allowed_picks_up_a_ui_write_without_the_serving_pod(monkeypatch): + """A pod that never served the PATCH enforces the new value after its own settings sync.""" + from fastapi import HTTPException + + from litellm.proxy.config_resolvers import SettingsStore + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _check_custom_key_allowed, + ) + from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( + apply_runtime_general_settings_flags, + ) + + general_settings = SettingsStore("general_settings") + general_settings.load_yaml({}) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", general_settings) + + await _check_custom_key_allowed("sk-custom-key-123456") + + apply_runtime_general_settings_flags({"disable_custom_api_keys": True}) + + with pytest.raises(HTTPException) as exc_info: + await _check_custom_key_allowed("sk-custom-key-123456") + + assert exc_info.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_check_custom_key_allowed_none_key_always_passes(monkeypatch): + """_check_custom_key_allowed does nothing when key is None, even if setting is on.""" + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _check_custom_key_allowed, + ) + + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"disable_custom_api_keys": True}) + # Should not raise — None means auto-generate await _check_custom_key_allowed(None) @@ -1726,8 +1788,6 @@ async def test_check_custom_key_allowed_none_key_always_passes(monkeypatch): @pytest.mark.asyncio async def test_get_new_token_rejected_when_custom_keys_disabled(monkeypatch): """get_new_token raises 403 when new_key is set and disable_custom_api_keys is true.""" - from unittest.mock import AsyncMock - from fastapi import HTTPException from litellm.proxy._types import RegenerateKeyRequest @@ -1735,10 +1795,7 @@ async def test_get_new_token_rejected_when_custom_keys_disabled(monkeypatch): get_new_token, ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={"disable_custom_api_keys": True}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"disable_custom_api_keys": True}) data = RegenerateKeyRequest(new_key="sk-custom-regen-key") @@ -1751,17 +1808,12 @@ async def test_get_new_token_rejected_when_custom_keys_disabled(monkeypatch): @pytest.mark.asyncio async def test_get_new_token_auto_generates_when_custom_keys_disabled(monkeypatch): """get_new_token auto-generates a key when new_key is None, even if setting is on.""" - from unittest.mock import AsyncMock - from litellm.proxy._types import RegenerateKeyRequest from litellm.proxy.management_endpoints.key_management_endpoints import ( get_new_token, ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={"disable_custom_api_keys": True}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"disable_custom_api_keys": True}) data = RegenerateKeyRequest() # no new_key result = await get_new_token(data) diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index f40848bc38b..0140fcaba21 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -3998,6 +3998,32 @@ class TestSyncUiSettingsToGeneralSettings: assert general_settings["forward_client_headers_to_llm_api"] is True assert general_settings.source("forward_client_headers_to_llm_api") == "db" + def test_every_runtime_flag_reaches_a_reader_once_applied(self, monkeypatch): + """A flag the settings rules do not route to the ui_settings row is stored but never read back.""" + from litellm.proxy import proxy_server + from litellm.proxy.config_resolvers import SettingsStore + from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( + TEAM_ADMIN_EDITABLE_TEAM_FIELDS_SETTING, + _RUNTIME_GENERAL_SETTINGS_FLAGS, + apply_runtime_general_settings_flags, + ) + + general_settings = SettingsStore("general_settings") + general_settings.load_yaml({}) + monkeypatch.setattr(proxy_server, "general_settings", general_settings) + + stored = { + key: (["tpm_limit"] if key == TEAM_ADMIN_EDITABLE_TEAM_FIELDS_SETTING else True) + for key in _RUNTIME_GENERAL_SETTINGS_FLAGS + } + assert stored + + apply_runtime_general_settings_flags(stored) + + read_back = {key: general_settings.get(key) for key in stored} + + assert read_back == stored + def test_applied_runtime_flags_cannot_override_the_config_file(self, monkeypatch): from litellm.proxy import proxy_server from litellm.proxy.config_resolvers import SettingsStore From fc0055497c3e603957fd7be7ef1f3b5994bfd048 Mon Sep 17 00:00:00 2001 From: togear Date: Wed, 23 Sep 2026 04:31:33 +0800 Subject: [PATCH 34/48] feat: add configurable provider affinity header mapping (#41033) * feat: add configurable provider affinity header mapping * fix: sync provider affinity API types * fix: harden provider affinity header mapping * fix: avoid provider affinity import cycle * fix: preserve input callback header mutations * fix: address provider affinity code scanning findings * fix: satisfy provider affinity type discipline gate * test: cover omitted pre-call argument isolation * fix: resolve remaining provider affinity codeql alerts * fix: redact provider affinity headers after calls * refactor: drop provider affinity header log redaction * fix: reject control characters in affinity session ids as a bad request * chore: regenerate the openapi snapshot on python 3.12 and reuse the session marker constant * fix(responses): read the affinity session from the named metadata argument --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../litellm_core_utils/get_litellm_params.py | 6 +- .../litellm_core_utils/provider_affinity.py | 98 +++++ litellm/llms/custom_httpx/http_handler.py | 2 +- litellm/llms/custom_httpx/llm_http_handler.py | 2 +- litellm/main.py | 28 +- litellm/responses/main.py | 31 +- litellm/types/router.py | 10 + litellm/types/utils.py | 1 + .../test_provider_affinity.py | 107 +++++ .../test_provider_affinity_forwarding.py | 389 ++++++++++++++++++ .../test_responses_api_bridge_flag.py | 24 ++ tests/test_litellm/types/test_router.py | 46 ++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 + 13 files changed, 734 insertions(+), 14 deletions(-) create mode 100644 litellm/litellm_core_utils/provider_affinity.py create mode 100644 tests/test_litellm/litellm_core_utils/test_provider_affinity.py create mode 100644 tests/test_litellm/llms/openai_like/test_provider_affinity_forwarding.py diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 7f373569b21..112d038039e 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -24,10 +24,7 @@ AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset( } ) -# Keys `completion()` forwards from its own kwargs into `get_litellm_params`, -# which are otherwise invisible to it because that call site passes explicit -# named arguments rather than `**kwargs`. -FORWARDED_KWARGS_KEYS: Final = AWS_CREDENTIAL_KWARGS_KEYS +PROVIDER_AFFINITY_HEADER_KWARG_KEY: Final = "provider_affinity_header" # Pre-define optional kwargs keys as frozenset for O(1) lookups # These are extracted from kwargs only if present, avoiding unnecessary .get() calls @@ -63,6 +60,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "itpm", "otpm", "use_xai_oauth", + PROVIDER_AFFINITY_HEADER_KWARG_KEY, } ) | AWS_CREDENTIAL_KWARGS_KEYS diff --git a/litellm/litellm_core_utils/provider_affinity.py b/litellm/litellm_core_utils/provider_affinity.py new file mode 100644 index 00000000000..31cd9a7ff69 --- /dev/null +++ b/litellm/litellm_core_utils/provider_affinity.py @@ -0,0 +1,98 @@ +import re +from collections.abc import Mapping +from typing import Final + +from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY + +_HTTP_HEADER_NAME_PATTERN: Final = re.compile(r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$") +_FORBIDDEN_AFFINITY_HEADERS: Final = frozenset( + { + "api-key", + "authorization", + "connection", + "content-length", + "content-type", + "cookie", + "host", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "proxy-connection", + "set-cookie", + "te", + "trailer", + "transfer-encoding", + "upgrade", + "www-authenticate", + "x-api-key", + "x-goog-api-key", + } +) + + +def validate_provider_affinity_header_name(header: str) -> str: + if not _HTTP_HEADER_NAME_PATTERN.fullmatch(header): + raise ValueError("provider_affinity_header must be a valid HTTP header name") + if header.lower() in _FORBIDDEN_AFFINITY_HEADERS: + raise ValueError("provider_affinity_header cannot be an authentication, cookie, or transport header") + return header + + +def _get_value(value: object, key: str) -> object | None: + if isinstance(value, Mapping): + return value.get(key) + return getattr(value, key, None) + + +def _get_provider_affinity_header_name(litellm_params: object | None) -> str | None: + header: Final = _get_value(litellm_params, "provider_affinity_header") if litellm_params is not None else None + if header is None: + return None + if not isinstance(header, str): + raise TypeError("provider_affinity_header must be a string") + return validate_provider_affinity_header_name(header) + + +def get_stable_session_id(litellm_params: object | None) -> str | None: + if litellm_params is None: + return None + + direct_session_id: Final = _get_value(litellm_params, "session_id") + if direct_session_id: + return str(direct_session_id) + + metadata_values: Final[tuple[object, ...]] = tuple( + value for key in ("metadata", "litellm_metadata") if (value := _get_value(litellm_params, key)) is not None + ) + has_generated_session_id: Final = any( + isinstance(metadata, Mapping) and metadata.get(SESSION_ID_GENERATED_METADATA_KEY) + for metadata in metadata_values + ) + + litellm_session_id: Final = _get_value(litellm_params, "litellm_session_id") + if litellm_session_id and not has_generated_session_id: + return str(litellm_session_id) + + for metadata in metadata_values: + if ( + isinstance(metadata, Mapping) + and not metadata.get(SESSION_ID_GENERATED_METADATA_KEY) + and (value := metadata.get("session_id")) + ): + return str(value) + return None + + +def add_provider_affinity_header( # mutable-ok: downstream handlers add auth and signing headers + headers: Mapping[str, object], litellm_params: object | None +) -> dict[str, object]: # mutable-ok: downstream handlers add auth and signing headers + header_name: Final = _get_provider_affinity_header_name(litellm_params) + if header_name is None or any(key.lower() == header_name.lower() for key in headers): + return dict(headers) # mutable-ok: downstream handlers add auth and signing headers + + session_id: Final = get_stable_session_id(litellm_params) + if session_id is None: + return dict(headers) # mutable-ok: downstream handlers add auth and signing headers + if any(character in session_id for character in ("\r", "\n", "\0")): + raise ValueError("session_id cannot contain HTTP header control characters") + return {**headers, header_name: session_id} # mutable-ok: downstream handlers add auth and signing headers diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 8ea46a6b261..312f46bb021 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -1711,7 +1711,7 @@ class HTTPHandler: def get_async_httpx_client( - llm_provider: LlmProviders | httpxSpecialProvider, + llm_provider: LlmProviders | httpxSpecialProvider | str, params: dict | None = None, shared_session: Optional["ClientSession"] = None, ) -> AsyncHTTPHandler: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index cbef44ce488..8cbc28362a8 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2857,7 +2857,7 @@ class BaseLLMHTTPHandler: id(shared_session) if shared_session else None, ) async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, shared_session=shared_session, ) diff --git a/litellm/main.py b/litellm/main.py index 7d231a1bf7a..7f4b34d28a0 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -78,8 +78,9 @@ from litellm.litellm_core_utils.chat_completion_agentic_loop import ( from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_litellm_params import ( - FORWARDED_KWARGS_KEYS, + AWS_CREDENTIAL_KWARGS_KEYS, OPTIONAL_KWARGS_KEYS, + PROVIDER_AFFINITY_HEADER_KWARG_KEY, ) from litellm.litellm_core_utils.get_provider_specific_headers import ( ProviderSpecificHeaderUtils, @@ -96,6 +97,7 @@ from litellm.litellm_core_utils.mock_functions import ( from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, ) +from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header from litellm.litellm_core_utils.request_timeout_resolver import ( get_configured_request_timeout, ) @@ -5644,8 +5646,30 @@ def completion( gigachat_scope=kwargs.get("gigachat_scope"), gigachat_auth_url=kwargs.get("gigachat_auth_url"), gigachat_access_token=kwargs.get("gigachat_access_token"), - **{key: kwargs[key] for key in FORWARDED_KWARGS_KEYS if key in kwargs}, + **{ + key: kwargs[key] + for key in (*AWS_CREDENTIAL_KWARGS_KEYS, PROVIDER_AFFINITY_HEADER_KWARG_KEY) + if key in kwargs + }, ) + if litellm_params.get("provider_affinity_header") is not None: + try: + headers = add_provider_affinity_header( + headers=headers or litellm.headers or MappingProxyType({}), + litellm_params=MappingProxyType( + { + "provider_affinity_header": litellm_params["provider_affinity_header"], + "litellm_session_id": kwargs.get("litellm_session_id"), + "session_id": kwargs.get("session_id"), + "metadata": metadata, + "litellm_metadata": kwargs.get("litellm_metadata"), + } + ), + ) + except ValueError as affinity_error: + raise litellm.BadRequestError( + message=str(affinity_error), model=model, llm_provider=custom_llm_provider + ) from affinity_error cast(LiteLLMLoggingObj, logging).update_environment_variables( model=model, user=user, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index c5032536df4..5c4c9ea3987 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -26,6 +26,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( update_responses_input_with_model_file_ids, update_responses_tools_with_model_file_ids, ) +from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig @@ -1218,6 +1219,27 @@ def responses( # get llm provider logic litellm_params: Final = GenericLiteLLMParams(**kwargs) + try: + effective_extra_headers: Final = ( + add_provider_affinity_header( + headers=extra_headers or MappingProxyType({}), + litellm_params=MappingProxyType( + { + "provider_affinity_header": litellm_params.provider_affinity_header, + "litellm_session_id": kwargs.get("litellm_session_id"), + "session_id": kwargs.get("session_id"), + "metadata": metadata, + "litellm_metadata": kwargs.get("litellm_metadata"), + } + ), + ) + if litellm_params.provider_affinity_header is not None + else extra_headers + ) + except ValueError as affinity_error: + raise litellm.BadRequestError( + message=str(affinity_error), model=model, llm_provider=custom_llm_provider + ) from affinity_error ######################################################### # MOCK RESPONSE LOGIC @@ -1261,7 +1283,7 @@ def responses( top_p=top_p, truncation=truncation, user=user, - extra_headers=extra_headers, + extra_headers=effective_extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout, @@ -1332,7 +1354,7 @@ def responses( safety_identifier=safety_identifier, text_format=text_format, allowed_openai_params=allowed_openai_params, - extra_headers=extra_headers, + extra_headers=effective_extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout, @@ -1352,7 +1374,7 @@ def responses( custom_llm_provider=custom_llm_provider, _is_async=_is_async, stream=stream, - extra_headers=extra_headers, + extra_headers=effective_extra_headers, extra_body=extra_body, timeout=timeout if timeout is not None else request_timeout, allowed_openai_params=allowed_openai_params, @@ -1381,6 +1403,7 @@ def responses( "model_info": kwargs.get("model_info"), "data_residency": infer_openai_data_residency(custom_llm_provider, litellm_params.api_base), "metadata": (kwargs["litellm_metadata"] if "litellm_metadata" in kwargs else kwargs.get("metadata")), + "provider_affinity_header": litellm_params.provider_affinity_header, }, custom_llm_provider=custom_llm_provider, ) @@ -1400,7 +1423,7 @@ def responses( custom_llm_provider=custom_llm_provider, litellm_params=litellm_params, logging_obj=litellm_logging_obj, - extra_headers=extra_headers, + extra_headers=effective_extra_headers, extra_body=extra_body, timeout=timeout or request_timeout, _is_async=_is_async, diff --git a/litellm/types/router.py b/litellm/types/router.py index 2353093d435..57bd4263894 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -16,6 +16,7 @@ from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_c from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.provider_affinity import validate_provider_affinity_header_name from litellm.types.router_weights import RouterWeights if TYPE_CHECKING: @@ -396,6 +397,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): organization: str | None = None # for openai orgs configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None litellm_credential_name: str | None = None + provider_affinity_header: str | None = None ## LOGGING PARAMS ## litellm_trace_id: str | None = None @@ -467,6 +469,13 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): valkey_text_field: str | None = None valkey_embedding_field: str | None = None + @field_validator("provider_affinity_header") + @classmethod + def validate_provider_affinity_header(cls, value: str | None) -> str | None: + if value is None: + return None + return validate_provider_affinity_header_name(value) + @model_validator(mode="before") @classmethod def preprocess_input_data(cls, data: object) -> object: @@ -569,6 +578,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): stream_timeout: float | str | None max_retries: int | None organization: list | str | None # for openai orgs + provider_affinity_header: ReadOnly[str | None] configurable_clientside_auth_params: ( CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS # for allowing api base switching on finetuned models ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index f10f102d8ec..f2c8f0e7044 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4073,6 +4073,7 @@ all_litellm_params = ( "litellm_credential_name", "allowed_openai_params", "litellm_session_id", + "provider_affinity_header", "use_litellm_proxy", "use_chat_completions_api", "rust", diff --git a/tests/test_litellm/litellm_core_utils/test_provider_affinity.py b/tests/test_litellm/litellm_core_utils/test_provider_affinity.py new file mode 100644 index 00000000000..edf4f5169b5 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_provider_affinity.py @@ -0,0 +1,107 @@ +import pytest + +from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY +from litellm.litellm_core_utils.provider_affinity import ( + add_provider_affinity_header, + get_stable_session_id, +) + + +@pytest.mark.parametrize( + ("litellm_params", "expected"), + [ + ({"litellm_session_id": "litellm-session"}, "litellm-session"), + ({"session_id": "direct-session"}, "direct-session"), + ({"metadata": {"session_id": "metadata-session"}}, "metadata-session"), + ({"litellm_metadata": {"session_id": "litellm-metadata-session"}}, "litellm-metadata-session"), + ], +) +def test_get_stable_session_id_uses_explicit_session_sources(litellm_params: dict, expected: str): + assert get_stable_session_id(litellm_params) == expected + + +def test_get_stable_session_id_does_not_use_trace_id(): + assert get_stable_session_id({"litellm_trace_id": "per-request-trace"}) is None + + +@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) +def test_get_stable_session_id_ignores_proxy_generated_session(metadata_key: str): + assert ( + get_stable_session_id( + { + "litellm_session_id": "generated-session", + metadata_key: { + "session_id": "generated-session", + SESSION_ID_GENERATED_METADATA_KEY: True, + }, + } + ) + is None + ) + + +def test_get_stable_session_id_prefers_explicit_session_over_proxy_generated_session(): + assert ( + get_stable_session_id( + { + "session_id": "explicit-session", + "litellm_session_id": "generated-session", + "metadata": { + "session_id": "generated-session", + SESSION_ID_GENERATED_METADATA_KEY: True, + }, + } + ) + == "explicit-session" + ) + + +def test_add_provider_affinity_header_maps_session_id(): + headers = add_provider_affinity_header( + headers={"Content-Type": "application/json"}, + litellm_params={ + "litellm_session_id": "session-123", + "provider_affinity_header": "X-Conversation-Id", + }, + ) + + assert headers == { + "Content-Type": "application/json", + "X-Conversation-Id": "session-123", + } + + +def test_add_provider_affinity_header_preserves_explicit_header_case_insensitively(): + headers = add_provider_affinity_header( + headers={"x-conversation-id": "explicit-session"}, + litellm_params={ + "litellm_session_id": "session-123", + "provider_affinity_header": "X-Conversation-Id", + }, + ) + + assert headers == {"x-conversation-id": "explicit-session"} + + +@pytest.mark.parametrize("session_id", ["session\r", "session\n", "session\0"]) +def test_add_provider_affinity_header_rejects_control_characters(session_id: str): + with pytest.raises(ValueError, match="session_id cannot contain HTTP header control characters"): + add_provider_affinity_header( + headers={}, + litellm_params={ + "litellm_session_id": session_id, + "provider_affinity_header": "X-Conversation-Id", + }, + ) + + +def test_add_provider_affinity_header_does_nothing_without_config_or_session(): + assert add_provider_affinity_header({}, {"litellm_session_id": "session-123"}) == {} + assert ( + add_provider_affinity_header( + {}, + {"provider_affinity_header": "X-Conversation-Id"}, + ) + == {} + ) + diff --git a/tests/test_litellm/llms/openai_like/test_provider_affinity_forwarding.py b/tests/test_litellm/llms/openai_like/test_provider_affinity_forwarding.py new file mode 100644 index 00000000000..43101234e63 --- /dev/null +++ b/tests/test_litellm/llms/openai_like/test_provider_affinity_forwarding.py @@ -0,0 +1,389 @@ +import json +from unittest.mock import MagicMock + +import httpx +import pytest + +import litellm +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.openai_like import dynamic_config +from litellm.llms.openai_like.json_loader import JSONProviderRegistry, SimpleProviderConfig + + +def _provider(*, responses: bool = False) -> SimpleProviderConfig: + endpoints = ["/v1/chat/completions"] + if responses: + endpoints.append("/v1/responses") + return SimpleProviderConfig( + "db_only_provider", + { + "base_url": "https://db-only.example/v1", + "api_key_env": "DYNAMIC_PROVIDER_API_KEY", + "supported_endpoints": endpoints, + }, + ) + + +def _chat_response_payload(content: str = "dynamic response") -> dict[str, object]: + return { + "id": "chatcmpl-dynamic-provider", + "object": "chat.completion", + "created": 1234567890, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": content}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 2, "total_tokens": 4}, + } + + +def _responses_payload() -> dict[str, object]: + return { + "id": "resp_dynamic_provider", + "object": "response", + "created_at": 1234567890, + "status": "completed", + "model": "test-model", + "output": [], + "parallel_tool_calls": True, + "usage": {"input_tokens": 2, "output_tokens": 2, "total_tokens": 4}, + "error": None, + } + + +@pytest.fixture(autouse=True) +def _isolate_registry_state(): + original_providers = dict(JSONProviderRegistry._providers) + dynamic_config._responses_config_cache.clear() + yield + JSONProviderRegistry._providers = original_providers + dynamic_config._responses_config_cache.clear() + + +def test_dynamic_provider_receives_affinity_header_for_chat(): + from openai import OpenAI + + JSONProviderRegistry._providers = {"db_only_provider": _provider()} + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_chat_response_payload()) + + client = OpenAI( + api_key="test-key", + base_url="https://db-only.example/v1", + http_client=httpx.Client(transport=httpx.MockTransport(respond)), + ) + try: + litellm.completion( + model="db_only_provider/test-model", + messages=[{"role": "user", "content": "hello"}], + api_key="test-key", + client=client, + extra_headers={"X-Customer-Header": "customer-value"}, + litellm_session_id="session-sync", + provider_affinity_header="X-Conversation-Id", + ) + finally: + client.close() + + assert requests[0].headers["x-conversation-id"] == "session-sync" + assert requests[0].headers["x-customer-header"] == "customer-value" + + +def test_dynamic_provider_does_not_use_trace_id_for_chat_affinity(): + from openai import OpenAI + + JSONProviderRegistry._providers = {"db_only_provider": _provider()} + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_chat_response_payload()) + + client = OpenAI( + api_key="test-key", + base_url="https://db-only.example/v1", + http_client=httpx.Client(transport=httpx.MockTransport(respond)), + ) + try: + litellm.completion( + model="db_only_provider/test-model", + messages=[{"role": "user", "content": "hello"}], + api_key="test-key", + client=client, + metadata={"trace_id": "per-request-trace"}, + provider_affinity_header="X-Conversation-Id", + ) + finally: + client.close() + + assert "x-conversation-id" not in requests[0].headers + + +@pytest.mark.asyncio +async def test_dynamic_provider_receives_affinity_header_for_async_chat(): + from openai import AsyncOpenAI + + JSONProviderRegistry._providers = {"db_only_provider": _provider()} + requests: list[httpx.Request] = [] + + async def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_chat_response_payload("async response")) + + client = AsyncOpenAI( + api_key="test-key", + base_url="https://db-only.example/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(respond)), + ) + try: + response = await litellm.acompletion( + model="db_only_provider/test-model", + messages=[{"role": "user", "content": "hello"}], + api_key="test-key", + client=client, + litellm_session_id="session-async", + provider_affinity_header="X-Conversation-Id", + ) + finally: + await client.close() + + assert response.choices[0].message.content == "async response" + assert requests[0].headers["x-conversation-id"] == "session-async" + + +def test_dynamic_provider_receives_affinity_header_for_streaming_chat(): + from openai import OpenAI + + JSONProviderRegistry._providers = {"db_only_provider": _provider()} + chunks = [ + { + "id": "chatcmpl-dynamic-stream", + "object": "chat.completion.chunk", + "created": 1234567890, + "model": "test-model", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "streamed"}, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-dynamic-stream", + "object": "chat.completion.chunk", + "created": 1234567890, + "model": "test-model", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + ] + stream_body = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n" + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, content=stream_body, headers={"content-type": "text/event-stream"}) + + client = OpenAI( + api_key="test-key", + base_url="https://db-only.example/v1", + http_client=httpx.Client(transport=httpx.MockTransport(respond)), + ) + try: + response_chunks = list( + litellm.completion( + model="db_only_provider/test-model", + messages=[{"role": "user", "content": "hello"}], + api_key="test-key", + client=client, + stream=True, + litellm_session_id="session-stream", + provider_affinity_header="X-Conversation-Id", + ) + ) + finally: + client.close() + + assert any(chunk.choices[0].delta.content == "streamed" for chunk in response_chunks) + assert requests[0].headers["x-conversation-id"] == "session-stream" + + +def test_dynamic_provider_receives_affinity_header_for_responses(): + JSONProviderRegistry._providers = {"db_only_provider": _provider(responses=True)} + logging_obj = MagicMock() + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_responses_payload()) + + http_client = httpx.Client(transport=httpx.MockTransport(respond)) + try: + litellm.responses( + model="db_only_provider/test-model", + input="hello", + api_key="test-key", + litellm_session_id="session-responses", + provider_affinity_header="X-Conversation-Id", + litellm_logging_obj=logging_obj, + client=HTTPHandler(client=http_client), + ) + finally: + http_client.close() + + assert requests[0].headers["X-Conversation-Id"] == "session-responses" + assert ( + logging_obj.update_from_kwargs.call_args.kwargs["litellm_params"]["provider_affinity_header"] + == "X-Conversation-Id" + ) + + +def test_dynamic_provider_uses_metadata_session_id_for_responses(): + JSONProviderRegistry._providers = {"db_only_provider": _provider(responses=True)} + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_responses_payload()) + + http_client = httpx.Client(transport=httpx.MockTransport(respond)) + try: + litellm.responses( + model="db_only_provider/test-model", + input="hello", + api_key="test-key", + metadata={"session_id": "session-from-metadata"}, + provider_affinity_header="X-Conversation-Id", + litellm_logging_obj=MagicMock(), + client=HTTPHandler(client=http_client), + ) + finally: + http_client.close() + + assert requests[0].headers["X-Conversation-Id"] == "session-from-metadata" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_dynamic_provider_receives_affinity_header_for_async_responses(stream: bool): + JSONProviderRegistry._providers = {"db_only_provider": _provider(responses=True)} + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_responses_payload()) + + client = AsyncHTTPHandler() + await client.close() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(respond)) + try: + response = await litellm.aresponses( + model="db_only_provider/test-model", + input="hello", + api_key="test-key", + stream=stream, + litellm_session_id="session-async-responses", + provider_affinity_header="X-Conversation-Id", + client=client, + ) + finally: + await client.client.aclose() + + assert requests[0].headers["X-Conversation-Id"] == "session-async-responses" + if stream: + assert hasattr(response, "__aiter__") + else: + assert getattr(response, "model", None) == "test-model" + + +def test_control_characters_in_session_id_are_a_bad_request_for_chat(): + from openai import OpenAI + + JSONProviderRegistry._providers = {"db_only_provider": _provider()} + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_chat_response_payload()) + + client = OpenAI( + api_key="test-key", + base_url="https://db-only.example/v1", + http_client=httpx.Client(transport=httpx.MockTransport(respond)), + ) + try: + with pytest.raises(litellm.BadRequestError, match="HTTP header control characters"): + litellm.completion( + model="db_only_provider/test-model", + messages=[{"role": "user", "content": "hello"}], + api_key="test-key", + client=client, + litellm_session_id="session\nsplit", + provider_affinity_header="X-Conversation-Id", + ) + finally: + client.close() + + assert requests == [] + + +def test_control_characters_in_session_id_are_a_bad_request_for_responses(): + JSONProviderRegistry._providers = {"db_only_provider": _provider(responses=True)} + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_responses_payload()) + + http_client = httpx.Client(transport=httpx.MockTransport(respond)) + try: + with pytest.raises(litellm.BadRequestError, match="HTTP header control characters"): + litellm.responses( + model="db_only_provider/test-model", + input="hello", + api_key="test-key", + litellm_session_id="session\nsplit", + provider_affinity_header="X-Conversation-Id", + litellm_logging_obj=MagicMock(), + client=HTTPHandler(client=http_client), + ) + finally: + http_client.close() + + assert requests == [] + + +def test_builtin_provider_receives_affinity_header(): + from openai import OpenAI + + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_chat_response_payload()) + + client = OpenAI( + api_key="test-key", + base_url="https://api.openai.com/v1", + http_client=httpx.Client(transport=httpx.MockTransport(respond)), + ) + try: + litellm.completion( + model="openai/test-model", + messages=[{"role": "user", "content": "hello"}], + api_key="test-key", + client=client, + litellm_session_id="session-builtin", + provider_affinity_header="X-Conversation-Id", + ) + finally: + client.close() + + assert requests[0].headers["X-Conversation-Id"] == "session-builtin" diff --git a/tests/test_litellm/responses/test_responses_api_bridge_flag.py b/tests/test_litellm/responses/test_responses_api_bridge_flag.py index 16135106b41..642495fab86 100644 --- a/tests/test_litellm/responses/test_responses_api_bridge_flag.py +++ b/tests/test_litellm/responses/test_responses_api_bridge_flag.py @@ -66,6 +66,30 @@ class TestUseResponsesApiBridgeFlag: mock_bridge_handler.assert_called_once() + @patch.object( + import_module("litellm.responses.main").litellm_completion_transformation_handler, "response_api_handler" + ) + @patch.object( + import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config" + ) + def test_provider_affinity_header_is_forwarded_through_bridge(self, mock_get_config, mock_bridge_handler): + mock_get_config.return_value = litellm.OpenAIResponsesAPIConfig() + mock_bridge_handler.return_value = MagicMock() + + litellm.responses( + model="openai/my-custom-model", + input="Hello", + use_chat_completions_api=True, + litellm_session_id="session-bridge", + provider_affinity_header="X-Conversation-Id", + extra_headers={"X-Customer-Header": "customer-value"}, + litellm_logging_obj=MagicMock(), + ) + + forwarded_headers = mock_bridge_handler.call_args.kwargs["extra_headers"] + assert forwarded_headers["X-Conversation-Id"] == "session-bridge" + assert forwarded_headers["X-Customer-Header"] == "customer-value" + @patch.object( import_module("litellm.responses.main").litellm_completion_transformation_handler, "response_api_handler" ) diff --git a/tests/test_litellm/types/test_router.py b/tests/test_litellm/types/test_router.py index df744a77fe4..4d4c326d1ca 100644 --- a/tests/test_litellm/types/test_router.py +++ b/tests/test_litellm/types/test_router.py @@ -91,7 +91,7 @@ def test_pricing_strings_are_coerced_to_float(): def test_invalid_pricing_is_rejected(): - with pytest.raises(ValueError, match='validation error for ModelInfo'): + with pytest.raises(ValueError, match="validation error for ModelInfo"): ModelInfo(id="x", input_cost_per_token="free") @@ -118,7 +118,9 @@ def test_drop_params_ignores_non_flag_non_string_values_with_a_warning(value, ca assert f"drop_params={value!r} is not a flag value" in caplog.text -@pytest.mark.parametrize("value", [True, "true", None, "os.environ/DROP_PARAMS", "v2:gcm:ciphertext-from-a-pre-fix-row"]) +@pytest.mark.parametrize( + "value", [True, "true", None, "os.environ/DROP_PARAMS", "v2:gcm:ciphertext-from-a-pre-fix-row"] +) def test_drop_params_flags_and_strings_log_nothing(value, caplog): with caplog.at_level(logging.WARNING, logger="LiteLLM"): GenericLiteLLMParams(drop_params=value) @@ -148,6 +150,46 @@ def test_aws_session_tags_reject_shapes_sts_would_refuse(aws_session_tags): LiteLLM_Params(model="bedrock/anthropic.claude-opus-5", aws_session_tags=aws_session_tags) +def test_provider_affinity_header_is_normalized(): + params = LiteLLM_Params( + model="openai/gpt-4o-mini", + provider_affinity_header="X-Conversation-Id", + ) + + assert params.provider_affinity_header == "X-Conversation-Id" + assert params.model_dump(exclude_none=True)["provider_affinity_header"] == "X-Conversation-Id" + + +@pytest.mark.parametrize( + "header", + [ + "Authorization", + "Proxy-Authorization", + "Cookie", + "Set-Cookie", + "Host", + "Content-Length", + "Content-Type", + "X-API-Key", + ], +) +def test_provider_affinity_header_rejects_sensitive_or_transport_headers(header: str): + with pytest.raises(ValueError, match="provider_affinity_header"): + LiteLLM_Params( + model="openai/gpt-4o-mini", + provider_affinity_header=header, + ) + + +@pytest.mark.parametrize("header", ["", "X Conversation Id", "X-Conversation-Id\r\nInjected: true"]) +def test_provider_affinity_header_rejects_invalid_header_names(header: str): + with pytest.raises(ValueError, match="provider_affinity_header"): + LiteLLM_Params( + model="openai/gpt-4o-mini", + provider_affinity_header=header, + ) + + def test_model_info_parses_access_windows_time_strings(): import datetime diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a53115b7af2..be1e000bd66 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -31476,6 +31476,8 @@ export interface components { output_cost_per_video_token?: number | null; /** Output Vector Size */ output_vector_size?: number | null; + /** Provider Affinity Header */ + provider_affinity_header?: string | null; /** Quality Router Config */ quality_router_config?: { [key: string]: unknown; @@ -42364,6 +42366,8 @@ export interface components { output_cost_per_video_token?: number | null; /** Output Vector Size */ output_vector_size?: number | null; + /** Provider Affinity Header */ + provider_affinity_header?: string | null; /** Quality Router Config */ quality_router_config?: { [key: string]: unknown; From c6c3881d7f17d392856b779e89881b8ed23355d0 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:31:58 -0700 Subject: [PATCH 35/48] fix(proxy): share model rate-limit buckets between a model_group_alias and its target (#42516) * fix(proxy): share model rate-limit buckets between a model_group_alias and its target A request sent under a model_group_alias counted in its own per-key, per-team, per-org, and per-project model bucket, so a key could double a deployment's default_api_key_rpm_limit / tpm_limit by alternating the alias and the model group name, and a metadata model_rpm_limit / model_tpm_limit keyed by the model group never applied to alias requests. The limiter now resolves the requested name to its model group before keying any model bucket, looks the limit up by the requested name first and the model group second, and charges post-call tokens to the same bucket. * fix(proxy): charge the model group resolved at admission when reconciling reserved tokens --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../hooks/parallel_request_limiter_v3.py | 222 +++++++++--------- .../coverage_registry/quota_management.yaml | 1 + tests/e2e/gateway/stage_mirror_ci_config.yml | 7 + .../test_model_group_alias_rate_limit_e2e.py | 85 +++++++ .../hooks/test_parallel_request_limiter_v3.py | 187 ++++++++++++++- .../proxy/hooks/test_tpm_concurrent.py | 3 +- 6 files changed, 389 insertions(+), 116 deletions(-) create mode 100644 tests/e2e/quota_management/ratelimit/test_model_group_alias_rate_limit_e2e.py diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 9d35178891c..33744906b13 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -63,6 +63,7 @@ from litellm.router_utils.add_retry_fallback_headers import ( ensure_response_additional_headers, response_has_hidden_params, ) +from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage from litellm.types.utils import ( @@ -91,6 +92,26 @@ else: _REQUEST_RATE_LIMIT_DATA: Final = TypeAdapter(Mapping[str, object]) +@dataclass(frozen=True, slots=True) +class RateLimitedModel: + requested: str + group: str + + def limit_in(self, limits: Mapping[str, int] | None) -> int | None: + if limits is None: + return None + requested_limit: Final = limits.get(self.requested) + return requested_limit if requested_limit is not None else limits.get(self.group) + + +def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None: + from litellm.proxy.proxy_server import llm_router + + if llm_router is None: + return None + return resolve_model_group_alias(llm_router.model_group_alias, model) + + def _sibling_counter_keys(window_key: str) -> tuple[str, str]: prefix: Final = window_key.removesuffix(":window") return f"{prefix}:requests", f"{prefix}:tokens" @@ -546,7 +567,7 @@ class RequestRateLimiterStash: parallel_slot: ParallelSlotAcquisition | None = None parallel_slot_release_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False, compare=False) reserved_tokens: int = 0 - reserved_model: str | None = None + reserved_model: RateLimitedModel | None = None reserved_scopes: frozenset[tuple[str, str]] = field(default_factory=frozenset) itpm_reserved_tokens: int = 0 itpm_reserved_scopes: frozenset[tuple[str, str]] = field(default_factory=frozenset) @@ -626,9 +647,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): self, internal_usage_cache: InternalUsageCache, time_provider: Callable[[], datetime] | None = None, + model_group_resolver: Callable[[str], str | None] = _resolve_model_group_alias_via_proxy_router, ): self.internal_usage_cache = internal_usage_cache self._time_provider = time_provider or datetime.now + self._model_group_resolver = model_group_resolver if self.internal_usage_cache.dual_cache.redis_cache is not None: self.batch_rate_limiter_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script( BATCH_RATE_LIMITER_SCRIPT @@ -2346,6 +2369,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if response["overall_code"] == "OVER_LIMIT": self._handle_rate_limit_error(response, descriptors, requested_model) + def _rate_limited_model(self, requested_model: str | None) -> RateLimitedModel | None: + if not requested_model: + return None + return RateLimitedModel( + requested=requested_model, + group=self._model_group_resolver(requested_model) or requested_model, + ) + def create_organization_rate_limit_descriptor( self, user_api_key_dict: UserAPIKeyAuth, requested_model: str | None = None ) -> list[RateLimitDescriptor]: @@ -2367,43 +2398,28 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) ) - # Model specific org rate limits - if ( + model: Final = self._rate_limited_model(requested_model) + if model is None: + return descriptors + model_specific_tpm_limit: Final = model.limit_in( + get_model_rate_limit_from_metadata(user_api_key_dict, "organization_metadata", "model_tpm_limit") + ) + model_specific_rpm_limit: Final = model.limit_in( get_model_rate_limit_from_metadata(user_api_key_dict, "organization_metadata", "model_rpm_limit") - is not None - or get_model_rate_limit_from_metadata(user_api_key_dict, "organization_metadata", "model_tpm_limit") - is not None - ): - _tpm_limit_for_team_model: Final = ( - get_model_rate_limit_from_metadata(user_api_key_dict, "organization_metadata", "model_tpm_limit") or {} + ) + if model_specific_tpm_limit is None and model_specific_rpm_limit is None: + return descriptors + descriptors.append( + RateLimitDescriptor( + key="model_per_organization", + value=f"{user_api_key_dict.org_id}:{model.group}", + rate_limit={ + "requests_per_unit": model_specific_rpm_limit, + "tokens_per_unit": model_specific_tpm_limit, + "window_size": self.window_size, + }, ) - _rpm_limit_for_team_model: Final = ( - get_model_rate_limit_from_metadata(user_api_key_dict, "organization_metadata", "model_rpm_limit") or {} - ) - - should_check_rate_limit = False - if requested_model in _tpm_limit_for_team_model or requested_model in _rpm_limit_for_team_model: - should_check_rate_limit = True - - if should_check_rate_limit: - model_specific_tpm_limit = None - model_specific_rpm_limit = None - if requested_model in _tpm_limit_for_team_model: - model_specific_tpm_limit = _tpm_limit_for_team_model[requested_model] - if requested_model in _rpm_limit_for_team_model: - model_specific_rpm_limit = _rpm_limit_for_team_model[requested_model] - descriptors.append( - RateLimitDescriptor( - key="model_per_organization", - value=f"{user_api_key_dict.org_id}:{requested_model}", - rate_limit={ - "requests_per_unit": model_specific_rpm_limit, - "tokens_per_unit": model_specific_tpm_limit, - "window_size": self.window_size, - }, - ) - ) - + ) return descriptors def _add_model_per_key_rate_limit_descriptor( @@ -2425,34 +2441,22 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): get_key_model_tpm_limit, ) - if not requested_model: + model: Final = self._rate_limited_model(requested_model) + if model is None: return - - _tpm_limit_for_key_model = get_key_model_tpm_limit(user_api_key_dict, model_name=requested_model) - _rpm_limit_for_key_model = get_key_model_rpm_limit(user_api_key_dict, model_name=requested_model) - - if _tpm_limit_for_key_model is None and _rpm_limit_for_key_model is None: - return - - _tpm_limit_for_key_model = _tpm_limit_for_key_model or {} - _rpm_limit_for_key_model = _rpm_limit_for_key_model or {} - - # Check if model has any rate limits configured - should_check_rate_limit: Final = ( - requested_model in _tpm_limit_for_key_model or requested_model in _rpm_limit_for_key_model + model_specific_tpm_limit: Final = model.limit_in( + get_key_model_tpm_limit(user_api_key_dict, model_name=model.group) ) - - if not should_check_rate_limit: + model_specific_rpm_limit: Final = model.limit_in( + get_key_model_rpm_limit(user_api_key_dict, model_name=model.group) + ) + if model_specific_tpm_limit is None and model_specific_rpm_limit is None: return - # Get model-specific limits - model_specific_tpm_limit: Final[int | None] = _tpm_limit_for_key_model.get(requested_model) - model_specific_rpm_limit: Final[int | None] = _rpm_limit_for_key_model.get(requested_model) - descriptors.append( RateLimitDescriptor( key="model_per_key", - value=f"{user_api_key_dict.api_key}:{requested_model}", + value=f"{user_api_key_dict.api_key}:{model.group}", rate_limit={ "requests_per_unit": model_specific_rpm_limit, "tokens_per_unit": model_specific_tpm_limit, @@ -2955,32 +2959,30 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): def _key_owns_model_limit( self, user_api_key_dict: UserAPIKeyAuth, - requested_model: str, + model: RateLimitedModel, rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit"], ) -> bool: - key_own_limits: Final = get_key_own_model_rate_limit(user_api_key_dict, rate_limit_key) - return key_own_limits is not None and key_own_limits.get(requested_model) is not None + return model.limit_in(get_key_own_model_rate_limit(user_api_key_dict, rate_limit_key)) is not None def _inherited_team_model_limit( self, user_api_key_dict: UserAPIKeyAuth, - requested_model: str, + model: RateLimitedModel, rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit"], ) -> int | None: - team_limits: Final = get_model_rate_limit_from_metadata(user_api_key_dict, "team_metadata", rate_limit_key) - team_limit: Final = team_limits.get(requested_model) if team_limits else None - if team_limit is None: - return None - if self._key_owns_model_limit(user_api_key_dict, requested_model, rate_limit_key): + team_limit: Final = model.limit_in( + get_model_rate_limit_from_metadata(user_api_key_dict, "team_metadata", rate_limit_key) + ) + if team_limit is None or self._key_owns_model_limit(user_api_key_dict, model, rate_limit_key): return None return team_limit def _key_owns_model_tpm_limit_from_request_metadata( self, request_metadata: Mapping[str, object], - model_group: str | None, + model: RateLimitedModel | None, ) -> bool: - if model_group is None: + if model is None: return False key_view: Final = UserAPIKeyAuth.model_validate( { @@ -2988,7 +2990,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): "model_max_budget": request_metadata.get("user_api_key_model_max_budget") or {}, } ) - return self._key_owns_model_limit(key_view, model_group, "model_tpm_limit") + return self._key_owns_model_limit(key_view, model, "model_tpm_limit") def _add_team_model_rate_limit_descriptor_from_metadata( self, @@ -2996,16 +2998,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): requested_model: str | None, descriptors: list[RateLimitDescriptor], ) -> None: - if requested_model is None: + model: Final = self._rate_limited_model(requested_model) + if model is None: return - team_rpm_limit: Final = self._inherited_team_model_limit(user_api_key_dict, requested_model, "model_rpm_limit") - team_tpm_limit: Final = self._inherited_team_model_limit(user_api_key_dict, requested_model, "model_tpm_limit") + team_rpm_limit: Final = self._inherited_team_model_limit(user_api_key_dict, model, "model_rpm_limit") + team_tpm_limit: Final = self._inherited_team_model_limit(user_api_key_dict, model, "model_tpm_limit") if team_rpm_limit is None and team_tpm_limit is None: return descriptors.append( RateLimitDescriptor( key="model_per_team", - value=f"{user_api_key_dict.team_id}:{requested_model}", + value=f"{user_api_key_dict.team_id}:{model.group}", rate_limit={ "requests_per_unit": team_rpm_limit, "tokens_per_unit": team_tpm_limit, @@ -3021,34 +3024,28 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): descriptors: list[RateLimitDescriptor], ) -> None: """Add project model rate limit descriptor from project_metadata if applicable.""" - if ( - get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_rpm_limit") is not None - or get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_tpm_limit") is not None - ): - _tpm_limit_for_project_model: Final = ( - get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_tpm_limit") or {} + model: Final = self._rate_limited_model(requested_model) + if model is None: + return + model_specific_tpm_limit: Final = model.limit_in( + get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_tpm_limit") + ) + model_specific_rpm_limit: Final = model.limit_in( + get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_rpm_limit") + ) + if model_specific_tpm_limit is None and model_specific_rpm_limit is None: + return + descriptors.append( + RateLimitDescriptor( + key="model_per_project", + value=f"{user_api_key_dict.project_id}:{model.group}", + rate_limit={ + "requests_per_unit": model_specific_rpm_limit, + "tokens_per_unit": model_specific_tpm_limit, + "window_size": self.window_size, + }, ) - _rpm_limit_for_project_model: Final = ( - get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_rpm_limit") or {} - ) - should_check_rate_limit: Final = ( - requested_model in _tpm_limit_for_project_model or requested_model in _rpm_limit_for_project_model - ) - - if should_check_rate_limit and requested_model is not None: - model_specific_tpm_limit: Final = _tpm_limit_for_project_model.get(requested_model) - model_specific_rpm_limit: Final = _rpm_limit_for_project_model.get(requested_model) - descriptors.append( - RateLimitDescriptor( - key="model_per_project", - value=f"{user_api_key_dict.project_id}:{requested_model}", - rate_limit={ - "requests_per_unit": model_specific_rpm_limit, - "tokens_per_unit": model_specific_tpm_limit, - "window_size": self.window_size, - }, - ) - ) + ) def add_project_io_token_rate_limit_descriptors_from_metadata( self, @@ -3062,25 +3059,21 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): TPM descriptor above -- these give Bedrock Mantle-style separate input/output token quotas at the project level. """ - if requested_model is None or user_api_key_dict.project_id is None: + model: Final = self._rate_limited_model(requested_model) + if model is None or user_api_key_dict.project_id is None: return - itpm_limit_for_project_model: Final = ( + model_itpm_limit: Final = model.limit_in( get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_itpm_limit") - or {} # mutable-ok: metadata helper returns an optional mapping ) - otpm_limit_for_project_model: Final = ( + model_otpm_limit: Final = model.limit_in( get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_otpm_limit") - or {} # mutable-ok: metadata helper returns an optional mapping ) - model_itpm_limit: Final = itpm_limit_for_project_model.get(requested_model) - model_otpm_limit: Final = otpm_limit_for_project_model.get(requested_model) - if model_itpm_limit is None and model_otpm_limit is None: return - descriptor_value: Final = f"{user_api_key_dict.project_id}:{requested_model}" + descriptor_value: Final = f"{user_api_key_dict.project_id}:{model.group}" if model_itpm_limit is not None: descriptors.append( RateLimitDescriptor( @@ -3767,7 +3760,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # the (actual - reserved) delta to those — unreserved # scopes get charged the full actual usage instead. stash.reserved_tokens = estimated_tokens - stash.reserved_model = requested_model + stash.reserved_model = self._rate_limited_model(requested_model) stash.reserved_scopes = frozenset( (d["key"], d["value"]) for d in descriptors @@ -4514,9 +4507,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): reserved_scopes: Final[frozenset[tuple[str, str]]] = stash.reserved_scopes if stash is not None else frozenset() # Reconciliation must target the same model-scoped counter that the # pre-call reservation incremented. If a reservation was made, - # ``reserved_model`` is authoritative; otherwise fall back to the - # router's ``model_group`` (covers the no-reservation charge path). - reconcile_model: Final = reserved_model or model_group + # ``reserved_model`` (resolved at admission, so an alias map reload + # mid-flight cannot move the charge) is authoritative; otherwise fall + # back to the router's ``model_group`` (the no-reservation charge path). + reconcile_model: Final = reserved_model if reserved_model is not None else self._rate_limited_model(model_group) pipeline_operations: Final[list[RedisPipelineIncrementOperation]] = [] @@ -4534,7 +4528,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): targets: Final = self._collect_tpm_scope_targets( standard_logging_metadata=standard_logging_metadata, kwargs=kwargs, - model_group=reconcile_model, + model_group=reconcile_model.group if reconcile_model is not None else None, ) charged_targets: Final = ( [target for target in targets if target[0] != "model_per_team"] diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index f6b38243896..8a0edf4a6ca 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -10,6 +10,7 @@ - {id: quota_management.ratelimit.redis_backed.blocks_over_limit, module: quota_management, tier: P0, behavior: ratelimit, variant: redis_backed, assertions: [blocks_over_limit], exercised_on: [chat_completions], source: "parallel_request_limiter_v3.py", rationale: "With Redis configured, RPM still enforces 429 across the shared limiter path customers run multi-replica"} - {id: quota_management.ratelimit.rpm.resets_after_window, module: quota_management, tier: P1, behavior: ratelimit, variant: rpm, assertions: [resets_after_window], exercised_on: [chat_completions], source: "parallel_request_limiter_v3.py", rationale: "Rate-limit window (LITELLM_RATE_LIMIT_WINDOW_SIZE, 60s default) expires; a blocked key serves again in the next window"} - {id: quota_management.ratelimit.rpm.headers_report_remaining, module: quota_management, tier: P1, behavior: ratelimit, variant: rpm, assertions: [headers_report_remaining], exercised_on: [chat_completions], source: "parallel_request_limiter_v3.py async_post_call_success_hook", rationale: "Successful responses carry x-ratelimit-api_key-{limit,remaining}-{requests,tokens} so clients can pace"} +- {id: quota_management.ratelimit.model_group_alias.shares_bucket, module: quota_management, tier: P1, behavior: ratelimit, variant: model_group_alias, assertions: [shares_bucket], exercised_on: [chat_completions], source: "parallel_request_limiter_v3.py:_add_model_per_key_rate_limit_descriptor", rationale: "A model_group_alias draws on the same per-key deployment rpm bucket as the resolved model group, so alias plus real-name traffic cannot exceed the configured limit"} - {id: quota_management.ratelimit.priority_generous.picks_under_tpm, module: quota_management, tier: P1, behavior: ratelimit, variant: priority_generous, assertions: [picks_under_tpm], exercised_on: [chat_completions, messages], source: "dynamic_rate_limiter_v3.py:36-52", rationale: "Generous mode (<80% sat) allows priority borrowing"} - {id: quota_management.ratelimit.priority_strict.picks_under_tpm, module: quota_management, tier: P1, behavior: ratelimit, variant: priority_strict, assertions: [picks_under_tpm], exercised_on: [chat_completions, messages], source: "dynamic_rate_limiter_v3.py:53-71", rationale: "Strict mode (>=80% sat) enforces priority fairness"} - {id: quota_management.budget.key.blocks_over_limit, module: quota_management, tier: P0, behavior: budget, variant: key, assertions: [blocks_over_limit], exercised_on: [chat_completions], source: "proxy/auth/auth_checks.py", rationale: "A key's max_budget blocks further paid calls once spend crosses it"} diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml index 03167692727..02a131f60ae 100644 --- a/tests/e2e/gateway/stage_mirror_ci_config.yml +++ b/tests/e2e/gateway/stage_mirror_ci_config.yml @@ -43,6 +43,8 @@ router_settings: num_retries: 3 allowed_fails: 5 cooldown_time: 30 + model_group_alias: + e2e-alias-rl-alias: e2e-alias-rl-target model_list: - model_name: gpt-5.5 @@ -63,6 +65,11 @@ model_list: litellm_params: model: gemini/gemini-2.5-flash api_key: os.environ/GEMINI_API_KEY + - model_name: e2e-alias-rl-target + litellm_params: + model: anthropic/claude-haiku-4-5 + api_key: os.environ/ANTHROPIC_API_KEY + default_api_key_rpm_limit: 3 - model_name: openai-text-embedding-3-small litellm_params: model: openai/text-embedding-3-small diff --git a/tests/e2e/quota_management/ratelimit/test_model_group_alias_rate_limit_e2e.py b/tests/e2e/quota_management/ratelimit/test_model_group_alias_rate_limit_e2e.py new file mode 100644 index 00000000000..1c3fff47b78 --- /dev/null +++ b/tests/e2e/quota_management/ratelimit/test_model_group_alias_rate_limit_e2e.py @@ -0,0 +1,85 @@ +"""Live e2e: a model group alias must share its per-key deployment rate-limit +bucket with the model group it resolves to. + +Covers quota_management.ratelimit.model_group_alias.shares_bucket: the proxy +config declares `e2e-alias-rl-target` (a cheap Anthropic deployment) with +`default_api_key_rpm_limit: 3` and `router_settings.model_group_alias` mapping +`e2e-alias-rl-alias` -> `e2e-alias-rl-target`. Both spellings must draw on +one per-key rpm bucket, so a key that exhausts the limit on one spelling is +blocked on the other spelling inside the same window; each test in this file +exhausts the budget on one name and asserts the other name 429s. + +All calls of one test must land inside a single window +(LITELLM_RATE_LIMIT_WINDOW_SIZE, 60s default), which real chat latency +comfortably allows. +""" + +from __future__ import annotations + +import time + +import pytest +from e2e_config import unique_marker +from e2e_http import StreamingResponse, require_successful_call +from quota_client import QuotaClient + +pytestmark = pytest.mark.e2e + +MODEL_GROUP = "e2e-alias-rl-target" +MODEL_ALIAS = "e2e-alias-rl-alias" +RPM_LIMIT = 3 +WINDOW_SECONDS = 60 +LAST_CALL_LATENCY_MARGIN_SECONDS = 10 + + +def _chat(client: QuotaClient, key: str, model: str) -> StreamingResponse: + return client.chat(key, model, f"reply with one word {unique_marker()}") + + +def _exhaust_rpm(client: QuotaClient, key: str, model: str) -> float: + """Send RPM_LIMIT successful calls on `model`, opening the rate-limit + window; returns the send timestamp of the first call as a lower bound on + the window start. A fresh key may briefly 401 until the data plane's auth + cache picks it up, so retry on 401 to a deadline; a 401 never reaches the + rate limiter.""" + deadline = time.monotonic() + client.proxy.poll_timeout + first_sent_at: float | None = None + sent = 0 + while sent < RPM_LIMIT: + if first_sent_at is None: + first_sent_at = time.monotonic() + outcome = _chat(client, key, model) + if outcome.status_code == 401 and time.monotonic() < deadline: + time.sleep(client.proxy.poll_interval) + continue + require_successful_call(outcome) + sent += 1 + assert first_sent_at is not None + return first_sent_at + + +def _assert_blocked_inside_window( + client: QuotaClient, key: str, model: str, window_opened_at: float +) -> StreamingResponse: + assert time.monotonic() < window_opened_at + WINDOW_SECONDS - LAST_CALL_LATENCY_MARGIN_SECONDS, ( + f"the {RPM_LIMIT} exhaust calls took too long; the follow-up call could land in the " + "next window and mask a shared-bucket regression" + ) + outcome = _chat(client, key, model) + assert outcome.status_code == 429, ( + f"{model} must share its rpm bucket with the model group/alias that was already " + f"exhausted, expected a 429 but got {outcome.status_code}: {outcome.body[:300]}" + ) + return outcome + + +class TestModelGroupAliasRateLimit: + @pytest.mark.covers("quota_management.ratelimit.model_group_alias.shares_bucket") + def test_alias_shares_rpm_bucket_with_model_group(self, client: QuotaClient, scoped_key: str) -> None: + opened_at = _exhaust_rpm(client, scoped_key, MODEL_GROUP) + _assert_blocked_inside_window(client, scoped_key, MODEL_ALIAS, opened_at) + + @pytest.mark.covers("quota_management.ratelimit.model_group_alias.shares_bucket") + def test_model_group_shares_rpm_bucket_with_alias(self, client: QuotaClient, scoped_key: str) -> None: + opened_at = _exhaust_rpm(client, scoped_key, MODEL_ALIAS) + _assert_blocked_inside_window(client, scoped_key, MODEL_GROUP, opened_at) diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 0f19675edb9..8c0dcd3383c 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -26,6 +26,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PARALLEL_REQUEST_SLOT_TTL_SECONDS, ParallelSlotAcquisition, RateLimitDescriptor, + RateLimitedModel, RateLimitResponse, RequestRateLimiterStash, _request_stash, @@ -3367,7 +3368,7 @@ async def test_pre_call_hook_keeps_internal_stash_out_of_request_body(): stash = get_request_stash() assert stash is not None assert stash.reserved_tokens > 0 - assert stash.reserved_model == "gpt-4o-mini" + assert stash.reserved_model == RateLimitedModel(requested="gpt-4o-mini", group="gpt-4o-mini") assert stash.reserved_scopes == frozenset({("api_key", _api_key)}) @@ -6965,3 +6966,187 @@ def test_rate_limit_error_reports_reset_time_in_utc_on_a_non_utc_proxy(process_t "Rate limit exceeded for api_key: sk-test. Limit type: requests. " f"Current limit: 2, Remaining: 0. Limit resets at: {expected_reset}" ) + + +def _resolve_alias_to_target(model: str) -> str | None: + return "target" if model == "alias" else None + + +async def _rpm_request(handler: _PROXY_MaxParallelRequestsHandler, cache: DualCache, auth: UserAPIKeyAuth, model: str) -> None: + await handler.async_pre_call_hook(user_api_key_dict=auth, cache=cache, data={"model": model}, call_type="acompletion") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("first_name, second_name", [("target", "alias"), ("alias", "target")]) +async def test_model_group_alias_shares_deployment_default_rpm_bucket_with_its_target( + monkeypatch: pytest.MonkeyPatch, first_name: str, second_name: str +) -> None: + import litellm.proxy.proxy_server as proxy_server + + router: Final = Router( + model_list=[ + { + "model_name": "target", + "litellm_params": {"model": "openai/gpt-test", "api_key": "test-key", "default_api_key_rpm_limit": 2}, + "model_info": {"id": "target-deployment"}, + } + ], + model_group_alias={"alias": "target"}, + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache)) + auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-alias-default")) + + await _rpm_request(handler, cache, auth, first_name) + await _rpm_request(handler, cache, auth, first_name) + with pytest.raises(HTTPException) as exc: + await _rpm_request(handler, cache, auth, second_name) + + assert exc.value.status_code == 429 + assert "model_per_key" in str(exc.value.detail) + assert f"{auth.api_key}:target" in str(exc.value.detail) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("first_name, second_name", [("target", "alias"), ("alias", "target")]) +@pytest.mark.parametrize( + "limits, counter_scope", + [ + ({"metadata": {"model_rpm_limit": {"target": 1}}}, "model_per_key"), + ( + { + "team_id": "t", + "metadata": {"model_rpm_limit": {"other-model": 100}}, + "team_metadata": {"model_rpm_limit": {"target": 1}}, + }, + "model_per_team", + ), + ({"org_id": "o", "organization_metadata": {"model_rpm_limit": {"target": 1}}}, "model_per_organization"), + ({"project_id": "p", "project_metadata": {"model_rpm_limit": {"target": 1}}}, "model_per_project"), + ], + ids=["key_metadata", "team_metadata", "organization_metadata", "project_metadata"], +) +async def test_model_group_alias_shares_metadata_model_rpm_bucket_with_its_target( + limits: dict[str, object], counter_scope: str, first_name: str, second_name: str +) -> None: + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(cache), model_group_resolver=_resolve_alias_to_target + ) + auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-alias-metadata"), **limits) + + await _rpm_request(handler, cache, auth, first_name) + with pytest.raises(HTTPException) as exc: + await _rpm_request(handler, cache, auth, second_name) + + assert exc.value.status_code == 429 + assert counter_scope in str(exc.value.detail) + assert ":target" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_model_rpm_limit_keyed_by_the_alias_name_still_limits_alias_requests_only() -> None: + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(cache), model_group_resolver=_resolve_alias_to_target + ) + auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-alias-keyed"), metadata={"model_rpm_limit": {"alias": 1}}) + + await _rpm_request(handler, cache, auth, "alias") + with pytest.raises(HTTPException) as exc: + await _rpm_request(handler, cache, auth, "alias") + assert exc.value.status_code == 429 + assert "model_per_key" in str(exc.value.detail) + + await _rpm_request(handler, cache, auth, "target") + + +@pytest.mark.parametrize( + "key_metadata, charges_team_model_pool", + [({}, True), ({"model_tpm_limit": {"target": 500}}, False)], + ids=["no_key_override", "key_owns_target_tpm_limit"], +) +def test_success_tpm_accounting_charges_the_alias_target_bucket( + key_metadata: dict[str, object], charges_team_model_pool: bool +) -> None: + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache()), model_group_resolver=_resolve_alias_to_target + ) + response: Final = ModelResponse( + id="alias-tpm", + object="chat.completion", + created=int(datetime.now().timestamp()), + model="alias", + usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), + choices=[], + ) + kwargs: Final = { + "standard_logging_object": { + "metadata": {"user_api_key_hash": hash_token("sk-alias-tpm"), "user_api_key_team_id": "t"} + }, + "litellm_params": { + "metadata": { + "model_group": "alias", + "user_api_key_metadata": key_metadata, + "user_api_key_team_metadata": {"model_tpm_limit": {"target": 500}}, + } + }, + "model": "alias", + } + + ops: Final = handler._build_success_event_pipeline_operations( + kwargs=kwargs, response_obj=response, rate_limit_type="output" + ) + + charged_keys: Final = {op["key"] for op in ops} + assert handler.create_rate_limit_keys("model_per_key", f"{hash_token('sk-alias-tpm')}:target", "tokens") in charged_keys + assert not any(":alias" in key for key in charged_keys) + team_pool_key: Final = handler.create_rate_limit_keys("model_per_team", "t:target", "tokens") + assert (team_pool_key in charged_keys) is charges_team_model_pool + + +@pytest.mark.asyncio +async def test_success_tpm_accounting_keeps_the_admission_target_after_an_alias_reload() -> None: + alias_map: Final[dict[str, str]] = {"alias": "target-a"} + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(cache), model_group_resolver=alias_map.get + ) + key_metadata: Final = {"model_tpm_limit": {"target-a": 1000, "target-b": 1000}} + auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-alias-reload"), metadata=key_metadata) + + await handler.async_pre_call_hook( + user_api_key_dict=auth, + cache=cache, + data={"model": "alias", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10}, + call_type="acompletion", + ) + stash: Final = get_request_stash() + assert stash is not None + assert stash.reserved_model == RateLimitedModel(requested="alias", group="target-a") + assert stash.reserved_tokens > 0 + + alias_map["alias"] = "target-b" + response: Final = ModelResponse( + id="alias-reload", + object="chat.completion", + created=int(datetime.now().timestamp()), + model="alias", + usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), + choices=[], + ) + kwargs: Final = { + "standard_logging_object": {"metadata": {"user_api_key_hash": auth.api_key}}, + "litellm_params": {"metadata": {"model_group": "alias", "user_api_key_metadata": key_metadata}}, + "model": "alias", + } + + ops: Final = handler._build_success_event_pipeline_operations( + kwargs=kwargs, response_obj=response, rate_limit_type="total" + ) + + admission_bucket: Final = handler.create_rate_limit_keys("model_per_key", f"{auth.api_key}:target-a", "tokens") + charged: Final = {op["key"]: op["increment_value"] for op in ops} + assert charged[admission_bucket] == 150 - stash.reserved_tokens + assert not any(":target-b" in key for key in charged) diff --git a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py index bdaca9ffc2d..e6795bb22f3 100644 --- a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py +++ b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py @@ -25,6 +25,7 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY, + RateLimitedModel, _AUDIO_BYTES_PER_TOKEN, _PROXY_MaxParallelRequestsHandler_v3 as RateLimitHandler, ) @@ -308,7 +309,7 @@ async def test_model_scope_refund_targets_reserved_model(rate_limiter): stash = get_or_create_request_stash() stash.reserved_tokens = 100 - stash.reserved_model = reserved_model + stash.reserved_model = RateLimitedModel(requested=reserved_model, group=reserved_model) stash.reserved_scopes = frozenset({("model_per_team", f"{team_id}:{reserved_model}")}) mock_kwargs = { From 1c602334ee7175f9efcb06c570b642928e99b293 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:42:00 -0700 Subject: [PATCH 36/48] test(e2e): one request lands the same spend on every surface (#42540) * test(e2e): one request lands the same spend on every surface One priced chat request must show the same response_cost on the spend log row, /key/info, /team/info, the usage export's /user/daily/activity/aggregated row, and the litellm_spend_metric Prometheus sample; each is a separate writer, so the test fails naming the surface that drifted * fix(e2e): scrape every replica's /metrics/ and enable prometheus in the replay lane --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../coverage_registry/quota_management.yaml | 1 + tests/e2e/gateway/record_replay_ci_config.yml | 3 + .../spend_tracking/spend_e2e_client.py | 79 ++++++++- .../test_spend_surface_consistency_e2e.py | 161 ++++++++++++++++++ 4 files changed, 241 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/quota_management/spend_tracking/test_spend_surface_consistency_e2e.py diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index 8a0edf4a6ca..5432c5095a2 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -46,6 +46,7 @@ - {id: quota_management.spend_tracking.cache_hit.zero_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_hit, assertions: [zero_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A response-cache hit logs at zero cost with the cache-hit marker"} - {id: quota_management.spend_tracking.key_rollup.matches_sum_of_logs, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_rollup, assertions: [matches_sum_of_logs], exercised_on: [chat_completions], source: "proxy/db/db_spend_update_writer.py", rationale: "A key's rolled-up spend equals the sum of its log rows"} - {id: quota_management.spend_tracking.concurrent_burst.loses_no_spend, module: quota_management, tier: P1, behavior: spend_tracking, variant: concurrent_burst, assertions: [loses_no_spend], exercised_on: [chat_completions], source: "proxy/db/db_spend_update_writer.py", rationale: "Concurrent calls all land as spend; no row lost to write contention"} +- {id: quota_management.spend_tracking.surface_consistency.matches_every_surface, module: quota_management, tier: P1, behavior: spend_tracking, variant: surface_consistency, assertions: [matches_every_surface], exercised_on: [chat_completions], source: "proxy/db/db_spend_update_writer.py", rationale: "One priced request lands the same response_cost on the spend log row, /key/info, /team/info, the usage export's /user/daily/activity/aggregated row, and the litellm_spend_metric Prometheus sample; each is a separate writer, so a rounding, dropped, or double-counted write on one drifts it from the rest (LIT-3620, LIT-5045)"} - {id: quota_management.spend_tracking.tags.attributes_spend, module: quota_management, tier: P1, behavior: spend_tracking, variant: tags, assertions: [attributes_spend], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Request tags round-trip to spend rows and tag rollups match tagged logs"} - {id: quota_management.spend_tracking.end_user.attributes_spend, module: quota_management, tier: P1, behavior: spend_tracking, variant: end_user, assertions: [attributes_spend], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "user= attribution lands the end-user id on the spend row"} - {id: quota_management.spend_tracking.per_model.writes_own_rows, module: quota_management, tier: P2, behavior: spend_tracking, variant: per_model, assertions: [writes_own_rows], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Each model on a shared key gets its own spend row"} diff --git a/tests/e2e/gateway/record_replay_ci_config.yml b/tests/e2e/gateway/record_replay_ci_config.yml index 5b3db0ff530..bc1f3f10562 100644 --- a/tests/e2e/gateway/record_replay_ci_config.yml +++ b/tests/e2e/gateway/record_replay_ci_config.yml @@ -2,3 +2,6 @@ general_settings: master_key: os.environ/LITELLM_MASTER_KEY store_model_in_db: true disable_model_info_refresh: true + +litellm_settings: + callbacks: ["prometheus"] diff --git a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py index b7f59fe5f89..8b63b063e14 100644 --- a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py +++ b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py @@ -12,9 +12,10 @@ helpers from one place. from __future__ import annotations import time -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from types import MappingProxyType from typing import Final from e2e_config import unique_marker @@ -48,6 +49,7 @@ from models import ( SpendLogsPageParams, SpendTagsResponse, TagSpend, + TeamInfoParams, UserDeleteBody, UserDeleteResponse, UserNewBody, @@ -57,6 +59,8 @@ from models import ( from proxy_client import Converged, ProxyClient, await_converged from pydantic import BaseModel, Field +METRICS_PATH: Final = "/metrics/" + __all__ = [ "BatchCreateBody", "CallbackLogMetadata", @@ -189,6 +193,7 @@ class DailyActivityKeyMetadata(BaseModel): class DailyActivityKeyMetrics(BaseModel): api_requests: int = 0 + spend: float = 0.0 class DailyActivityKeyBreakdown(BaseModel): @@ -209,6 +214,14 @@ class DailyActivityResponse(BaseModel): results: list[DailyActivityRow] = [] +class TeamInfoSpend(BaseModel): + spend: float | None = None + + +class TeamInfoSpendResponse(BaseModel): + team_info: TeamInfoSpend + + def _chat_body( model: str, content: str, @@ -334,6 +347,42 @@ class SpendClient: time.sleep(self.proxy.poll_interval) return spend + def team_spend(self, team_id: str) -> float: + return ( + unwrap( + self.proxy.transport.get( + "/team/info", + headers=self.proxy.transport.master, + params=TeamInfoParams(team_id=team_id), + response_type=TeamInfoSpendResponse, + ) + ).team_info.spend + or 0.0 + ) + + def poll_team_spend(self, team_id: str, *, minimum: float = 0.0) -> float: + outcome: Final = await_converged( + lambda: self.team_spend(team_id), + converged=lambda spend: spend > minimum, + timeout=self.proxy.poll_timeout, + interval=self.proxy.poll_interval, + now=time.monotonic, + sleep=time.sleep, + ) + return outcome.result if isinstance(outcome, Converged) else outcome.last_result + + def scrape_metrics(self) -> Mapping[str, ProbeResult]: + """GET /metrics/ on every replica in PROXY_REPLICA_URLS, keyed by replica. The + counter is per pod, so the union of the replicas is the fleet's exposition; the + trailing slash is the mounted app's own path, since bare /metrics answers a 307 + whose Location drops the port behind a Host-rewriting balancer.""" + return MappingProxyType( + { + replica: transport.probe(METRICS_PATH, params=NoBody()) + for replica, transport in self.proxy.replicas.items() + } + ) + def spend_logs_page( self, *, api_key: str | None, page: int, page_size: int ) -> SpendLogsPage: @@ -500,9 +549,21 @@ class SpendClient: return self.proxy.transport.probe("/health", params=HealthParams(model=model)) def daily_activity_for_key(self, token: str, *, start: datetime, end: datetime) -> DailyActivityKeyBreakdown | None: + return self._key_breakdown("/user/daily/activity", token, start=start, end=end) + + def usage_export_row_for_key( + self, token: str, *, start: datetime, end: datetime + ) -> DailyActivityKeyBreakdown | None: + """The key's row on /user/daily/activity/aggregated, the response the + dashboard's Export Usage Data CSV serializes.""" + return self._key_breakdown("/user/daily/activity/aggregated", token, start=start, end=end) + + def _key_breakdown( + self, route: str, token: str, *, start: datetime, end: datetime + ) -> DailyActivityKeyBreakdown | None: response: Final = unwrap( self.proxy.transport.get( - "/user/daily/activity", + route, headers=self.proxy.transport.master, params=DailyActivityParams( start_date=start.strftime("%Y-%m-%d"), @@ -519,9 +580,21 @@ class SpendClient: def poll_daily_activity_for_key( self, token: str, *, start: datetime, end: datetime, min_requests: int + ) -> DailyActivityKeyBreakdown | None: + return self._poll_key_breakdown(lambda: self.daily_activity_for_key(token, start=start, end=end), min_requests) + + def poll_usage_export_row_for_key( + self, token: str, *, start: datetime, end: datetime, min_requests: int + ) -> DailyActivityKeyBreakdown | None: + return self._poll_key_breakdown( + lambda: self.usage_export_row_for_key(token, start=start, end=end), min_requests + ) + + def _poll_key_breakdown( + self, fetch: Callable[[], DailyActivityKeyBreakdown | None], min_requests: int ) -> DailyActivityKeyBreakdown | None: outcome: Final = await_converged( - lambda: self.daily_activity_for_key(token, start=start, end=end), + fetch, converged=lambda found: found is not None and found.metrics.api_requests >= min_requests, timeout=self.proxy.poll_timeout, interval=self.proxy.poll_interval, diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_surface_consistency_e2e.py b/tests/e2e/quota_management/spend_tracking/test_spend_surface_consistency_e2e.py new file mode 100644 index 00000000000..9033b9d75c4 --- /dev/null +++ b/tests/e2e/quota_management/spend_tracking/test_spend_surface_consistency_e2e.py @@ -0,0 +1,161 @@ +"""One priced request must land the same response_cost on every spend surface. + +A customer reconciles the bill from whichever surface they look at: the spend +log row, the key's and the team's rolled-up spend on /key/info and /team/info, +the usage page's Export Usage Data CSV (the dashboard serializes the +/user/daily/activity/aggregated rows it already holds; there is no server-side +CSV endpoint), and the litellm_spend_metric counter Prometheus scrapes. Each is +written by a different writer (the spend log insert, the key and team rollups in +db_spend_update_writer, the daily spend tables, the Prometheus success callback), +so one of them can drift without the others noticing: the cause of the +key-versus-log mismatch in LIT-3620 and the export-versus-console mismatch in +LIT-5045. The deployment carries its own per-token rates, so the expected cost +is computed from the returned usage rather than read off any one surface, and +every surface is held to that number. + +/metrics is per pod, so every replica the stack exports (PROXY_REPLICA_URLS) is +scraped directly and the samples merged; a stack that exports only its balancer +is scraped there until the pod that served the call answers. The request itself +is sent once. +""" + +from __future__ import annotations + +import time +from collections.abc import Mapping +from datetime import datetime, timedelta, timezone +from itertools import groupby +from math import isclose +from types import MappingProxyType +from typing import Final + +import pytest +from e2e_config import provider_edge_base, unique_marker +from e2e_http import ProbeResult +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, KeyGenerateBody, LiteLLMParamsBody, TeamNewBody +from prometheus_client.parser import text_string_to_metric_families +from proxy_client import Converged, await_converged +from spend_e2e_client import SpendClient, unwrap +from spend_reconciliation import INPUT_RATE, OUTPUT_RATE + +pytestmark = pytest.mark.e2e + +SPEND_METRIC: Final = "litellm_spend_metric_total" +KEY_HASH_LABEL: Final = "hashed_api_key" +TEAM_LABEL: Final = "team" + +SeriesLabels = tuple[tuple[str, str], ...] + + +def _spend_series_for_key(scrapes: Mapping[str, ProbeResult], token: str) -> Mapping[SeriesLabels, float]: + samples: Final = sorted( + (tuple(sorted(sample.labels.items())), sample.value) + for scrape in scrapes.values() + if scrape.status_code == 200 + for family in text_string_to_metric_families(scrape.body) + for sample in family.samples + if sample.name == SPEND_METRIC and sample.labels.get(KEY_HASH_LABEL) == token + ) + return MappingProxyType( + {labels: sum(value for _, value in group) for labels, group in groupby(samples, key=lambda sample: sample[0])} + ) + + +def _poll_spend_series_for_key(client: SpendClient, token: str) -> Mapping[SeriesLabels, float]: + outcome: Final = await_converged( + client.scrape_metrics, + converged=lambda scrapes: bool(_spend_series_for_key(scrapes, token)), + timeout=client.proxy.poll_timeout, + interval=client.proxy.poll_interval, + now=time.monotonic, + sleep=time.sleep, + ) + scrapes: Final = outcome.result if isinstance(outcome, Converged) else outcome.last_result + assert _spend_series_for_key(scrapes, token), ( + f"{SPEND_METRIC} never exposed a series for {KEY_HASH_LABEL}={token} on any replica; " + f"last scrape status per replica: {({replica: scrape.status_code for replica, scrape in scrapes.items()})}" + ) + return _spend_series_for_key(scrapes, token) + + +def _same_spend(actual: float | None, expected: float) -> bool: + return actual is not None and isclose(actual, expected, rel_tol=1e-6, abs_tol=1e-9) + + +class TestSpendSurfaceConsistency: + @pytest.mark.replayable + @pytest.mark.covers("quota_management.spend_tracking.surface_consistency.matches_every_surface") + def test_one_request_lands_the_same_spend_on_every_surface( + self, client: SpendClient, resources: ResourceManager + ) -> None: + started: Final = datetime.now(timezone.utc) + marker: Final = unique_marker() + base: Final = provider_edge_base("openai") + model: Final = f"e2e-spend-surfaces-{marker}" + model_id: Final = client.proxy.create_model( + model, + LiteLLMParamsBody( + model="openai/gpt-5.6-luna", + api_key="os.environ/OPENAI_API_KEY", + api_base=None if base is None else f"{base}/v1", + input_cost_per_token=INPUT_RATE, + output_cost_per_token=OUTPUT_RATE, + ), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + team_id: Final = client.proxy.create_team(TeamNewBody(team_alias=f"e2e-spend-surfaces-{marker}")) + resources.defer(lambda: client.proxy.delete_team(team_id)) + record: Final = client.generate_key_record( + KeyGenerateBody(team_id=team_id, models=[model], key_alias=f"e2e-spend-surfaces-{marker}") + ) + resources.defer(lambda: client.proxy.delete_key(record.key)) + assert record.token, "/key/generate answered without the key's token hash" + token: Final = record.token + + response: Final = unwrap( + client.proxy.chat( + record.key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content=f"Reply with one word. {marker}")], + max_completion_tokens=128, + ), + ) + ) + usage: Final = response.usage + assert response.id, "successful response must have an ID" + assert usage is not None and usage.prompt_tokens and usage.completion_tokens, f"no billable usage: {usage}" + expected: Final = usage.prompt_tokens * INPUT_RATE + usage.completion_tokens * OUTPUT_RATE + + rows: Final = client.proxy.poll_logs_for_request_id(response.id) + assert len(rows) == 1, f"expected one spend row for {response.id}, saw {len(rows)}: {rows}" + row: Final = rows[0] + assert row.api_key == token, f"spend row keyed by {row.api_key}, not the key's token hash {token}" + assert row.team_id == team_id, f"spend row attributed to team {row.team_id}, not {team_id}" + assert row.status == "success", f"spend row status {row.status}" + + key_spend: Final = client.poll_key_spend(record.key, minimum=expected * 0.999999) + team_spend: Final = client.poll_team_spend(team_id, minimum=expected * 0.999999) + export_row: Final = client.poll_usage_export_row_for_key( + token, start=started - timedelta(days=1), end=datetime.now(timezone.utc), min_requests=1 + ) + assert export_row is not None, f"/user/daily/activity/aggregated never listed key {token} under api_keys" + series: Final = _poll_spend_series_for_key(client, token) + off_team: Final = tuple(labels for labels in series if dict(labels).get(TEAM_LABEL) != team_id) + assert not off_team, f"{SPEND_METRIC} series for the key carry a team other than {team_id}: {off_team}" + + observed: Final = MappingProxyType( + { + "/spend/logs row": row.spend, + "/key/info spend": key_spend, + "/team/info spend": team_spend, + "usage export row (/user/daily/activity/aggregated)": export_row.metrics.spend, + SPEND_METRIC: sum(series.values()), + } + ) + drifted: Final = tuple(surface for surface, spend in observed.items() if not _same_spend(spend, expected)) + assert not drifted, ( + f"response_cost {expected} (usage {usage.prompt_tokens}x{INPUT_RATE} + " + f"{usage.completion_tokens}x{OUTPUT_RATE}) drifted on {drifted}; every surface: {dict(observed)}" + ) From 7210403e2334504872f2c46220d9a0fd36eab622 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:44:25 -0700 Subject: [PATCH 37/48] fix(mcp): apply post-call rewrites without stale structured output (#41530) * fix(mcp): apply async_post_mcp_tool_call_hook content changes to the tool result Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(mcp): drop structuredContent when a post-call hook rewrites tool content Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(mcp): satisfy type discipline and result contract Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(mcp): document internal logging patch Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(mcp): avoid Final assignments inside callback loops Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(mcp): run every post-call hook and chain the rewritten content Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(mcp): credit the original fix from #33403 Co-authored-by: eric Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(mcp): cover post-call logging fallback paths Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(mcp): cover proxy hook logging context Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(mcp): preserve native structured guardrail replacements * fix(mcp): invalidate stale structure after direct content edits * fix(mcp): reconcile direct edits after callback exceptions * fix(mcp): preserve successful in-place callback rewrites --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: eric Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/integrations/custom_logger.py | 9 +- litellm/litellm_core_utils/litellm_logging.py | 67 +++-- .../_experimental/mcp_server/operations.py | 2 +- .../mcp/litellm_proxy_mcp_handler.py | 2 +- tests/mcp_tests/test_mcp_server.py | 4 +- .../test_litellm_logging.py | 273 +++++++++++++++++- .../mcp_server/test_mcp_server.py | 62 +++- .../test_cisco_ai_defense_mcp.py | 35 +++ .../mcp/test_litellm_proxy_mcp_handler.py | 158 +++++++++- 9 files changed, 572 insertions(+), 40 deletions(-) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 5b5261fab6b..326abd5c6a3 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -574,11 +574,10 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac Useful if you want to modify the standard logging payload after the MCP tool call is made. - To change what the caller sends back to the MCP client, mutate ``response_obj`` - in place: every call site discards the returned object, because the - dispatcher unwraps it to ``mcp_tool_call_response`` (a raw content list, not - a ``CallToolResult``) which the tool-call paths cannot forward. Guardrails - that mask or reject tool output should use ``post_mcp_call`` instead. + Modify ``mcp_tool_call_response`` in place or return a replacement response + object to change what the caller sends back to the MCP client. Content rewrites + discard stale structured output and mark those results as tool errors. + Use ``post_mcp_call`` guardrails for schema-preserving structured redaction. """ return None diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 357ee51a44a..6e5a37a7226 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -221,7 +221,7 @@ from .initialize_dynamic_callback_params import ( from .specialty_caches.dynamic_logging_cache import DynamicLoggingCache if TYPE_CHECKING: - from mcp.types import EmbeddedResource, ImageContent, TextContent + from mcp.types import CallToolResult, EmbeddedResource, ImageContent, TextContent from litellm.integrations.otel.logger import OpenTelemetryV2 from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config @@ -1634,15 +1634,11 @@ class Logging(LiteLLMLoggingBaseClass): async def async_post_mcp_tool_call_hook( self, kwargs: dict, - response_obj: Any, + response_obj: "CallToolResult", start_time: datetime.datetime, end_time: datetime.datetime, - ): - """ - Post MCP Tool Call Hook - - Use this to modify the MCP tool call response before it is returned to the user. - """ + ) -> "CallToolResult": + """Apply ordered MCP content callbacks to the result returned to the caller.""" from litellm.types.llms.base import HiddenParams from litellm.types.mcp import MCPPostCallResponseObject @@ -1650,24 +1646,51 @@ class Logging(LiteLLMLoggingBaseClass): dynamic_success_callbacks=self.dynamic_success_callbacks, global_callbacks=litellm.success_callback, ) - post_mcp_tool_call_response_obj: Final[MCPPostCallResponseObject] = MCPPostCallResponseObject( - mcp_tool_call_response=response_obj, hidden_params=HiddenParams() - ) + hidden_params = HiddenParams() for callback in callbacks: try: if isinstance(callback, CustomLogger): - response: MCPPostCallResponseObject | None = await callback.async_post_mcp_tool_call_hook( - kwargs=kwargs, - response_obj=post_mcp_tool_call_response_obj, - start_time=start_time, - end_time=end_time, + original_content = copy.deepcopy(response_obj.content) + original_structured_content = copy.deepcopy(response_obj.structured_content) + callback_response = MCPPostCallResponseObject( + mcp_tool_call_response=copy.deepcopy(original_content), hidden_params=hidden_params ) - ###################################################################### - # if any of the callbacks modify the response, use the modified response - # current implementation returns the first modified response - ###################################################################### - if response is not None: - response_obj = self._parse_post_mcp_call_hook_response(response=response) + try: + response = await callback.async_post_mcp_tool_call_hook( + kwargs=kwargs, + response_obj=callback_response, + start_time=start_time, + end_time=end_time, + ) + hook_content = ( + self._parse_post_mcp_call_hook_response(response=response) + if response is not None + else callback_response.mcp_tool_call_response + ) + if response is not None: + hidden_params = response.hidden_params + except Exception as e: + verbose_logger.exception( + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e + ) + hook_content = None + structured_replacement_matches = ( + response_obj.structured_content != original_structured_content + and ( + hook_content is None + or hook_content == original_content + or response_obj.content == hook_content + ) + ) + if hook_content is not None and hook_content != original_content: + response_obj.content[:] = hook_content + if ( + response_obj.content != original_content + and response_obj.structured_content is not None + and not structured_replacement_matches + ): + response_obj.structured_content = None + response_obj.is_error = True except Exception as e: verbose_logger.exception("LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e) return response_obj diff --git a/litellm/proxy/_experimental/mcp_server/operations.py b/litellm/proxy/_experimental/mcp_server/operations.py index fcee3483e15..2bd6d186f24 100644 --- a/litellm/proxy/_experimental/mcp_server/operations.py +++ b/litellm/proxy/_experimental/mcp_server/operations.py @@ -2198,7 +2198,7 @@ async def _fire_mcp_tool_call_logging( from litellm.proxy.proxy_server import proxy_logging_obj logging_obj.post_call(original_response=result) - await logging_obj.async_post_mcp_tool_call_hook( + result = await logging_obj.async_post_mcp_tool_call_hook( kwargs=logging_obj.model_call_details, response_obj=result, start_time=start_time, diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 88a2b92c680..319f10a9b22 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -872,7 +872,7 @@ class LiteLLM_Proxy_MCP_Handler: if litellm_logging_obj: try: litellm_logging_obj.post_call(original_response=result) - await litellm_logging_obj.async_post_mcp_tool_call_hook( + result = await litellm_logging_obj.async_post_mcp_tool_call_hook( kwargs=litellm_logging_obj.model_call_details, response_obj=result, start_time=start_time, diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index d38a11ec927..dfd250338ff 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -2962,7 +2962,7 @@ async def test_call_mcp_tool_uses_manager_permission_lookup(): mcp_info={"server_name": "test_server"}, ) - expected_response = [TextContent(type="text", text="ok")] + expected_response = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False) with ( patch.object( @@ -3038,7 +3038,7 @@ async def test_call_mcp_tool_resolves_unprefixed_tool_name_and_checks_permission mcp_info={"server_name": "test_server"}, ) - expected_response = [TextContent(type="text", text="ok")] + expected_response = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False) with ( patch.object( diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index b0fc725b50a..4541250e896 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -5,16 +5,15 @@ import json import logging import os import sys +import time from collections.abc import Callable, Iterator, Mapping from types import MappingProxyType from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch -import pytest - -import time - import httpx +import pytest +from mcp.types import AudioContent, CallToolResult, ImageContent, TextContent from openai._legacy_response import HttpxBinaryResponseContent import litellm @@ -51,6 +50,272 @@ def logging_obj(): ) +@pytest.mark.asyncio +async def test_async_post_mcp_tool_call_hook_preserves_and_returns_content(logging_obj): + from litellm.types.mcp import MCPPostCallResponseObject + + class RedactingLogger(CustomLogger): + async def async_post_mcp_tool_call_hook( + self, + kwargs: dict[str, object], + response_obj: MCPPostCallResponseObject, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> MCPPostCallResponseObject: + assert isinstance(response_obj.mcp_tool_call_response, list) + assert isinstance(response_obj.mcp_tool_call_response[0], TextContent) + response_obj.mcp_tool_call_response = [TextContent(type="text", text="[REDACTED]")] + return response_obj + + logging_obj.dynamic_success_callbacks = [RedactingLogger()] + result = CallToolResult(content=[TextContent(type="text", text="SECRET-1234")], isError=False) + + hooked_content = await logging_obj.async_post_mcp_tool_call_hook( + kwargs=logging_obj.model_call_details, + response_obj=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + ) + + assert hooked_content.content == [TextContent(type="text", text="[REDACTED]")] + + +@pytest.mark.asyncio +async def test_async_post_mcp_tool_call_hook_chains_every_callback(logging_obj): + from litellm.types.mcp import MCPPostCallResponseObject + + class ReplacingLogger(CustomLogger): + def __init__(self, old: str, new: str) -> None: + super().__init__() + self.old: Final = old + self.new: Final = new + self.seen: list[str] = [] # mutable-ok: test records what each callback observed + + async def async_post_mcp_tool_call_hook( + self, + kwargs: dict[str, object], + response_obj: MCPPostCallResponseObject, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> MCPPostCallResponseObject: + first = response_obj.mcp_tool_call_response[0] + assert isinstance(first, TextContent) + self.seen.append(first.text) + return MCPPostCallResponseObject( + mcp_tool_call_response=[TextContent(type="text", text=first.text.replace(self.old, self.new))], + hidden_params=response_obj.hidden_params, + ) + + first_logger: Final = ReplacingLogger("SECRET", "[S]") + second_logger: Final = ReplacingLogger("1234", "[N]") + logging_obj.dynamic_success_callbacks = [first_logger, second_logger] + result = CallToolResult(content=[TextContent(type="text", text="SECRET-1234")], isError=False) + + hooked_content = await logging_obj.async_post_mcp_tool_call_hook( + kwargs=logging_obj.model_call_details, + response_obj=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + ) + + assert first_logger.seen == ["SECRET-1234"] + assert second_logger.seen == ["[S]-1234"] + assert hooked_content.content == [TextContent(type="text", text="[S]-[N]")] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["replace", "inplace", "empty", "inplace_none", "replace_none"]) +@pytest.mark.parametrize("structured", [False, True]) +async def test_mcp_content_rewrite_never_returns_stale_structured_data(logging_obj, mode, structured): + from litellm.types.llms.base import HiddenParams + from litellm.types.mcp import MCPPostCallResponseObject + + class Redactor(CustomLogger): + async def async_post_mcp_tool_call_hook(self, kwargs, response_obj, start_time, end_time): + block = response_obj.mcp_tool_call_response[0] + assert isinstance(block, TextContent) + if mode in ("inplace", "inplace_none"): + block.text = "[REDACTED]" + return None if mode == "inplace_none" else response_obj + if mode == "replace_none": + response_obj.mcp_tool_call_response = [TextContent(type="text", text="[REDACTED]")] + return None + return MCPPostCallResponseObject( + mcp_tool_call_response=[] if mode == "empty" else [TextContent(type="text", text="[REDACTED]")], + hidden_params=HiddenParams(response_cost=0.25), + ) + + logging_obj.dynamic_success_callbacks = [Redactor()] + result = CallToolResult( + content=[TextContent(type="text", text="SECRET-1234")], + structured_content={"nested": {"secret": "SECRET-1234"}} if structured else None, + meta={"request": "trace-1"}, + ) + logging_obj.model_call_details["original_response"] = result + returned = await logging_obj.async_post_mcp_tool_call_hook( + kwargs=logging_obj.model_call_details, + response_obj=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + ) + assert "SECRET-1234" not in result.model_dump_json(by_alias=True) + assert returned is result + assert result.content == ([] if mode == "empty" else [TextContent(type="text", text="[REDACTED]")]) + assert result.structured_content is None + assert result.is_error is structured + assert result.meta == {"request": "trace-1"} + assert logging_obj.model_call_details["original_response"] is result + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["none", "cost", "direct", "block", "exception"]) +async def test_mcp_callbacks_preserve_effective_result_and_cost(logging_obj, mode): + from litellm.types.llms.base import HiddenParams + from litellm.types.mcp import MCPPostCallResponseObject + + class Callback(CustomLogger): + async def async_post_mcp_tool_call_hook(self, kwargs, response_obj, start_time, end_time): + if mode == "exception": + response_obj.mcp_tool_call_response[0].text = "discarded" + raise ValueError("non-blocking callback") + if mode in ("direct", "block"): + original = kwargs["original_response"] + original.content = [TextContent(type="text", text="safe")] + original.structured_content = {"result": "safe"} + original.is_error = mode == "block" + if mode == "none" or mode == "direct": + return None + return MCPPostCallResponseObject( + mcp_tool_call_response=response_obj.mcp_tool_call_response, + hidden_params=HiddenParams(response_cost=0.25), + ) + + logging_obj.dynamic_success_callbacks = [Callback()] + result = CallToolResult( + content=[TextContent(type="text", text="original")], structured_content={"result": "original"} + ) + logging_obj.model_call_details["original_response"] = result + returned = await logging_obj.async_post_mcp_tool_call_hook( + kwargs=logging_obj.model_call_details, + response_obj=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + ) + expected = "safe" if mode in ("direct", "block") else "original" + assert result.content == [TextContent(type="text", text=expected)] + assert result.structured_content == {"result": expected} + assert result.is_error is (mode == "block") + assert returned is result + assert logging_obj.model_call_details.get("response_cost") == (0.25 if mode in ("cost", "block") else None) + + +@pytest.mark.asyncio +async def test_mcp_callback_cancellation_propagates_without_mutating_result(logging_obj): + class CancelledCallback(CustomLogger): + async def async_post_mcp_tool_call_hook(self, kwargs, response_obj, start_time, end_time): + response_obj.mcp_tool_call_response[0].text = "partial" + raise asyncio.CancelledError + + logging_obj.dynamic_success_callbacks = [CancelledCallback()] + result = CallToolResult(content=[TextContent(type="text", text="original")]) + with pytest.raises(asyncio.CancelledError): + await logging_obj.async_post_mcp_tool_call_hook( + kwargs={}, + response_obj=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + ) + assert result.content == [TextContent(type="text", text="original")] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("callbacks", [[], ["prometheus"]]) +@pytest.mark.parametrize("is_error", [False, True]) +async def test_mcp_without_custom_callbacks_preserves_mixed_content(logging_obj, callbacks, is_error): + logging_obj.dynamic_success_callbacks = callbacks + result = CallToolResult( + content=[ + TextContent(type="text", text="ok"), + ImageContent(type="image", data="aW1n", mime_type="image/png"), + AudioContent(type="audio", data="c291bmQ=", mime_type="audio/wav"), + ], + structured_content={"result": "ok"}, + is_error=is_error, + ) + before = result.model_dump() + returned = await logging_obj.async_post_mcp_tool_call_hook( + kwargs={}, + response_obj=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + ) + assert returned is result + assert returned.model_dump() == before + + +@pytest.mark.asyncio +@pytest.mark.parametrize("replace_structured", [False, True]) +@pytest.mark.parametrize("same_content", [False, True]) +async def test_mcp_native_structured_replacement_must_match_returned_content( + logging_obj, replace_structured, same_content +): + from litellm.types.mcp import MCPPostCallResponseObject + + class NativeReplacement(CustomLogger): + async def async_post_mcp_tool_call_hook(self, kwargs, response_obj, start_time, end_time): + original = kwargs["original_response"] + original.content[0].text = "native-safe" + if replace_structured: + original.structured_content["result"] = "native-safe" + return MCPPostCallResponseObject( + mcp_tool_call_response=[TextContent(type="text", text="native-safe" if same_content else "final-safe")], + hidden_params=response_obj.hidden_params, + ) + + result = CallToolResult( + content=[TextContent(type="text", text="SECRET-1234")], + structured_content={"result": "SECRET-1234"}, + ) + logging_obj.dynamic_success_callbacks = [NativeReplacement()] + returned = await logging_obj.async_post_mcp_tool_call_hook( + kwargs={"original_response": result}, response_obj=result, + start_time=datetime.datetime.now(), end_time=datetime.datetime.now(), + ) + assert returned is result + assert result.content == [TextContent(type="text", text="native-safe" if same_content else "final-safe")] + assert result.structured_content == ({"result": "native-safe"} if replace_structured and same_content else None) + assert result.is_error is not (replace_structured and same_content) + assert "SECRET-1234" not in result.model_dump_json() + + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["none", "wrapper", "exception"]) +@pytest.mark.parametrize("structured", [False, True]) +async def test_mcp_direct_content_edit_invalidates_stale_structured_data(logging_obj, mode, structured): + class DirectRedactor(CustomLogger): + async def async_post_mcp_tool_call_hook(self, kwargs, response_obj, start_time, end_time): + kwargs["original_response"].content[0].text = "[REDACTED]" + if mode == "exception": + raise ValueError("non-blocking callback after direct edit") + return response_obj if mode == "wrapper" else None + + result = CallToolResult( + content=[TextContent(type="text", text="SECRET-1234")], + structured_content={"result": "SECRET-1234"} if structured else None, + ) + logging_obj.dynamic_success_callbacks = [DirectRedactor()] + returned = await logging_obj.async_post_mcp_tool_call_hook( + kwargs={"original_response": result}, response_obj=result, + start_time=datetime.datetime.now(), end_time=datetime.datetime.now(), + ) + assert returned is result + assert result.content == [TextContent(type="text", text="[REDACTED]")] + assert result.structured_content is None + assert result.is_error is structured + assert "SECRET-1234" not in result.model_dump_json() + + def test_get_combined_callback_list_preserves_insertion_order(logging_obj): assert logging_obj.get_combined_callback_list( dynamic_success_callbacks=["prometheus", "langfuse", "datadog", "otel", "s3"], diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index 6588ab4c87b..ea82c9b069d 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -8755,7 +8755,7 @@ def _call_tool_result(is_error: bool, text: str) -> CallToolResult: def _mock_mcp_logging_obj() -> MagicMock: logging_obj = MagicMock() logging_obj.model_call_details = {} - logging_obj.async_post_mcp_tool_call_hook = AsyncMock() + logging_obj.async_post_mcp_tool_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["response_obj"]) logging_obj.async_success_handler = AsyncMock() logging_obj.async_failure_handler = AsyncMock() return logging_obj @@ -8857,6 +8857,64 @@ async def test_fire_mcp_tool_call_logging_success_path_unchanged(): proxy_logging_mock.post_call_failure_hook.assert_not_awaited() +@pytest.mark.asyncio +async def test_fire_mcp_tool_call_logging_applies_hook_content(): + from litellm.integrations.custom_logger import CustomLogger + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy._experimental.mcp_server.server import ( + _fire_mcp_tool_call_logging, + ) + from litellm.types.mcp import MCPPostCallResponseObject + + class RedactingLogger(CustomLogger): + async def async_post_mcp_tool_call_hook( + self, + kwargs: dict[str, object], + response_obj: MCPPostCallResponseObject, + start_time: datetime, + end_time: datetime, + ) -> MCPPostCallResponseObject: + assert isinstance(response_obj.mcp_tool_call_response, list) + assert isinstance(response_obj.mcp_tool_call_response[0], TextContent) + response_obj.mcp_tool_call_response = [TextContent(type="text", text="[REDACTED]")] + return response_obj + + logging_obj = Logging( + model="MCP: weather/get_forecast", + messages=[{"role": "user", "content": "tool call"}], + stream=False, + call_type="call_mcp_tool", + start_time=datetime.now(), + litellm_call_id="test-mcp-hook-content", + function_id="test-fn", + dynamic_success_callbacks=[RedactingLogger()], + ) + proxy_logging_mock = _mock_mcp_proxy_logging() + result = CallToolResult( + content=[TextContent(type="text", text="SECRET-1234")], + structuredContent={"result": "SECRET-1234"}, + isError=False, + ) + + with patch( # test-quality-ok: [TQ008] inject proxy logging collaborator + "litellm.proxy.proxy_server.proxy_logging_obj", + proxy_logging_mock, + ): + hooked_result = await _fire_mcp_tool_call_logging( + logging_obj=logging_obj, + result=result, + start_time=datetime.now(), + end_time=datetime.now(), + user_api_key_auth=UserAPIKeyAuth(api_key="test-key", user_id="test-user"), + request_data={}, + ) + + assert isinstance(hooked_result.content[0], TextContent) + assert hooked_result.content[0].text == "[REDACTED]" + assert hooked_result.structured_content is None + assert hooked_result.is_error is True + + @pytest.mark.asyncio async def test_fire_mcp_tool_call_logging_iserror_without_auth_skips_failure_hook(): """Without a UserAPIKeyAuth the failure handlers still fire but the proxy @@ -8871,7 +8929,7 @@ async def test_fire_mcp_tool_call_logging_iserror_without_auth_skips_failure_hoo with patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_mock): await _fire_mcp_tool_call_logging( logging_obj=logging_obj, - result={"isError": True, "content": [{"type": "text", "text": "denied"}]}, + result=_call_tool_result(True, "denied"), start_time=datetime.now(), end_time=datetime.now(), ) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_mcp.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_mcp.py index 826edab694d..2e3bf760e68 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_mcp.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_mcp.py @@ -827,3 +827,38 @@ class TestCiscoAIDefenseJsonRpcSuccessEnvelope: assert unwrapped is verdict else: assert unwrapped == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ["block", "redact"]) +async def test_cisco_native_hook_through_logging_preserves_sanitized_result(action): + from mcp.types import CallToolResult, TextContent + from litellm.litellm_core_utils.litellm_logging import Logging + + guardrail = _make_guardrail( + inspection_type="mcp", event_hook=["pre_mcp_call", "during_mcp_call"] + ) + verdict = ( + _violation_response(url=MCP_URL) if action == "block" + else _redact_response(sanitized_text="[REDACTED]", url=MCP_URL) + ) + result = CallToolResult( + content=[TextContent(type="text", text="SECRET-1234")], + structured_content={"result": "SECRET-1234"}, + ) + logging_obj = Logging( + model="MCP: probe/search", messages=[], stream=False, call_type="call_mcp_tool", + start_time=datetime.now(), litellm_call_id="cisco-hook", function_id="cisco-hook", + dynamic_success_callbacks=[guardrail], + ) + logging_obj.model_call_details.update({"name": "search", "arguments": {}, "original_response": result}) + with _patch_inspection_post(guardrail, AsyncMock(return_value=verdict)): + returned = await logging_obj.async_post_mcp_tool_call_hook( + kwargs=logging_obj.model_call_details, response_obj=result, + start_time=datetime.now(), end_time=datetime.now(), + ) + assert returned is result + assert "SECRET-1234" not in returned.model_dump_json() + assert returned.is_error is (action == "block") + assert ("Blocked by Cisco AI Defense" if action == "block" else "[REDACTED]") in returned.content[0].text + assert returned.structured_content == {"result": returned.content[0].text} diff --git a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py index 83537c236a3..56e40206ebe 100644 --- a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -1,13 +1,15 @@ +import importlib import subprocess import sys import textwrap import types +from typing import Any, cast from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import HTTPException +from mcp.types import CallToolResult, TextContent from openai.types.responses.tool_param import Mcp -import importlib from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing from litellm.responses import main as responses_main @@ -15,10 +17,9 @@ from litellm.responses.mcp import litellm_proxy_mcp_handler as mcp_handler_modul from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) -from typing import Any, cast from litellm.types.llms.openai import ResponsesAPIResponse -from litellm.types.utils import ModelResponse from litellm.types.responses.main import OutputFunctionToolCall +from litellm.types.utils import ModelResponse class _DummyMCPResult: @@ -492,6 +493,157 @@ async def test_execute_tool_calls_threads_logging_obj_into_call_tool(monkeypatch assert call_tool_mock.await_args.kwargs["litellm_logging_obj"] is sentinel_logging_obj +@pytest.mark.asyncio +async def test_execute_tool_calls_applies_post_call_hook_content(monkeypatch): + proxy_module = types.SimpleNamespace(proxy_logging_obj=None) + monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", proxy_module) + + result = CallToolResult( + content=[TextContent(type="text", text="SECRET-1234")], + structuredContent={"result": "SECRET-1234"}, + isError=False, + ) + fake_manager = types.SimpleNamespace( + get_registry=MagicMock(return_value={}), + call_tool=AsyncMock(return_value=result), + _get_mcp_server_from_tool_name=MagicMock(return_value=None), + get_mcp_server_by_name=MagicMock(return_value=None), + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + fake_manager, + ) + + logging_obj = MagicMock() + logging_obj.model_call_details = {} + logging_obj.async_post_mcp_tool_call_hook = AsyncMock(return_value=CallToolResult(content=[TextContent(type="text", text="[REDACTED]")], is_error=True)) + logging_obj.async_success_handler = AsyncMock() + handler_module = importlib.import_module("litellm.responses.mcp.litellm_proxy_mcp_handler") + monkeypatch.setattr(handler_module, "function_setup", lambda *_args, **_kwargs: (logging_obj, None)) + + tool_name = "deepwiki-read_wiki_structure" + results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map={tool_name: "deepwiki"}, + tool_calls=[{"id": "call-1", "function": {"name": tool_name, "arguments": "{}"}}], + user_api_key_auth=None, + ) + + assert results == [{"tool_call_id": "call-1", "result": "[REDACTED]", "name": tool_name}] + assert logging_obj.async_success_handler.await_args.kwargs["result"].content[0].text == "[REDACTED]" + assert logging_obj.async_success_handler.await_args.kwargs["result"].structured_content is None + + +@pytest.mark.asyncio +async def test_execute_tool_calls_returns_proxy_result_without_logging(monkeypatch): + result = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False) + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_mcp_call_hook = AsyncMock(side_effect=lambda response, **_: response) + monkeypatch.setitem( + sys.modules, "litellm.proxy.proxy_server", types.SimpleNamespace(proxy_logging_obj=proxy_logging_obj) + ) + + fake_manager = types.SimpleNamespace( + get_registry=MagicMock(return_value={}), + call_tool=AsyncMock(return_value=result), + _get_mcp_server_from_tool_name=MagicMock(return_value=None), + get_mcp_server_by_name=MagicMock(return_value=None), + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + fake_manager, + ) + handler_module = importlib.import_module("litellm.responses.mcp.litellm_proxy_mcp_handler") + monkeypatch.setattr(handler_module, "function_setup", lambda *_args, **_kwargs: (None, None)) + + tool_name = "deepwiki-read_wiki_structure" + results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map={tool_name: "deepwiki"}, + tool_calls=[{"id": "call-1", "function": {"name": tool_name, "arguments": "{}"}}], + user_api_key_auth=None, + ) + + assert results == [{"tool_call_id": "call-1", "result": "ok", "name": tool_name}] + proxy_logging_obj.post_mcp_call_hook.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_execute_tool_calls_passes_logging_details_to_proxy_hook(monkeypatch): + result = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False) + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_mcp_call_hook = AsyncMock(side_effect=lambda response, **_: response) + monkeypatch.setitem( + sys.modules, "litellm.proxy.proxy_server", types.SimpleNamespace(proxy_logging_obj=proxy_logging_obj) + ) + + fake_manager = types.SimpleNamespace( + get_registry=MagicMock(return_value={}), + call_tool=AsyncMock(return_value=result), + _get_mcp_server_from_tool_name=MagicMock(return_value=None), + get_mcp_server_by_name=MagicMock(return_value=None), + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + fake_manager, + ) + logging_obj = MagicMock() + logging_obj.model_call_details = {"request_id": "request-1"} + logging_obj.async_post_mcp_tool_call_hook = AsyncMock(return_value=result) + logging_obj.async_success_handler = AsyncMock() + handler_module = importlib.import_module("litellm.responses.mcp.litellm_proxy_mcp_handler") + monkeypatch.setattr(handler_module, "function_setup", lambda *_args, **_kwargs: (logging_obj, None)) + + tool_name = "deepwiki-read_wiki_structure" + results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map={tool_name: "deepwiki"}, + tool_calls=[{"id": "call-1", "function": {"name": tool_name, "arguments": "{}"}}], + user_api_key_auth=None, + ) + + assert results == [{"tool_call_id": "call-1", "result": "ok", "name": tool_name}] + assert proxy_logging_obj.post_mcp_call_hook.await_args.kwargs["request_data"] == logging_obj.model_call_details + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure_stage", ["post_call_hook", "success_handler"]) +async def test_execute_tool_calls_continues_when_post_call_logging_fails(monkeypatch, failure_stage: str): + proxy_module = types.SimpleNamespace(proxy_logging_obj=None) + monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", proxy_module) + + result = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False) + fake_manager = types.SimpleNamespace( + get_registry=MagicMock(return_value={}), + call_tool=AsyncMock(return_value=result), + _get_mcp_server_from_tool_name=MagicMock(return_value=None), + get_mcp_server_by_name=MagicMock(return_value=None), + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + fake_manager, + ) + + logging_obj = MagicMock() + logging_obj.model_call_details = {} + logging_obj.post_call = MagicMock() + logging_obj.async_post_mcp_tool_call_hook = AsyncMock( + side_effect=RuntimeError("hook failed") if failure_stage == "post_call_hook" else None, + return_value=result, + ) + logging_obj.async_success_handler = AsyncMock( + side_effect=RuntimeError("success logging failed") if failure_stage == "success_handler" else None + ) + handler_module = importlib.import_module("litellm.responses.mcp.litellm_proxy_mcp_handler") + monkeypatch.setattr(handler_module, "function_setup", lambda *_args, **_kwargs: (logging_obj, None)) + + tool_name = "deepwiki-read_wiki_structure" + results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map={tool_name: "deepwiki"}, + tool_calls=[{"id": "call-1", "function": {"name": tool_name, "arguments": "{}"}}], + user_api_key_auth=None, + ) + + assert results == [{"tool_call_id": "call-1", "result": "ok", "name": tool_name}] + + @pytest.mark.asyncio async def test_get_mcp_tools_from_manager_enables_list_tools_logging(monkeypatch): """ From e54f228ed87b78659bee066ccd2f4e36bfe97926 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:46:15 -0700 Subject: [PATCH 38/48] chore(prices): sync Baseten prices: 4 models, 4 new [enrichment failed: Baseten, 3 held] (#42578) baseten/deepseek-ai/DeepSeek-V4-Flash-0731: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_reasoning, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost baseten/deepseek-ai/DeepSeek-V4-Pro: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_reasoning, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost baseten/deepseek-ai/DeepSeek-V4-Pro-0813: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_reasoning, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost baseten/zai-org/GLM-5.2-Fast: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_reasoning, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 68 +++++++++++++++++++ model_prices_and_context_window.json | 68 +++++++++++++++++++ 2 files changed, 136 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a2863f6b0dc..b8ca5ff27c7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -77665,5 +77665,73 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "baseten/deepseek-ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.3e-07, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.6e-07, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "baseten/deepseek-ai/DeepSeek-V4-Pro": { + "cache_read_input_token_cost": 1.45e-07, + "input_cost_per_token": 1.74e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.48e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "baseten/deepseek-ai/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 1.32e-07, + "input_cost_per_token": 1.32e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "baseten/zai-org/GLM-5.2-Fast": { + "cache_read_input_token_cost": 2.1e-07, + "input_cost_per_token": 2.1e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a2863f6b0dc..b8ca5ff27c7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -77665,5 +77665,73 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "baseten/deepseek-ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.3e-07, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.6e-07, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "baseten/deepseek-ai/DeepSeek-V4-Pro": { + "cache_read_input_token_cost": 1.45e-07, + "input_cost_per_token": 1.74e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.48e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "baseten/deepseek-ai/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 1.32e-07, + "input_cost_per_token": 1.32e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "baseten/zai-org/GLM-5.2-Fast": { + "cache_read_input_token_cost": 2.1e-07, + "input_cost_per_token": 2.1e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } From b5b116a519fbef0ad894f4500f686270d6d860a9 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 20:50:53 +0000 Subject: [PATCH 39/48] fix(otel): honor SSL_CERT_FILE and ssl_verify in OTLP HTTP exporters (#42106) * fix(otel): honor SSL_CERT_FILE and ssl_verify in OTLP HTTP exporters Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(otel): assert OTLP HTTP TLS behavior against a real TLS sink Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(otel): assert rejected exports by outcome, not by exception type Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): honor SSL_CERT_FILE and ssl_verify in the v2 OTLP HTTP exporters Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(otel): hoist otlp_tls imports and type the TLS sink fixture Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): gate the OTLP TLS export test behind an otel_tls opt-in Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * docs(e2e): drop CONTRIBUTING.md edit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: yucheng --- .github/e2e-stack/up.sh | 29 +++- litellm/integrations/opentelemetry.py | 21 ++- .../integrations/otel/plumbing/otlp_json.py | 11 +- .../integrations/otel/plumbing/otlp_tls.py | 41 +++++ .../integrations/otel/plumbing/providers.py | 13 ++ tests/e2e/conftest.py | 6 + tests/e2e/e2e_config.py | 2 + tests/e2e/logging/test_otel_trace_e2e.py | 40 ++++- tests/e2e/pytest.ini | 1 + tests/test_litellm/integrations/conftest.py | 96 +++++++++++ .../otel/test_otel_v2_components.py | 105 +++++++++++- .../integrations/test_opentelemetry.py | 152 +++++++++++++++++- 12 files changed, 496 insertions(+), 21 deletions(-) create mode 100644 litellm/integrations/otel/plumbing/otlp_tls.py create mode 100644 tests/test_litellm/integrations/conftest.py diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh index 928b58e93bb..931b5eb2169 100755 --- a/.github/e2e-stack/up.sh +++ b/.github/e2e-stack/up.sh @@ -24,6 +24,7 @@ DATABASE_USER="${E2E_DATABASE_USER:-litellm}" DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}" DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}" JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}" +JAEGER_OTLP_TLS_PORT="${E2E_JAEGER_OTLP_TLS_PORT:-4319}" JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}" KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}" @@ -122,7 +123,7 @@ SERVER_ENV=( "CONFIG_FILE_PATH=${CONFIG_PATH}" "STORE_MODEL_IN_DB=True" "OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf" - "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}" + "OTEL_EXPORTER_OTLP_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}" "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem" "PYTHONPATH=${REPO_ROOT}" "JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs" @@ -147,16 +148,12 @@ start_server() { echo $! > "${PIDS_DIR}/${name}.pid" } -start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}" -start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}" -start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}" - if [[ "$(uname)" == "Linux" ]]; then NGINX_UPSTREAM_HOST=127.0.0.1 NGINX_DOCKER_ARGS=(--network host) else NGINX_UPSTREAM_HOST=host.docker.internal - NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}") + NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}" -p "${JAEGER_OTLP_TLS_PORT}:${JAEGER_OTLP_TLS_PORT}") fi cat > "${STACK_DIR}/nginx.conf" </dev/null 2>&1 || true docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \ - -v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" "${NGINX_IMAGE}" >/dev/null + -v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" \ + -v "${CERTS_DIR}:/certs:ro" "${NGINX_IMAGE}" >/dev/null + +wait_for "Jaeger OTLP TLS listener" \ + "curl -sS --cacert ${CERTS_DIR}/ca.crt https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}/ -o /dev/null -w '%{http_code}' | grep -qE '^[2345]'" + +start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}" +start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}" +start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}" wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300 wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300 @@ -206,6 +220,7 @@ LITELLM_MASTER_KEY=${MASTER_KEY} REDIS_HOST=127.0.0.1 REDIS_PORT=${REDIS_PORT} E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT} +E2E_OTEL_EXPORTER_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT} E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT} E2E_KEYCLOAK_ADMIN_USER=admin E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index feaeaec27a3..749f0ce4fcb 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -26,6 +26,7 @@ from litellm.integrations.otel.model.baggage import promoted_metadata from litellm.integrations.otel.model.db_endpoint import db_span_attributes from litellm.integrations.otel.model.metadata import flatten_metadata from litellm.integrations.otel.model.semconv import Metric +from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.secret_redaction import redact_string @@ -99,6 +100,7 @@ _MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256 # Dedicated so a slow exporter shutdown cannot starve the shared logging executor. _PROVIDER_SHUTDOWN_EXECUTOR: Final = ThreadPoolExecutor(max_workers=4, thread_name_prefix="OtelProviderShutdown") + LITELLM_TRACER_NAME: Final = os.getenv("OTEL_TRACER_NAME", "litellm") LITELLM_METER_NAME: Final = os.getenv("LITELLM_METER_NAME", "litellm") LITELLM_LOGGER_NAME: Final = os.getenv("LITELLM_LOGGER_NAME", "litellm") @@ -3090,8 +3092,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): otel_exporter, ) normalized_endpoint = self._normalize_otel_endpoint(otel_endpoint, "traces") + tls: Final = resolve_otlp_http_tls("TRACES") return BatchSpanProcessor( - OTLPSpanExporterHTTP(endpoint=normalized_endpoint, headers=_split_otel_headers), + OTLPSpanExporterHTTP( + endpoint=normalized_endpoint, + headers=_split_otel_headers, + certificate_file=tls.certificate_file, + session=tls.session, + ), ) elif otel_exporter == "otlp_grpc" or otel_exporter == "grpc": try: @@ -3172,7 +3180,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): self.OTEL_EXPORTER, normalized_endpoint, ) - return OTLPLogExporter(endpoint=normalized_endpoint, headers=_split_otel_headers) + tls: Final = resolve_otlp_http_tls("LOGS") + return OTLPLogExporter( + endpoint=normalized_endpoint, + headers=_split_otel_headers, + certificate_file=tls.certificate_file, + session=tls.session, + ) elif self.OTEL_EXPORTER == "otlp_grpc" or self.OTEL_EXPORTER == "grpc": try: from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( @@ -3235,9 +3249,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): OTLPMetricExporter, ) + tls: Final = resolve_otlp_http_tls("METRICS") exporter = OTLPMetricExporter( endpoint=normalized_endpoint, headers=_split_otel_headers, + certificate_file=tls.certificate_file, + session=tls.session, ) return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) diff --git a/litellm/integrations/otel/plumbing/otlp_json.py b/litellm/integrations/otel/plumbing/otlp_json.py index b4b659f1e01..bc6d7d435b0 100644 --- a/litellm/integrations/otel/plumbing/otlp_json.py +++ b/litellm/integrations/otel/plumbing/otlp_json.py @@ -10,6 +10,7 @@ from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import Final, TypeAlias +import requests from google.protobuf.json_format import MessageToDict from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter @@ -62,8 +63,14 @@ def encode_spans_json(spans: Sequence[ReadableSpan]) -> bytes: class OTLPJsonSpanExporter(OTLPSpanExporter): - def __init__(self, endpoint: str | None, headers: dict[str, str]) -> None: # mutable-ok: SDK __init__ takes Dict - super().__init__(endpoint=endpoint, headers=headers) + def __init__( + self, + endpoint: str | None, + headers: dict[str, str], # mutable-ok: SDK __init__ takes Dict + certificate_file: str | None = None, + session: "requests.Session | None" = None, + ) -> None: + super().__init__(endpoint=endpoint, headers=headers, certificate_file=certificate_file, session=session) self._session.headers["Content-Type"] = JSON_CONTENT_TYPE def _serialize_spans(self, spans: Sequence[ReadableSpan]) -> bytes: diff --git a/litellm/integrations/otel/plumbing/otlp_tls.py b/litellm/integrations/otel/plumbing/otlp_tls.py new file mode 100644 index 00000000000..9de97e0c77a --- /dev/null +++ b/litellm/integrations/otel/plumbing/otlp_tls.py @@ -0,0 +1,41 @@ +import os +from dataclasses import dataclass +from typing import Final, Literal + +import requests +from requests.adapters import HTTPAdapter + + +@dataclass(frozen=True, slots=True) +class OtlpHttpTls: + certificate_file: str | None + session: requests.Session | None + + +class _NoVerifyAdapter(HTTPAdapter): + def cert_verify( + self, + conn: object, + url: str, + verify: bool | str, + cert: str | tuple[str, str] | None, + ) -> None: + super().cert_verify( # pyright: ignore[reportUnknownMemberType] # requests stubs omit HTTPAdapter.cert_verify + conn, url, False, cert + ) + + +def resolve_otlp_http_tls(signal: Literal["TRACES", "METRICS", "LOGS"]) -> OtlpHttpTls: + if os.getenv(f"OTEL_EXPORTER_OTLP_{signal}_CERTIFICATE") or os.getenv("OTEL_EXPORTER_OTLP_CERTIFICATE"): + return OtlpHttpTls(certificate_file=None, session=None) + + from litellm.llms.custom_httpx.http_handler import get_ssl_verify + + verify: Final = get_ssl_verify() + if verify is False: + session: Final = requests.Session() + session.mount("https://", _NoVerifyAdapter()) + return OtlpHttpTls(certificate_file=None, session=session) + if isinstance(verify, str): + return OtlpHttpTls(certificate_file=verify, session=None) + return OtlpHttpTls(certificate_file=None, session=None) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 2c4375ce5f7..d52736a1303 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -58,6 +58,7 @@ from litellm.integrations.otel.plumbing.context import ( request_destinations, suppressed_backends, ) +from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls if TYPE_CHECKING: from opentelemetry.metrics import Meter @@ -193,18 +194,24 @@ def _exporter_from_spec(spec: ExporterSpec) -> SpanExporter: if kind in _OTLP_HTTP_JSON_KINDS: from litellm.integrations.otel.plumbing.otlp_json import OTLPJsonSpanExporter + tls: Final = resolve_otlp_http_tls("TRACES") return OTLPJsonSpanExporter( endpoint=spec.traces_endpoint or _otlp_traces_endpoint(spec.endpoint), headers=parse_headers(spec.headers), + certificate_file=tls.certificate_file, + session=tls.session, ) if kind in _OTLP_HTTP_KINDS: from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( OTLPSpanExporter as HTTPExporter, ) + http_tls: Final = resolve_otlp_http_tls("TRACES") return HTTPExporter( endpoint=spec.traces_endpoint or _otlp_traces_endpoint(spec.endpoint), headers=parse_headers(spec.headers), + certificate_file=http_tls.certificate_file, + session=http_tls.session, ) if kind in _OTLP_GRPC_KINDS: from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( @@ -902,9 +909,12 @@ def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader": OTLPMetricExporter as HTTPMetricExporter, ) + tls: Final = resolve_otlp_http_tls("METRICS") exporter: Any = HTTPMetricExporter( endpoint=_otlp_metrics_endpoint(config.endpoint), headers=parse_headers(config.headers), + certificate_file=tls.certificate_file, + session=tls.session, ) elif kind in ("otlp_grpc", "grpc"): try: @@ -962,9 +972,12 @@ def build_log_exporter(config: OpenTelemetryV2Config) -> LogExporter: OTLPLogExporter as HTTPLogExporter, ) + tls: Final = resolve_otlp_http_tls("LOGS") return HTTPLogExporter( endpoint=_otlp_logs_endpoint(config.endpoint), headers=parse_headers(config.headers), + certificate_file=tls.certificate_file, + session=tls.session, ) if kind in ("otlp_grpc", "grpc"): try: diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index ca0fbd84c35..a2398ef7c3c 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -30,6 +30,7 @@ from e2e_config import ( FIXTURE_MODE_RAW, MANAGED_FILES_OPT_IN_ENV, MCP_OAUTH_LIVE_OPT_IN_ENV, + OTEL_TLS_OPT_IN_ENV, OTEL_V2_OPT_IN_ENV, PROMPT_CACHING_OPT_IN_ENV, PROVIDER_EDGE_HOST_OPT_IN_ENV, @@ -63,6 +64,7 @@ OPT_IN_MARKERS: Final = MappingProxyType( "mcp_oauth_live": MCP_OAUTH_LIVE_OPT_IN_ENV, "provider_edge_host": PROVIDER_EDGE_HOST_OPT_IN_ENV, "otel_v2": OTEL_V2_OPT_IN_ENV, + "otel_tls": OTEL_TLS_OPT_IN_ENV, } ) @@ -156,6 +158,10 @@ def pytest_configure(config: pytest.Config) -> None: "markers", "otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set", ) + config.addinivalue_line( + "markers", + "otel_tls: needs a stack whose gateway exports OTLP over TLS signed by the CA in SSL_CERT_FILE; deselected unless E2E_OTEL_EXPORTER_ENDPOINT is set", + ) def pytest_sessionstart(session: pytest.Session) -> None: diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 71f01834788..77395a066ac 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -58,6 +58,7 @@ LINEAR_READONLY_TOOL: Final = "list_teams" # as listed by tools/list on mcp.lin # service in docker-compose.yml maps it to host 16686). Trace-completeness tests # read exported spans back through it. OTEL_QUERY_URL = os.environ.get("E2E_OTEL_QUERY_URL", "http://localhost:16686").rstrip("/") +OTEL_EXPORTER_ENDPOINT = os.environ.get("E2E_OTEL_EXPORTER_ENDPOINT", "") # Real-DataDog read-back (no local sink - destination fakes cannot be deployed # on the cluster): the proxy delivers with DD_API_KEY as in production, and the @@ -148,6 +149,7 @@ CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM" MCP_OAUTH_LIVE_OPT_IN_ENV: Final = "E2E_MCP_OAUTH_LIVE" PROVIDER_EDGE_HOST_OPT_IN_ENV: Final = "E2E_PROVIDER_EDGE_HOST_REACHABLE" OTEL_V2_OPT_IN_ENV: Final = "E2E_OTEL_V2" +OTEL_TLS_OPT_IN_ENV: Final = "E2E_OTEL_EXPORTER_ENDPOINT" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) diff --git a/tests/e2e/logging/test_otel_trace_e2e.py b/tests/e2e/logging/test_otel_trace_e2e.py index 9f08fa6c4e7..8d154ca0837 100644 --- a/tests/e2e/logging/test_otel_trace_e2e.py +++ b/tests/e2e/logging/test_otel_trace_e2e.py @@ -12,21 +12,24 @@ commit 1bd603d1ac). Both halves of the contract are asserted: the recorded state (the proxy reports the OTEL v2 logger active via /health/readiness/details) and the enforced behavior (the complete span tree at the destination, read back through the -destination's own query API - never proxy-side "export succeeded" logs). +destination's own query API - never proxy-side "export succeeded" logs). The +TLS coverage requires the stack to export OTLP over HTTPS with a certificate +signed by the CA in SSL_CERT_FILE, and treats a missing or plaintext endpoint +as a stack misconfiguration rather than skipping the test. """ from __future__ import annotations import time +from typing import Final import pytest -from pydantic import BaseModel, ConfigDict, ValidationError - -from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, unique_marker +from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, OTEL_EXPORTER_ENDPOINT, unique_marker from lifecycle import ResourceManager from logging_client import INVALID_UPSTREAM_API_KEY, LoggingClient, first_ok, readiness_details_body from models import LiteLLMParamsBody from otel_client import JaegerSpan, JaegerTrace, OtelReader +from pydantic import BaseModel, ConfigDict, ValidationError pytestmark = pytest.mark.e2e @@ -312,6 +315,35 @@ class TestOtelTraceCompleteness: ) _assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}") + @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"]) + @pytest.mark.otel_tls + def test_otel_export_over_tls_with_internal_ca_reaches_destination( + self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager + ) -> None: + _assert_otel_destination_configured(client) + assert OTEL_EXPORTER_ENDPOINT.startswith("https://"), ( + "the stack must export OTLP over TLS signed by the CA in SSL_CERT_FILE " + "(E2E_OTEL_EXPORTER_ENDPOINT) for this test to prove anything; a " + "missing or plaintext value is a stack misconfiguration" + ) + + route: Final = "/chat/completions" + key: Final = client.key_with_alias(f"otel-trace-tls-{unique_marker()}", models=[MODEL]) + resources.defer(lambda: client.delete_key(key)) + + marker: Final = unique_marker() + outcome: Final = first_ok( + client, lambda: client.chat_raw(key, MODEL, f"reply with one word {marker}", max_tokens=16) + ) + assert outcome.call_id is not None, "success response must carry x-litellm-call-id" + + hits: Final = otel_reader.poll_traces_for_call( + call_id=outcome.call_id, + settled_names=_settled_names(route=route, genai_span=f"chat {MODEL}"), + settled_prefixes={DB_SPAN_PREFIX}, + ) + _assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}") + @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"]) def test_messages_exports_complete_trace( self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index 6f9f57d333e..a77459d3683 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -15,3 +15,4 @@ markers = mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless E2E_MCP_OAUTH_LIVE is set provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set + otel_tls: needs a stack whose gateway exports OTLP over TLS signed by the CA in SSL_CERT_FILE; deselected unless E2E_OTEL_EXPORTER_ENDPOINT is set diff --git a/tests/test_litellm/integrations/conftest.py b/tests/test_litellm/integrations/conftest.py new file mode 100644 index 00000000000..adc8e36e0af --- /dev/null +++ b/tests/test_litellm/integrations/conftest.py @@ -0,0 +1,96 @@ +import functools +import http.server +import ipaddress +import queue +import ssl +import threading +from collections.abc import Iterator +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Final + +import pytest + + +@dataclass(frozen=True, slots=True) +class TlsSink: + url: str + certificate_path: str + received: "queue.Queue[str]" + + +class _RecordingOtelHandler(http.server.BaseHTTPRequestHandler): + def __init__(self, *args: object, received: "queue.Queue[str]", **kwargs: object) -> None: + self._received: Final = received + super().__init__(*args, **kwargs) + + def do_POST(self) -> None: + length: Final = int(self.headers.get("Content-Length") or 0) + if length: + self.rfile.read(length) + self._received.put(self.path) + self.send_response(200) + self.send_header("Content-Type", "application/x-protobuf") + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, format: str, *args: object) -> None: + pass + + +def write_self_signed_cert(directory: Path, stem: str) -> tuple[Path, Path]: + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.x509.oid import NameOID + + key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")]) + certificate: Final = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.now(timezone.utc) - timedelta(minutes=1)) + .not_valid_after(datetime.now(timezone.utc) + timedelta(hours=1)) + .add_extension( + x509.SubjectAlternativeName([x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), + critical=False, + ) + .sign(key, hashes.SHA256()) + ) + certificate_path: Final = directory / f"{stem}.crt" + certificate_path.write_bytes(certificate.public_bytes(serialization.Encoding.PEM)) + key_path: Final = directory / f"{stem}.key" + key_path.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + serialization.NoEncryption(), + ) + ) + return certificate_path, key_path + + +@pytest.fixture +def tls_sink(tmp_path: Path) -> Iterator[TlsSink]: + certificate_path, key_path = write_self_signed_cert(tmp_path, "sink") + received: queue.Queue[str] = queue.Queue() + context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(str(certificate_path), str(key_path)) + server: Final = http.server.ThreadingHTTPServer( + ("127.0.0.1", 0), functools.partial(_RecordingOtelHandler, received=received) + ) + server.socket = context.wrap_socket(server.socket, server_side=True) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield TlsSink( + url=f"https://127.0.0.1:{server.server_port}", + certificate_path=str(certificate_path), + received=received, + ) + server.shutdown() + server.server_close() + thread.join(timeout=5) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 0c95049ce05..79747ac9956 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -2,14 +2,17 @@ baggage helpers, metrics, the typed coercion helpers, mapper branches, span-name builders, and the registry validator's failure paths. Needs the OTel SDK.""" +import contextlib import json import threading +import time from collections.abc import Iterator from contextvars import Context as ContextVarContext from dataclasses import replace from http.server import BaseHTTPRequestHandler, HTTPServer, ThreadingHTTPServer import pytest +import requests pytest.importorskip("opentelemetry") @@ -18,6 +21,9 @@ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( # noqa: ) from opentelemetry import baggage # noqa: E402 from opentelemetry.context import attach, detach # noqa: E402 +from opentelemetry._logs.severity import SeverityNumber # noqa: E402 +from opentelemetry.sdk._logs import LogData, LogRecord # noqa: E402 +from opentelemetry.sdk._logs.export import LogExportResult # noqa: E402 from opentelemetry.sdk.metrics import MeterProvider # noqa: E402 from opentelemetry.sdk.metrics.export import InMemoryMetricReader # noqa: E402 from opentelemetry.sdk.trace import TracerProvider # noqa: E402 @@ -29,11 +35,14 @@ from opentelemetry.sdk.trace.export import ( # noqa: E402 from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402 InMemorySpanExporter, ) -from opentelemetry.trace import SpanKind, get_current_span # noqa: E402 +from opentelemetry.sdk.util.instrumentation import InstrumentationScope # noqa: E402 +from opentelemetry.trace import SpanKind, TraceFlags, get_current_span # noqa: E402 from opentelemetry.trace.propagation.tracecontext import ( # noqa: E402 TraceContextTextMapPropagator, ) +import litellm # noqa: E402 +from conftest import TlsSink # noqa: E402 from litellm.integrations.otel.plumbing import context as ctx_mod # noqa: E402 from litellm.integrations.otel.plumbing import providers # noqa: E402 from litellm.integrations.otel.model.config import OpenTelemetryV2Config # noqa: E402 @@ -1414,3 +1423,97 @@ def test_genai_mapper_guardrail_cost_in_spend_attr(): billed = dict(entry) del billed["guardrail_cost_in_spend"] assert LiteLLM.GUARDRAIL_COST_IN_SPEND not in GenAIMapper().map(GuardrailSpanData.from_logging_entry(billed)) + + +def _isolate_v2_otlp_tls_env(monkeypatch: pytest.MonkeyPatch) -> None: + for key in ( + "SSL_VERIFY", + "SSL_CERT_FILE", + "OTEL_EXPORTER_OTLP_CERTIFICATE", + "OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", + "OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE", + "OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE", + ): + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TIMEOUT", "2") + monkeypatch.setattr(litellm, "ssl_verify", True) + + +def test_v2_otlp_http_span_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None: + _isolate_v2_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path) + cfg = OpenTelemetryV2Config(exporter="otlp_http", endpoint=tls_sink.url) + _export_one_span(cfg) + assert tls_sink.received.get(timeout=5) == "/v1/traces" + + +def test_v2_http_json_span_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None: + _isolate_v2_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path) + cfg = OpenTelemetryV2Config(exporter="http/json", endpoint=tls_sink.url) + _export_one_span(cfg) + assert tls_sink.received.get(timeout=5) == "/v1/traces" + + +def test_v2_otlp_http_metric_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None: + _isolate_v2_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path) + cfg = OpenTelemetryV2Config(exporter="otlp_http", endpoint=tls_sink.url) + reader = providers.build_metric_reader(cfg) + provider = MeterProvider(metric_readers=[reader]) + try: + provider.get_meter("v2-tls-test").create_counter("tls_export_test").add(1) + assert provider.force_flush(), "metric flush failed" + assert tls_sink.received.get(timeout=5) == "/v1/metrics" + finally: + provider.shutdown() + + +def test_v2_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None: + _isolate_v2_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path) + cfg = OpenTelemetryV2Config(exporter="otlp_http", endpoint=tls_sink.url) + exporter = providers.build_log_exporter(cfg) + try: + record = LogRecord( + timestamp=int(time.time() * 1e9), + observed_timestamp=int(time.time() * 1e9), + trace_id=0, + span_id=0, + trace_flags=TraceFlags(0), + severity_number=SeverityNumber.INFO, + body="v2-tls-test", + ) + log_data = LogData(log_record=record, instrumentation_scope=InstrumentationScope("v2-tls-test")) + result = exporter.export([log_data]) + assert result is LogExportResult.SUCCESS, f"log export failed: {result}" + assert tls_sink.received.get(timeout=5) == "/v1/logs" + finally: + exporter.shutdown() + + +def test_v2_otlp_http_export_skips_verification_when_ssl_verify_false( + monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink +) -> None: + _isolate_v2_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_VERIFY", "false") + cfg = OpenTelemetryV2Config(exporter="otlp_http", endpoint=tls_sink.url) + _export_one_span(cfg) + assert tls_sink.received.get(timeout=5) == "/v1/traces" + + +def test_v2_otlp_http_export_rejects_untrusted_collector_by_default( + monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink +) -> None: + + + _isolate_v2_otlp_tls_env(monkeypatch) + cfg = OpenTelemetryV2Config(exporter="otlp_http", endpoint=tls_sink.url) + provider = providers.build_tracer_provider(cfg) + provider.get_tracer("probe").start_span("probe").end() + try: + with contextlib.suppress(requests.exceptions.SSLError): + provider.force_flush() + assert tls_sink.received.empty(), "sink received a request it should never have trusted" + finally: + provider.shutdown() diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index bea9a38e9dd..974961f2eb5 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -1,5 +1,6 @@ import asyncio import concurrent.futures +import contextlib import gc import json import os @@ -9,21 +10,30 @@ import time import unittest import weakref from datetime import datetime, timedelta, timezone +from pathlib import Path from types import MappingProxyType -from parameterized import parameterized +from typing import Final from unittest.mock import MagicMock, patch +import pytest + # Adds the grandparent directory to sys.path to allow importing project modules from opentelemetry import trace -from opentelemetry.sdk._logs import LogData +from opentelemetry._logs.severity import SeverityNumber +from opentelemetry.sdk._logs import LogData, LogRecord from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider -from opentelemetry.sdk._logs.export import InMemoryLogExporter, SimpleLogRecordProcessor +from opentelemetry.sdk._logs.export import InMemoryLogExporter, LogExportResult, SimpleLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import InMemoryMetricReader, MetricsData -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult +from opentelemetry.sdk.util.instrumentation import InstrumentationScope from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from parameterized import parameterized +import requests + +from conftest import TlsSink, write_self_signed_cert import litellm from litellm.integrations import opentelemetry as otel_module from litellm.integrations.opentelemetry import ( @@ -2081,6 +2091,138 @@ class TestOpenTelemetryEndpointNormalization(unittest.TestCase): self.assertEqual(traces, "http://collector:4318/v1/traces") +def _isolate_otlp_tls_env(monkeypatch: pytest.MonkeyPatch) -> None: + for key in ( + "SSL_VERIFY", + "SSL_CERT_FILE", + "OTEL_EXPORTER_OTLP_CERTIFICATE", + "OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", + "OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE", + "OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE", + ): + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TIMEOUT", "2") + monkeypatch.setattr(litellm, "ssl_verify", True) + + +def _ended_span() -> tuple[TracerProvider, ReadableSpan]: + provider: Final = TracerProvider() + span = provider.get_tracer(__name__).start_span("tls-export-test") + span.end() + return provider, span + + +def _assert_export_rejected(processor, span: ReadableSpan, sink: TlsSink) -> None: + with contextlib.suppress(requests.exceptions.SSLError): + result: Final = processor.span_exporter.export([span]) + assert result is SpanExportResult.FAILURE, f"rejected export must report failure, got {result}" + assert sink.received.empty(), "sink received a request it should never have trusted" + + +def _otlp_http_otel(endpoint: str) -> OpenTelemetry: + return OpenTelemetry(config=OpenTelemetryConfig(exporter="otlp_http", endpoint=endpoint)) + + +def test_otlp_http_span_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None: + _isolate_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path) + otel: Final = _otlp_http_otel(tls_sink.url) + processor: Final = otel._get_span_processor() + provider, span = _ended_span() + try: + result: Final = processor.span_exporter.export([span]) + assert result is SpanExportResult.SUCCESS, f"span export failed: {result}" + assert tls_sink.received.get(timeout=5) == "/v1/traces" + finally: + processor.shutdown() + provider.shutdown() + + +def test_otlp_http_metric_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None: + _isolate_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path) + otel: Final = _otlp_http_otel(tls_sink.url) + reader: Final = otel._get_metric_reader() + provider: Final = MeterProvider(metric_readers=[reader]) + try: + provider.get_meter(__name__).create_counter("tls_export_test").add(1) + assert provider.force_flush(), "metric flush failed" + assert tls_sink.received.get(timeout=5) == "/v1/metrics" + finally: + provider.shutdown() + + +def test_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None: + _isolate_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path) + otel: Final = _otlp_http_otel(tls_sink.url) + exporter: Final = otel._get_log_exporter() + try: + record: Final = LogRecord( + timestamp=int(time.time() * 1e9), + observed_timestamp=int(time.time() * 1e9), + trace_id=0, + span_id=0, + trace_flags=trace.TraceFlags(0), + severity_number=SeverityNumber.INFO, + body="tls-export-test", + ) + log_data: Final = LogData(log_record=record, instrumentation_scope=InstrumentationScope("tls-export-test")) + result: Final = exporter.export([log_data]) + assert result is LogExportResult.SUCCESS, f"log export failed: {result}" + assert tls_sink.received.get(timeout=5) == "/v1/logs" + finally: + exporter.shutdown() + + +def test_otlp_http_export_skips_verification_when_ssl_verify_false( + monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink +) -> None: + _isolate_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_VERIFY", "false") + otel: Final = _otlp_http_otel(tls_sink.url) + processor: Final = otel._get_span_processor() + provider, span = _ended_span() + try: + result: Final = processor.span_exporter.export([span]) + assert result is SpanExportResult.SUCCESS, f"span export failed: {result}" + assert tls_sink.received.get(timeout=5) == "/v1/traces" + finally: + processor.shutdown() + provider.shutdown() + + +def test_otlp_http_export_rejects_untrusted_collector_by_default( + monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink +) -> None: + _isolate_otlp_tls_env(monkeypatch) + otel: Final = _otlp_http_otel(tls_sink.url) + processor: Final = otel._get_span_processor() + provider, span = _ended_span() + try: + _assert_export_rejected(processor, span, tls_sink) + finally: + processor.shutdown() + provider.shutdown() + + +def test_otel_certificate_env_takes_precedence_over_ssl_cert_file( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, tls_sink: TlsSink +) -> None: + _isolate_otlp_tls_env(monkeypatch) + unrelated_certificate, _ = write_self_signed_cert(tmp_path, "unrelated") + monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_CERTIFICATE", str(unrelated_certificate)) + otel: Final = _otlp_http_otel(tls_sink.url) + processor: Final = otel._get_span_processor() + provider, span = _ended_span() + try: + _assert_export_rejected(processor, span, tls_sink) + finally: + processor.shutdown() + provider.shutdown() + + class TestOpenTelemetryProtocolSelection(unittest.TestCase): """Test suite for verifying correct exporter selection based on protocol""" From b73696a15d8385ee8a83b1289867306979a709d6 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:52:30 -0700 Subject: [PATCH 40/48] chore(prices): sync OpenAI prices: 3 models (#42557) gpt-5-2025-08-07: cache_read_input_token_cost_batches gpt-5-mini-2025-08-07: cache_read_input_token_cost_batches gpt-5-nano-2025-08-07: cache_read_input_token_cost_batches Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 3 +++ model_prices_and_context_window.json | 3 +++ 2 files changed, 6 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b8ca5ff27c7..b725094ae91 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -35613,6 +35613,7 @@ }, "gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", @@ -36038,6 +36039,7 @@ }, "gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", @@ -36138,6 +36140,7 @@ }, "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", "input_cost_per_token": 5e-08, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b8ca5ff27c7..b725094ae91 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -35613,6 +35613,7 @@ }, "gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", @@ -36038,6 +36039,7 @@ }, "gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", @@ -36138,6 +36140,7 @@ }, "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", "input_cost_per_token": 5e-08, From 0e6a288458640f4499e646c1d0ad445c14a85bfb Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:04:16 -0700 Subject: [PATCH 41/48] fix(ssrf): point the blocked-address remediation at litellm_settings (#42508) * fix(ssrf): point the blocked-address remediation at litellm_settings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(ssrf): pin the block message's named section to litellm_settings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(compaction): pin litellm.max_budget so leaked proxy budget cannot 401 the child auth Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: shivam Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../prompt_templates/image_handling.py | 2 +- litellm/litellm_core_utils/url_utils.py | 2 +- .../litellm_core_utils/test_image_handling.py | 2 +- .../proxy/proxy_server/test_proxy_config.py | 28 +++++++++++++++++++ .../proxy/test_native_compaction.py | 2 ++ 5 files changed, 33 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index c9933422cc3..c44c80bc0a0 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -82,7 +82,7 @@ def _rejected_image_fetch(url: str, verdict: SSRFError) -> "litellm.ImageFetchEr verbose_logger.warning("Image fetch of %s rejected before any request went out: %s", url, verdict) return litellm.ImageFetchError( "Error: Unable to fetch image from URL. The proxy could not resolve this host or its URL policy rejected it; " - f"an admin can check the proxy log and `user_url_allowed_hosts` in general_settings. url={url}" + f"an admin can check the proxy log and `user_url_allowed_hosts` in litellm_settings. url={url}" ) diff --git a/litellm/litellm_core_utils/url_utils.py b/litellm/litellm_core_utils/url_utils.py index b94d5a6886d..6c87ef4a3de 100644 --- a/litellm/litellm_core_utils/url_utils.py +++ b/litellm/litellm_core_utils/url_utils.py @@ -336,7 +336,7 @@ def validate_url(url: str) -> tuple[str, str]: raise SSRFError( f"URL targets a blocked address ({resolved_ip}). " "If this is a legitimate internal service, add the host " - "to `user_url_allowed_hosts` in general_settings." + "to `user_url_allowed_hosts` in litellm_settings." ) # For HTTPS with SSL verification enabled, TLS certificate validation diff --git a/tests/test_litellm/litellm_core_utils/test_image_handling.py b/tests/test_litellm/litellm_core_utils/test_image_handling.py index 8fa4bd6c14d..21e97e97357 100644 --- a/tests/test_litellm/litellm_core_utils/test_image_handling.py +++ b/tests/test_litellm/litellm_core_utils/test_image_handling.py @@ -427,7 +427,7 @@ async def test_async_inline_remote_media_cancels_the_other_fetches_when_one_fail _SSRF_VERDICTS = ( SSRFError( "URL targets a blocked address (10.0.0.8). If this is a legitimate internal service, " - "add the host to `user_url_allowed_hosts` in general_settings." + "add the host to `user_url_allowed_hosts` in litellm_settings." ), SSRFError("DNS resolution failed for 'internal.example': [Errno 8] nodename nor servname provided, or not known"), SSRFError("No addresses found for 'internal.example'"), diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 832ac6402d9..b47cce43dcc 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -2198,6 +2198,34 @@ async def test_ProxyConfig_load_config_wires_general_settings_url_validation(tmp litellm.provider_url_destination_allowed_hosts = original_provider_hosts +@pytest.mark.asyncio +async def test_ssrf_block_message_names_a_config_section_load_config_honors(tmp_path, monkeypatch): + """Regression for LIT-8349: the remediation in the SSRF block message must point at a section that works.""" + from litellm.litellm_core_utils.url_utils import SSRFError, validate_url + + monkeypatch.setattr(litellm, "user_url_allowed_hosts", []) + monkeypatch.setattr(litellm, "user_url_validation", True) + with pytest.raises(SSRFError) as blocked: + validate_url("http://10.96.3.245:10002/agent.json") + section_match = re.search(r"add the host to `user_url_allowed_hosts` in (\w+)\.", str(blocked.value)) + assert section_match is not None, str(blocked.value) + section: Final = section_match.group(1) + assert section == "litellm_settings", f"block message points admins at {section}, which the docs contradict" + + f = tmp_path / "c.yaml" + f.write_text(f"model_list: []\n{section}:\n user_url_allowed_hosts:\n - '10.96.3.245:10002'\n") + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + await ProxyConfig().load_config(router=None, config_file_path=str(f)) + + assert litellm.user_url_allowed_hosts == ["10.96.3.245:10002"], f"{section} did not apply the allowlist" + assert validate_url("http://10.96.3.245:10002/agent.json") == ( + "http://10.96.3.245:10002/agent.json", + "10.96.3.245:10002", + ) + + @pytest.mark.asyncio async def test_ProxyConfig_load_config_wires_config_reload_interval(tmp_path, monkeypatch): """general_settings.proxy_config_reload_interval_seconds must reach the proxy_server diff --git a/tests/test_litellm/proxy/test_native_compaction.py b/tests/test_litellm/proxy/test_native_compaction.py index d24624aacc5..9a6b4cdda65 100644 --- a/tests/test_litellm/proxy/test_native_compaction.py +++ b/tests/test_litellm/proxy/test_native_compaction.py @@ -7,6 +7,7 @@ import pytest from fastapi import FastAPI, Request from pydantic import TypeAdapter +import litellm from litellm.caching.caching import DualCache from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.initialize_dynamic_callback_params import inherit_message_logging_privacy @@ -112,6 +113,7 @@ async def test_real_proxy_child_auth_privacy_and_body_policy( }))) return asyncio.sleep(0, result=ModelResponse(id="private-summary", model="compactor")) + monkeypatch.setattr(litellm, "max_budget", 0) monkeypatch.setattr(proxy_server.app, "dependency_overrides", {}) monkeypatch.setattr(proxy_server, "master_key", "sk-master-fixture") monkeypatch.setattr(proxy_server, "prisma_client", object()) From e30f9f578c9af0dd48b837a318a6246ad6e2f23e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:05:33 -0700 Subject: [PATCH 42/48] fix(model_prices): registry audit 2026-09-22, absorb open pricing PRs (#42543) * fix(model_prices): registry audit 2026-09-22, absorb open pricing PRs Rolls the open registry-only PRs into one PR after re-verifying every value against the official provider source: OpenAI, Azure, Vertex AI and Gemini batch cache-read prices, Baseten model metadata from the authenticated inference API, Bedrock eu-west-2 Nemotron Super 3 pricing from the AWS offer file, and OpenRouter prices refreshed from the live OpenRouter models API Co-authored-by: sinksilk <785976238@qq.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): add groq/llama-guard-3-8b from the Groq model page Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): refresh openrouter deepseek aliases from live api and drop stale off-peak windows Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): resolve baseten merge conflicts against main Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: sinksilk <785976238@qq.com> --- ...odel_prices_and_context_window_backup.json | 376 ++++++++++++++---- model_prices_and_context_window.json | 376 ++++++++++++++---- tests/test_litellm/test_cost_calculator.py | 24 ++ 3 files changed, 608 insertions(+), 168 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b725094ae91..94786f250b0 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4417,7 +4417,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 6.875e-08 }, "azure/eu/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-08, @@ -4456,7 +4457,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.375e-08 }, "azure/eu/gpt-5.1": { "deprecation_date": "2027-05-15", @@ -4636,7 +4638,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 2.75e-09 }, "azure/eu/o1-2024-12-17": { "cache_read_input_token_cost": 8.25e-06, @@ -6094,7 +6097,8 @@ "input_cost_per_token_batches": 6.25e-07, "output_cost_per_token_batches": 5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_batches": 6.25e-08 }, "azure/gpt-5.1-chat-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, @@ -6282,7 +6286,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 6.25e-08 }, "azure/gpt-5-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -6460,7 +6465,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.25e-08 }, "azure/gpt-5-nano": { "deprecation_date": "2027-02-09", @@ -6533,7 +6539,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 2.5e-09 }, "azure/gpt-5-pro": { "deprecation_date": "2027-04-07", @@ -6822,7 +6829,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 8.75e-08 }, "azure/gpt-5.2-chat": { "cache_read_input_token_cost": 1.75e-07, @@ -7287,7 +7295,11 @@ "output_cost_per_token_flex": 7.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, + "cache_read_input_token_cost_batches": 1.3e-07, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05 }, "azure/us/gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.75e-07, @@ -7333,7 +7345,11 @@ "output_cost_per_token_batches": 8.25e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, + "cache_read_input_token_cost_batches": 1.43e-07, + "input_cost_per_token_above_272k_tokens_batches": 2.75e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.2375e-05 }, "azure/eu/gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.75e-07, @@ -7379,7 +7395,11 @@ "output_cost_per_token_batches": 8.25e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, + "cache_read_input_token_cost_batches": 1.43e-07, + "input_cost_per_token_above_272k_tokens_batches": 2.75e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.2375e-05 }, "azure/gpt-5.4-pro": { "deprecation_date": "2027-09-07", @@ -7477,7 +7497,9 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135 }, "azure/gpt-5.6": { "cache_creation_input_token_cost": 6.25e-06, @@ -8900,7 +8922,11 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05 }, "azure/us/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, @@ -9003,7 +9029,11 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, + "cache_read_input_token_cost_batches": 2.75e-07, + "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.475e-05 }, "azure/eu/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, @@ -9106,7 +9136,11 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, + "cache_read_input_token_cost_batches": 2.75e-07, + "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.475e-05 }, "azure/gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -9293,7 +9327,8 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_priority": 9e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "cache_read_input_token_cost_batches": 3.75e-08 }, "azure/gpt-5.4-nano": { "deprecation_date": "2027-09-21", @@ -9390,7 +9425,8 @@ "output_cost_per_token_batches": 6.25e-07, "output_cost_per_token_flex": 6.25e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "cache_read_input_token_cost_batches": 1e-08 }, "azure/gpt-image-1": { "cache_read_input_token_cost": 1.25e-06, @@ -10404,7 +10440,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 6.875e-08 }, "azure/us/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-08, @@ -10443,7 +10480,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.375e-08 }, "azure/us/gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5.5e-09, @@ -10479,7 +10517,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 2.75e-09 }, "azure/us/gpt-5.1": { "deprecation_date": "2027-05-15", @@ -12908,6 +12947,23 @@ "supports_tool_choice": true, "output_cost_per_token": 1.86e-06 }, + "bedrock/eu-west-2/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 2.3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.01e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, "litellm_provider": "bedrock", @@ -26782,7 +26838,8 @@ "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, - "web_search_billing_unit": "per_query" + "web_search_billing_unit": "per_query", + "cache_read_input_token_cost_batches": 1e-07 }, "gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -26867,7 +26924,8 @@ "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, - "web_search_billing_unit": "per_query" + "web_search_billing_unit": "per_query", + "cache_read_input_token_cost_batches": 2.5e-08 }, "gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, @@ -26946,7 +27004,8 @@ "supports_response_schema": false, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.25e-08 }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, @@ -27055,7 +27114,8 @@ }, "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_audio_token_batches": 2.5e-07 + "input_cost_per_audio_token_batches": 2.5e-07, + "cache_read_input_token_cost_batches": 1.25e-08 }, "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", @@ -27112,7 +27172,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 1.5e-08 }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, @@ -27147,7 +27208,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "cache_read_input_token_cost_batches": 1e-07 }, "gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, @@ -27880,7 +27942,8 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_read_input_token_cost_batches": 7.5e-08 }, "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -27937,7 +28000,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "vertex_ai/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, @@ -27995,7 +28059,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "vertex_ai/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, @@ -28053,7 +28118,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "vertex_ai/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, @@ -30385,7 +30451,8 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_read_input_token_cost_batches": 7.5e-08 }, "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -30442,7 +30509,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, @@ -30500,7 +30568,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, @@ -30558,7 +30627,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, @@ -35661,7 +35731,8 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_batches": 6.25e-08 }, "gpt-5-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -36087,7 +36158,8 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_batches": 1.25e-08 }, "gpt-5-nano": { "cache_read_input_token_cost": 5e-09, @@ -36186,7 +36258,8 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_batches": 2.5e-09 }, "gpt-image-1": { "cache_read_input_token_cost": 1.25e-06, @@ -36809,6 +36882,15 @@ "supports_response_schema": false, "supports_tool_choice": true }, + "groq/llama-guard-3-8b": { + "input_cost_per_token": 2e-07, + "litellm_provider": "groq", + "max_input_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2e-07, + "source": "https://console.groq.com/docs/model/llama-guard-3-8b" + }, "groq/gemma-7b-it": { "deprecation_date": "2024-12-18", "input_cost_per_token": 5e-08, @@ -43157,6 +43239,28 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "openrouter/anthropic/claude-opus-5.5": { + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "openrouter/bytedance/ui-tars-1.5-7b": { "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-07, @@ -43328,21 +43432,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.31074e-07, + "input_cost_per_token": 8.92272e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.862148e-06, + "output_cost_per_token": 1.784544e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.75895e-08, + "cache_read_input_token_cost": 7.4356e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -52043,7 +52147,8 @@ "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05, "supports_reasoning": false, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "cache_read_input_token_cost_batches": 1e-07 }, "vertex_ai/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -52080,7 +52185,8 @@ "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, "supports_reasoning": false, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "cache_read_input_token_cost_batches": 2.5e-08 }, "vertex_ai/gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, @@ -52135,7 +52241,8 @@ "supports_response_schema": false, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.25e-08 }, "vertex_ai/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, @@ -52245,7 +52352,8 @@ }, "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_audio_token_batches": 2.5e-07 + "input_cost_per_audio_token_batches": 2.5e-07, + "cache_read_input_token_cost_batches": 1.25e-08 }, "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", @@ -52303,7 +52411,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 1.5e-08 }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, @@ -52319,7 +52428,8 @@ "output_cost_per_image_token": 0.00012, "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "cache_read_input_token_cost_batches": 1e-07 }, "vertex_ai/imagegeneration@006": { "deprecation_date": "2025-09-24", @@ -69143,13 +69253,13 @@ "supports_web_search": false }, "openrouter/qwen/qwen3.6-27b": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 2e-06, - "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 3.2e-07, + "output_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_output_tokens": 262140, + "max_tokens": 262140, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -73178,16 +73288,16 @@ "supports_web_search": true }, "openrouter/~anthropic/claude-opus-latest": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_1hr": 1e-05, - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 5e-06, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, "litellm_provider": "openrouter", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 2.5e-05, + "output_cost_per_token": 2e-05, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73222,15 +73332,14 @@ "supports_web_search": true }, "openrouter/~deepseek/deepseek-flash-latest": { - "cache_read_input_token_cost": 6e-09, - "input_cost_per_token": 3e-07, + "cache_read_input_token_cost": 3.6e-09, + "input_cost_per_token": 1.2e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, + "max_output_tokens": 943718, + "max_tokens": 943718, "mode": "chat", - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":1.5e-7,"output_cost_per_token":6e-7,"cache_read_input_token_cost":3e-9}, - "output_cost_per_token": 1.2e-06, + "output_cost_per_token": 4.8e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73243,15 +73352,14 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-pro-latest": { - "cache_read_input_token_cost": 4.4e-08, - "input_cost_per_token": 1.32e-06, + "cache_read_input_token_cost": 1.2726e-08, + "input_cost_per_token": 3.9996e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, + "max_output_tokens": 393216, + "max_tokens": 393216, "mode": "chat", - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":6.6e-7,"output_cost_per_token":0.00000198,"cache_read_input_token_cost":2.2e-8}, - "output_cost_per_token": 3.96e-06, + "output_cost_per_token": 1.19988e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73264,14 +73372,14 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-v4-flash-latest": { - "cache_read_input_token_cost": 1.6e-08, - "input_cost_per_token": 4e-08, + "cache_read_input_token_cost": 8e-09, + "input_cost_per_token": 3e-08, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 943718, "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 6.4e-07, + "output_cost_per_token": 8e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73334,13 +73442,13 @@ }, "openrouter/~moonshotai/kimi-latest": { "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "input_cost_per_token": 1.4989e-06, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 943718, "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1.0758e-05, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73378,19 +73486,19 @@ "supports_web_search": true }, "openrouter/~openai/gpt-luna-latest": { - "cache_creation_input_token_cost": 2.5e-07, - "cache_creation_input_token_cost_above_272k_tokens": 5e-07, - "cache_read_input_token_cost": 2e-08, - "cache_read_input_token_cost_above_272k_tokens": 4e-08, - "input_cost_per_token": 2e-07, - "input_cost_per_token_above_272k_tokens": 4e-07, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, "litellm_provider": "openrouter", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_above_272k_tokens": 1.8e-06, + "output_cost_per_token": 5e-07, + "output_cost_per_token_above_272k_tokens": 7.5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73496,14 +73604,14 @@ "supports_web_search": true }, "openrouter/~z-ai/glm-flash-latest": { - "cache_read_input_token_cost": 5e-08, - "input_cost_per_token": 1.5e-07, + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 7.5e-08, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 943718, "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 5e-07, + "output_cost_per_token": 2.5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -75959,6 +76067,106 @@ "supports_vision": true, "supports_web_search": true }, + "openrouter/openai/gpt-6-luna": { + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_above_272k_tokens": 7.5e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "openrouter/openai/gpt-6-luna-pro": { + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_above_272k_tokens": 7.5e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "openrouter/openai/gpt-6-sol": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_272k_tokens": 1.5e-05, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "openrouter/openai/gpt-6-sol-pro": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_272k_tokens": 1.5e-05, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "openrouter/openai/o3-mini:batch": { "cache_read_input_token_cost": 2.75e-07, "input_cost_per_token": 5.5e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b725094ae91..94786f250b0 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4417,7 +4417,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 6.875e-08 }, "azure/eu/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-08, @@ -4456,7 +4457,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.375e-08 }, "azure/eu/gpt-5.1": { "deprecation_date": "2027-05-15", @@ -4636,7 +4638,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 2.75e-09 }, "azure/eu/o1-2024-12-17": { "cache_read_input_token_cost": 8.25e-06, @@ -6094,7 +6097,8 @@ "input_cost_per_token_batches": 6.25e-07, "output_cost_per_token_batches": 5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_batches": 6.25e-08 }, "azure/gpt-5.1-chat-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, @@ -6282,7 +6286,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 6.25e-08 }, "azure/gpt-5-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -6460,7 +6465,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.25e-08 }, "azure/gpt-5-nano": { "deprecation_date": "2027-02-09", @@ -6533,7 +6539,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 2.5e-09 }, "azure/gpt-5-pro": { "deprecation_date": "2027-04-07", @@ -6822,7 +6829,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 8.75e-08 }, "azure/gpt-5.2-chat": { "cache_read_input_token_cost": 1.75e-07, @@ -7287,7 +7295,11 @@ "output_cost_per_token_flex": 7.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, + "cache_read_input_token_cost_batches": 1.3e-07, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05 }, "azure/us/gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.75e-07, @@ -7333,7 +7345,11 @@ "output_cost_per_token_batches": 8.25e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, + "cache_read_input_token_cost_batches": 1.43e-07, + "input_cost_per_token_above_272k_tokens_batches": 2.75e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.2375e-05 }, "azure/eu/gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.75e-07, @@ -7379,7 +7395,11 @@ "output_cost_per_token_batches": 8.25e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, + "cache_read_input_token_cost_batches": 1.43e-07, + "input_cost_per_token_above_272k_tokens_batches": 2.75e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.2375e-05 }, "azure/gpt-5.4-pro": { "deprecation_date": "2027-09-07", @@ -7477,7 +7497,9 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135 }, "azure/gpt-5.6": { "cache_creation_input_token_cost": 6.25e-06, @@ -8900,7 +8922,11 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05 }, "azure/us/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, @@ -9003,7 +9029,11 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, + "cache_read_input_token_cost_batches": 2.75e-07, + "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.475e-05 }, "azure/eu/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, @@ -9106,7 +9136,11 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, + "cache_read_input_token_cost_batches": 2.75e-07, + "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.475e-05 }, "azure/gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -9293,7 +9327,8 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_priority": 9e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "cache_read_input_token_cost_batches": 3.75e-08 }, "azure/gpt-5.4-nano": { "deprecation_date": "2027-09-21", @@ -9390,7 +9425,8 @@ "output_cost_per_token_batches": 6.25e-07, "output_cost_per_token_flex": 6.25e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "cache_read_input_token_cost_batches": 1e-08 }, "azure/gpt-image-1": { "cache_read_input_token_cost": 1.25e-06, @@ -10404,7 +10440,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 6.875e-08 }, "azure/us/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-08, @@ -10443,7 +10480,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.375e-08 }, "azure/us/gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5.5e-09, @@ -10479,7 +10517,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 2.75e-09 }, "azure/us/gpt-5.1": { "deprecation_date": "2027-05-15", @@ -12908,6 +12947,23 @@ "supports_tool_choice": true, "output_cost_per_token": 1.86e-06 }, + "bedrock/eu-west-2/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 2.3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.01e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, "litellm_provider": "bedrock", @@ -26782,7 +26838,8 @@ "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, - "web_search_billing_unit": "per_query" + "web_search_billing_unit": "per_query", + "cache_read_input_token_cost_batches": 1e-07 }, "gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -26867,7 +26924,8 @@ "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, - "web_search_billing_unit": "per_query" + "web_search_billing_unit": "per_query", + "cache_read_input_token_cost_batches": 2.5e-08 }, "gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, @@ -26946,7 +27004,8 @@ "supports_response_schema": false, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.25e-08 }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, @@ -27055,7 +27114,8 @@ }, "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_audio_token_batches": 2.5e-07 + "input_cost_per_audio_token_batches": 2.5e-07, + "cache_read_input_token_cost_batches": 1.25e-08 }, "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", @@ -27112,7 +27172,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 1.5e-08 }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, @@ -27147,7 +27208,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "cache_read_input_token_cost_batches": 1e-07 }, "gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, @@ -27880,7 +27942,8 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_read_input_token_cost_batches": 7.5e-08 }, "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -27937,7 +28000,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "vertex_ai/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, @@ -27995,7 +28059,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "vertex_ai/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, @@ -28053,7 +28118,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "vertex_ai/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, @@ -30385,7 +30451,8 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_read_input_token_cost_batches": 7.5e-08 }, "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -30442,7 +30509,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, @@ -30500,7 +30568,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, @@ -30558,7 +30627,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, @@ -35661,7 +35731,8 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_batches": 6.25e-08 }, "gpt-5-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -36087,7 +36158,8 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_batches": 1.25e-08 }, "gpt-5-nano": { "cache_read_input_token_cost": 5e-09, @@ -36186,7 +36258,8 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_batches": 2.5e-09 }, "gpt-image-1": { "cache_read_input_token_cost": 1.25e-06, @@ -36809,6 +36882,15 @@ "supports_response_schema": false, "supports_tool_choice": true }, + "groq/llama-guard-3-8b": { + "input_cost_per_token": 2e-07, + "litellm_provider": "groq", + "max_input_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2e-07, + "source": "https://console.groq.com/docs/model/llama-guard-3-8b" + }, "groq/gemma-7b-it": { "deprecation_date": "2024-12-18", "input_cost_per_token": 5e-08, @@ -43157,6 +43239,28 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "openrouter/anthropic/claude-opus-5.5": { + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "openrouter/bytedance/ui-tars-1.5-7b": { "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-07, @@ -43328,21 +43432,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.31074e-07, + "input_cost_per_token": 8.92272e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.862148e-06, + "output_cost_per_token": 1.784544e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.75895e-08, + "cache_read_input_token_cost": 7.4356e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -52043,7 +52147,8 @@ "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05, "supports_reasoning": false, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "cache_read_input_token_cost_batches": 1e-07 }, "vertex_ai/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -52080,7 +52185,8 @@ "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, "supports_reasoning": false, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "cache_read_input_token_cost_batches": 2.5e-08 }, "vertex_ai/gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, @@ -52135,7 +52241,8 @@ "supports_response_schema": false, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.25e-08 }, "vertex_ai/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, @@ -52245,7 +52352,8 @@ }, "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_audio_token_batches": 2.5e-07 + "input_cost_per_audio_token_batches": 2.5e-07, + "cache_read_input_token_cost_batches": 1.25e-08 }, "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", @@ -52303,7 +52411,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 1.5e-08 }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, @@ -52319,7 +52428,8 @@ "output_cost_per_image_token": 0.00012, "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "cache_read_input_token_cost_batches": 1e-07 }, "vertex_ai/imagegeneration@006": { "deprecation_date": "2025-09-24", @@ -69143,13 +69253,13 @@ "supports_web_search": false }, "openrouter/qwen/qwen3.6-27b": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 2e-06, - "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 3.2e-07, + "output_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_output_tokens": 262140, + "max_tokens": 262140, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -73178,16 +73288,16 @@ "supports_web_search": true }, "openrouter/~anthropic/claude-opus-latest": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_1hr": 1e-05, - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 5e-06, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, "litellm_provider": "openrouter", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 2.5e-05, + "output_cost_per_token": 2e-05, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73222,15 +73332,14 @@ "supports_web_search": true }, "openrouter/~deepseek/deepseek-flash-latest": { - "cache_read_input_token_cost": 6e-09, - "input_cost_per_token": 3e-07, + "cache_read_input_token_cost": 3.6e-09, + "input_cost_per_token": 1.2e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, + "max_output_tokens": 943718, + "max_tokens": 943718, "mode": "chat", - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":1.5e-7,"output_cost_per_token":6e-7,"cache_read_input_token_cost":3e-9}, - "output_cost_per_token": 1.2e-06, + "output_cost_per_token": 4.8e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73243,15 +73352,14 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-pro-latest": { - "cache_read_input_token_cost": 4.4e-08, - "input_cost_per_token": 1.32e-06, + "cache_read_input_token_cost": 1.2726e-08, + "input_cost_per_token": 3.9996e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, + "max_output_tokens": 393216, + "max_tokens": 393216, "mode": "chat", - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":6.6e-7,"output_cost_per_token":0.00000198,"cache_read_input_token_cost":2.2e-8}, - "output_cost_per_token": 3.96e-06, + "output_cost_per_token": 1.19988e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73264,14 +73372,14 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-v4-flash-latest": { - "cache_read_input_token_cost": 1.6e-08, - "input_cost_per_token": 4e-08, + "cache_read_input_token_cost": 8e-09, + "input_cost_per_token": 3e-08, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 943718, "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 6.4e-07, + "output_cost_per_token": 8e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73334,13 +73442,13 @@ }, "openrouter/~moonshotai/kimi-latest": { "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "input_cost_per_token": 1.4989e-06, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 943718, "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1.0758e-05, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73378,19 +73486,19 @@ "supports_web_search": true }, "openrouter/~openai/gpt-luna-latest": { - "cache_creation_input_token_cost": 2.5e-07, - "cache_creation_input_token_cost_above_272k_tokens": 5e-07, - "cache_read_input_token_cost": 2e-08, - "cache_read_input_token_cost_above_272k_tokens": 4e-08, - "input_cost_per_token": 2e-07, - "input_cost_per_token_above_272k_tokens": 4e-07, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, "litellm_provider": "openrouter", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_above_272k_tokens": 1.8e-06, + "output_cost_per_token": 5e-07, + "output_cost_per_token_above_272k_tokens": 7.5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73496,14 +73604,14 @@ "supports_web_search": true }, "openrouter/~z-ai/glm-flash-latest": { - "cache_read_input_token_cost": 5e-08, - "input_cost_per_token": 1.5e-07, + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 7.5e-08, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 943718, "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 5e-07, + "output_cost_per_token": 2.5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -75959,6 +76067,106 @@ "supports_vision": true, "supports_web_search": true }, + "openrouter/openai/gpt-6-luna": { + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_above_272k_tokens": 7.5e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "openrouter/openai/gpt-6-luna-pro": { + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_above_272k_tokens": 7.5e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "openrouter/openai/gpt-6-sol": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_272k_tokens": 1.5e-05, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "openrouter/openai/gpt-6-sol-pro": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_272k_tokens": 1.5e-05, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "openrouter/openai/o3-mini:batch": { "cache_read_input_token_cost": 2.75e-07, "input_cost_per_token": 5.5e-07, diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 7d2a04c500f..ecc01857f8c 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -4847,3 +4847,27 @@ def test_cost_per_token_bedrock_qwen3_next_uses_regional_entry_not_us_rate( assert prompt_usd == pytest.approx(prompt_tokens * regional["input_cost_per_token"]) assert completion_usd == pytest.approx(completion_tokens * regional["output_cost_per_token"]) + + +def test_cost_per_token_bedrock_nemotron_super_3_uses_eu_west_2_entry_not_us_rate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + regional_key: Final = "bedrock/eu-west-2/nvidia.nemotron-super-3-120b" + regional: Final = litellm.model_cost[regional_key] + us: Final = litellm.model_cost["nvidia.nemotron-super-3-120b"] + assert regional["input_cost_per_token"] != us["input_cost_per_token"] + assert regional["output_cost_per_token"] != us["output_cost_per_token"] + + prompt_tokens, completion_tokens = 1000, 500 + prompt_usd, completion_usd = cost_per_token( + model=regional_key, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + custom_llm_provider="bedrock", + ) + + assert prompt_usd == pytest.approx(prompt_tokens * regional["input_cost_per_token"]) + assert completion_usd == pytest.approx(completion_tokens * regional["output_cost_per_token"]) From bbc702cd8be9025c405e4789f37aa24ec45c7159 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:09:40 -0700 Subject: [PATCH 43/48] fix(gateway): expose /api/event_logging/batch on the gateway allowlist (#42572) * fix(gateway): expose /api/event_logging/batch on the gateway allowlist Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(gateway): route /api/event_logging to gateway pods in helm and terraform Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): stop max_budget leaking between proxy_server and native_compaction tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- gateway/routes/allowlist.py | 1 + helm/litellm/templates/ingress.yaml | 2 +- terraform/litellm/aws/locals.tf | 2 +- terraform/litellm/gcp/locals.tf | 2 +- tests/test_litellm/proxy/test_native_compaction.py | 1 + tests/test_litellm/proxy/test_proxy_server.py | 12 ++++++------ 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index fe58e2dd58c..c4a3d3f7473 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -131,6 +131,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset( "/redoc", "/test", "/debug/memory/summary", + "/api/event_logging/batch", } ) diff --git a/helm/litellm/templates/ingress.yaml b/helm/litellm/templates/ingress.yaml index e6717821a57..e9f7ed4ec3f 100644 --- a/helm/litellm/templates/ingress.yaml +++ b/helm/litellm/templates/ingress.yaml @@ -61,7 +61,7 @@ "/v1/fine-tuning" "/fine-tuning" "/v1/responses" "/responses" "/v1/threads" "/threads" "/v1/assistants" "/assistants" "/v1/vector_stores" "/vector_stores" "/v1/indexes" "/v1/models" "/models" "/openai" "/engines" - "/v1/messages" "/messages" "/v1/skills" "/v1/a2a" "/a2a" + "/v1/messages" "/messages" "/v1/skills" "/v1/a2a" "/a2a" "/api/event_logging" "/v1/rerank" "/v2/rerank" "/rerank" "/v1/ocr" "/ocr" "/v1/rag" "/rag" "/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search" "/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat" diff --git a/terraform/litellm/aws/locals.tf b/terraform/litellm/aws/locals.tf index 778d31642c1..b89bd486d02 100644 --- a/terraform/litellm/aws/locals.tf +++ b/terraform/litellm/aws/locals.tf @@ -74,7 +74,7 @@ locals { "/v1/models*", "/models*", "/openai/*", "/engines/*", "/v1/messages*", "/messages*", - "/v1/skills/*", "/v1/a2a/*", + "/v1/skills/*", "/v1/a2a/*", "/api/event_logging*", "/v1/rerank*", "/v2/rerank*", "/rerank*", "/v1/ocr*", "/ocr*", "/v1/rag/*", "/rag/*", diff --git a/terraform/litellm/gcp/locals.tf b/terraform/litellm/gcp/locals.tf index d263c781449..e82892b27cb 100644 --- a/terraform/litellm/gcp/locals.tf +++ b/terraform/litellm/gcp/locals.tf @@ -43,7 +43,7 @@ locals { "/v1/models*", "/models*", "/openai/*", "/engines/*", "/v1/messages*", "/messages*", - "/v1/skills/*", "/v1/a2a/*", + "/v1/skills/*", "/v1/a2a/*", "/api/event_logging*", "/v1/rerank*", "/v2/rerank*", "/rerank*", "/v1/ocr*", "/ocr*", "/v1/rag/*", "/rag/*", diff --git a/tests/test_litellm/proxy/test_native_compaction.py b/tests/test_litellm/proxy/test_native_compaction.py index 9a6b4cdda65..778c1715530 100644 --- a/tests/test_litellm/proxy/test_native_compaction.py +++ b/tests/test_litellm/proxy/test_native_compaction.py @@ -116,6 +116,7 @@ async def test_real_proxy_child_auth_privacy_and_body_policy( monkeypatch.setattr(litellm, "max_budget", 0) monkeypatch.setattr(proxy_server.app, "dependency_overrides", {}) monkeypatch.setattr(proxy_server, "master_key", "sk-master-fixture") + monkeypatch.setattr(litellm, "max_budget", 0.0) monkeypatch.setattr(proxy_server, "prisma_client", object()) monkeypatch.setattr(proxy_server, "user_api_key_cache", cache) monkeypatch.setattr(proxy_server, "llm_router", None) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 26bfd5c52bd..32b89885ac2 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -3200,7 +3200,7 @@ def test_normalize_datetime_for_sorting(): @pytest.mark.asyncio -async def test_add_proxy_budget_to_db_only_creates_user_no_keys(): +async def test_add_proxy_budget_to_db_only_creates_user_no_keys(monkeypatch: pytest.MonkeyPatch): """ Test that _add_proxy_budget_to_db only creates a user and no keys are added. @@ -3218,8 +3218,8 @@ async def test_add_proxy_budget_to_db_only_creates_user_no_keys(): from litellm.proxy.proxy_server import ProxyStartupEvent # Set up required litellm settings - litellm.budget_duration = "30d" - litellm.max_budget = 100.0 + monkeypatch.setattr(litellm, "budget_duration", "30d") + monkeypatch.setattr(litellm, "max_budget", 100.0) litellm_proxy_budget_name = "litellm-proxy-budget" @@ -3258,7 +3258,7 @@ async def test_add_proxy_budget_to_db_only_creates_user_no_keys(): @pytest.mark.asyncio -async def test_add_proxy_budget_to_db_backfills_budget_reset_at(): +async def test_add_proxy_budget_to_db_backfills_budget_reset_at(monkeypatch: pytest.MonkeyPatch): """ Test that _upsert_proxy_budget_with_reset_at_backfill issues a conditional update_many with `WHERE budget_reset_at IS NULL` to backfill the column on @@ -3276,8 +3276,8 @@ async def test_add_proxy_budget_to_db_backfills_budget_reset_at(): import litellm from litellm.proxy.proxy_server import ProxyStartupEvent - litellm.budget_duration = "30d" - litellm.max_budget = 100.0 + monkeypatch.setattr(litellm, "budget_duration", "30d") + monkeypatch.setattr(litellm, "max_budget", 100.0) litellm_proxy_budget_name = "litellm-proxy-budget" mock_prisma = MagicMock() From 12dffbafd6dce04a0ea81c93b35ff83f871027d6 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 16:10:49 -0500 Subject: [PATCH 44/48] fix(proxy): honor DATABASE_DISABLE_PREPARED_STATEMENTS in the litellm CLI (#42556) * fix(proxy): honor DATABASE_DISABLE_PREPARED_STATEMENTS in the litellm CLI Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): pooled pgbouncer url keeps a single pgbouncer=true when the upstream already carries it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): reject a malformed DATABASE_DISABLE_PREPARED_STATEMENTS even when the config already enables it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_cli.py | 7 +- tests/test_litellm/proxy/db/test_pgbouncer.py | 7 + tests/test_litellm/proxy/test_proxy_cli.py | 166 ++++++++++++++++++ 3 files changed, 179 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 78885461724..40140974198 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -1283,6 +1283,7 @@ def run_server( if os.getenv("DATABASE_URL", None) is not None or os.getenv("DIRECT_URL", None) is not None: from litellm.proxy.db.db_url_settings import ( + DISABLE_PREPARED_STATEMENTS_ENV_VAR, add_missing_query_params, idle_lifetime_params, reader_shareable_params, @@ -1305,12 +1306,16 @@ def run_server( sys.exit(1) from litellm.secret_managers.main import get_secret + env_disable_prepared_statements: Final = token_auth_flag_enabled( + os.getenv(DISABLE_PREPARED_STATEMENTS_ENV_VAR), env_var=DISABLE_PREPARED_STATEMENTS_ENV_VAR + ) + disable_prepared_statements: Final = db_disable_prepared_statements or env_disable_prepared_statements connection_url_params: Final = _build_db_connection_url_params( connection_limit=db_connection_pool_limit, pool_timeout=db_connection_timeout, connect_timeout=db_connect_timeout, socket_timeout=db_socket_timeout, - disable_prepared_statements=db_disable_prepared_statements, + disable_prepared_statements=disable_prepared_statements, extra_params=db_extra_connection_params, ) lifetime_params: Final = idle_lifetime_params(general_settings.get("database_max_idle_connection_lifetime")) diff --git a/tests/test_litellm/proxy/db/test_pgbouncer.py b/tests/test_litellm/proxy/db/test_pgbouncer.py index bf7df3077ea..c69c8d015a3 100644 --- a/tests/test_litellm/proxy/db/test_pgbouncer.py +++ b/tests/test_litellm/proxy/db/test_pgbouncer.py @@ -122,6 +122,13 @@ class TestPlanPgBouncer: "pgbouncer": "true", } + def test_an_upstream_that_already_disables_prepared_statements_gets_a_single_pgbouncer_flag(self): + pooled: Final = _plan("postgresql://app:pw@db/litellm?connection_limit=5&pgbouncer=true").pooled_url + assert urllib.parse.parse_qsl(urllib.parse.urlsplit(pooled).query) == [ + ("connection_limit", "5"), + ("pgbouncer", "true"), + ] + @pytest.mark.parametrize("hop_param", ["channel_binding=require", "gssencmode=require"]) def test_transport_params_for_the_postgres_hop_stay_off_the_plain_tcp_loopback_url(self, hop_param: str): pooled: Final = _plan(f"postgresql://app:pw@db/litellm?connection_limit=5&{hop_param}").pooled_url diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index a38470d1fdf..d2d71d6df05 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -1174,6 +1174,172 @@ class TestProxyInitializationHelpers: else: assert "pgbouncer" not in appended_params + @pytest.mark.parametrize( + "env_value, config_value, expect_pgbouncer", + [ + ("true", None, True), + ("1", None, True), + ("false", None, False), + (None, None, False), + ("true", False, True), + ("false", True, True), + ], + ) + @patch("subprocess.run") + @patch("atexit.register") + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") + @patch( + "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False + ) + def test_disable_prepared_statements_env_var_forwarded_to_url( + self, + mock_should_update, + mock_setup_db, + mock_atexit_register, + mock_subprocess_run, + env_value, + config_value, + expect_pgbouncer, + ): + from click.testing import CliRunner + + from litellm.proxy.proxy_cli import run_server + + runner = CliRunner() + mock_subprocess_run.return_value = MagicMock(returncode=0) + + general_settings = {"database_url": "postgresql://test:test@localhost:5432/test"} + if config_value is not None: + general_settings["database_disable_prepared_statements"] = config_value + mock_proxy_module = MagicMock( + app=MagicMock(), + ProxyConfig=MagicMock(), + KeyManagementSettings=MagicMock(), + save_worker_config=MagicMock(), + ) + mock_proxy_module.ProxyConfig.return_value.get_config = AsyncMock( + return_value={"general_settings": general_settings} + ) + + clean_env = { + k: v + for k, v in os.environ.items() + if k not in ("DATABASE_URL", "DIRECT_URL", "DATABASE_DISABLE_PREPARED_STATEMENTS") + } + if env_value is not None: + clean_env["DATABASE_DISABLE_PREPARED_STATEMENTS"] = env_value + + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.dict( + "sys.modules", + { + "proxy_server": mock_proxy_module, + "litellm.proxy.proxy_server": mock_proxy_module, + }, + ), + patch( + "litellm.proxy.proxy_cli.ProxyInitializationHelpers._get_default_unvicorn_init_args" + ) as mock_get_args, + patch( + "litellm.proxy.proxy_cli.append_query_params", + side_effect=lambda url, params: str(url), + ) as mock_append_query_params, + ): + mock_get_args.return_value = { + "app": "litellm.proxy.proxy_server:app", + "host": "localhost", + "port": 8000, + } + + result = runner.invoke( + run_server, + ["--local", "--config", "test-config.yaml", "--skip_server_startup"], + ) + + assert ( + result.exit_code == 0 + ), f"exit_code={result.exit_code}, output={result.output}" + appended_params = mock_append_query_params.call_args.args[1] + if expect_pgbouncer: + assert appended_params["pgbouncer"] == "true", appended_params + else: + assert "pgbouncer" not in appended_params, appended_params + + @patch("subprocess.run") + @patch("atexit.register") + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") + @patch( + "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False + ) + def test_malformed_disable_prepared_statements_env_var_is_rejected_even_when_config_enables_it( + self, + mock_should_update, + mock_setup_db, + mock_atexit_register, + mock_subprocess_run, + ): + from click.testing import CliRunner + + from litellm.proxy.proxy_cli import run_server + + runner = CliRunner() + mock_subprocess_run.return_value = MagicMock(returncode=0) + + mock_proxy_module = MagicMock( + app=MagicMock(), + ProxyConfig=MagicMock(), + KeyManagementSettings=MagicMock(), + save_worker_config=MagicMock(), + ) + mock_proxy_module.ProxyConfig.return_value.get_config = AsyncMock( + return_value={ + "general_settings": { + "database_url": "postgresql://test:test@localhost:5432/test", + "database_disable_prepared_statements": True, + } + } + ) + + clean_env = { + k: v + for k, v in os.environ.items() + if k not in ("DATABASE_URL", "DIRECT_URL", "DATABASE_DISABLE_PREPARED_STATEMENTS") + } + clean_env["DATABASE_DISABLE_PREPARED_STATEMENTS"] = "enabled" + + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.dict( + "sys.modules", + { + "proxy_server": mock_proxy_module, + "litellm.proxy.proxy_server": mock_proxy_module, + }, + ), + patch( + "litellm.proxy.proxy_cli.ProxyInitializationHelpers._get_default_unvicorn_init_args" + ) as mock_get_args, + patch( + "litellm.proxy.proxy_cli.append_query_params", + side_effect=lambda url, params: str(url), + ) as mock_append_query_params, + ): + mock_get_args.return_value = { + "app": "litellm.proxy.proxy_server:app", + "host": "localhost", + "port": 8000, + } + + result = runner.invoke( + run_server, + ["--local", "--config", "test-config.yaml", "--skip_server_startup"], + ) + + assert isinstance(result.exception, ValueError), f"exit_code={result.exit_code}, output={result.output}" + assert "DATABASE_DISABLE_PREPARED_STATEMENTS" in str(result.exception), result.exception + mock_append_query_params.assert_not_called() + @patch("uvicorn.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") From cd69342eef5a317c8f54d39bfaddf06884f3e0c5 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:11:31 -0700 Subject: [PATCH 45/48] chore(prices): sync Vertex AI prices: 20 models (#42577) deep-research-pro-preview-12-2025: cache_read_input_token_cost_batches vertex_ai/deep-research-pro-preview-12-2025: cache_read_input_token_cost_batches gemini-3-pro-image: cache_read_input_token_cost_batches vertex_ai/gemini-3-pro-image: cache_read_input_token_cost_batches gemini-3.1-flash-image: cache_read_input_token_cost_batches vertex_ai/gemini-3.1-flash-image: cache_read_input_token_cost_batches gemini-3.1-flash-lite: cache_read_input_token_cost_batches vertex_ai/gemini-3.1-flash-lite: cache_read_input_token_cost_batches gemini-3.1-flash-lite-image: cache_read_input_token_cost_batches vertex_ai/gemini-3.1-flash-lite-image: cache_read_input_token_cost_batches gemini-3.5-flash: cache_read_input_token_cost_batches vertex_ai/gemini-3.5-flash: cache_read_input_token_cost_batches gemini-3.5-flash-lite: cache_read_input_token_cost_batches vertex_ai/gemini-3.5-flash-lite: cache_read_input_token_cost_batches gemini-3.6-flash: cache_read_input_token_cost_batches vertex_ai/gemini-3.6-flash: cache_read_input_token_cost_batches gemini-3.7-flash: cache_read_input_token_cost_batches vertex_ai/gemini-3.7-flash: cache_read_input_token_cost_batches gemini-3.8-flash: cache_read_input_token_cost_batches vertex_ai/gemini-3.8-flash: cache_read_input_token_cost_batches Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 20 +++++++++++++++++++ model_prices_and_context_window.json | 20 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 94786f250b0..431e965524e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -26790,6 +26790,7 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -26883,6 +26884,7 @@ }, "gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -26967,6 +26969,7 @@ }, "gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -27060,6 +27063,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -27120,6 +27124,7 @@ "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -27177,6 +27182,7 @@ }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -27888,6 +27894,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_token": 1.5e-06, "input_cost_per_audio_token": 1.5e-06, "litellm_provider": "vertex_ai", @@ -27948,6 +27955,7 @@ "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -28006,6 +28014,7 @@ "vertex_ai/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -28065,6 +28074,7 @@ "vertex_ai/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30397,6 +30407,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", @@ -30457,6 +30468,7 @@ "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30515,6 +30527,7 @@ "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30574,6 +30587,7 @@ "gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -52123,6 +52137,7 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -52168,6 +52183,7 @@ }, "vertex_ai/gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -52204,6 +52220,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -52297,6 +52314,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -52358,6 +52376,7 @@ "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -52416,6 +52435,7 @@ }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 94786f250b0..431e965524e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -26790,6 +26790,7 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -26883,6 +26884,7 @@ }, "gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -26967,6 +26969,7 @@ }, "gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -27060,6 +27063,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -27120,6 +27124,7 @@ "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -27177,6 +27182,7 @@ }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -27888,6 +27894,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_token": 1.5e-06, "input_cost_per_audio_token": 1.5e-06, "litellm_provider": "vertex_ai", @@ -27948,6 +27955,7 @@ "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -28006,6 +28014,7 @@ "vertex_ai/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -28065,6 +28074,7 @@ "vertex_ai/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30397,6 +30407,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", @@ -30457,6 +30468,7 @@ "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30515,6 +30527,7 @@ "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30574,6 +30587,7 @@ "gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -52123,6 +52137,7 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -52168,6 +52183,7 @@ }, "vertex_ai/gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -52204,6 +52220,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -52297,6 +52314,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -52358,6 +52376,7 @@ "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -52416,6 +52435,7 @@ }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, From 989d7b87b2d30982a6300dde227ec45d8f60f4ce Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:14:13 -0700 Subject: [PATCH 46/48] fix(proxy): attribute provider and model_info on pre_call_hook rejections (#41077) * fix(proxy): attribute provider and model info on pre-call rejected requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep pre-call rejections out of deployment cooldown and prometheus deployment state Stamp model_info only into the logging metadata so the router's failure callbacks do not count a key-level 429 or guardrail 403 against the deployment, treat a resolved plus an unresolved deployment as ambiguous provider attribution, and stop the prometheus deployment counters and deployment_state from treating a proxy-side reject as a selected deployment Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): skip deployment attribution when the rejected body's model is not a string Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(prometheus): bucket non-string request models as other instead of raising in failure hook Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): resolve team deployments and treat guardrail rejects as proxy-side in failure attribution Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(prometheus): flag pre-routing rejects instead of matching exception names Post-call GuardrailRaisedException failures kept their deployment labels on main but lost them on this branch because every GuardrailRaisedException was treated as a pre-routing reject. The proxy failure path now flags litellm_params with proxy_rejected_before_routing only when it adds deployment attribution itself, and the Prometheus logger keys deployment selection off that flag Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): key pre-routing reject flag off provider handoff, not caller metadata Caller-supplied metadata.model_info (kept for keys allowed to override pricing) no longer suppresses proxy_rejected_before_routing. The hook now checks the logging object's first_api_call_start_time, which only the provider handoff sets, so Prometheus never records a deployment failure for a request that was rejected before routing. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): poll for both served and rejected spend rows before asserting attribution Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: yucheng --- litellm/constants.py | 3 + litellm/integrations/prometheus.py | 19 +- litellm/proxy/utils.py | 101 ++++ tests/e2e/AGENTS.md | 2 +- .../coverage_registry/quota_management.yaml | 1 + tests/e2e/models.py | 1 + .../spend_tracking/test_spend_tracking_e2e.py | 46 +- ...ometheus_deployment_state_proxy_rejects.py | 101 ++++ ..._prometheus_requested_model_cardinality.py | 16 + .../test_post_call_failure_hook.py | 456 +++++++++++++++++- 10 files changed, 734 insertions(+), 12 deletions(-) create mode 100644 tests/test_litellm/integrations/test_prometheus_deployment_state_proxy_rejects.py diff --git a/litellm/constants.py b/litellm/constants.py index 1012ca831f2..e5b662bd515 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -238,6 +238,9 @@ LOGS_GUARDRAIL_INFORMATION_MARKER: Final = "_litellm_logs_guardrail_information" # llm_provider stamped on proxy-side rate limit errors when the model resolves to no deployment PROXY_LLM_PROVIDER_FALLBACK: Final = "litellm_proxy" +# litellm_params flag on failure logs for requests the proxy rejected before routing to a deployment +PROXY_REJECTED_BEFORE_ROUTING_KEY: Final = "proxy_rejected_before_routing" + # Generic fallback for unknown models DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = int( os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET", 128) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 28ac9f5cdae..d62a6c3427a 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -18,7 +18,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import print_verbose, verbose_logger -from litellm.constants import PROXY_LLM_PROVIDER_FALLBACK +from litellm.constants import PROXY_LLM_PROVIDER_FALLBACK, PROXY_REJECTED_BEFORE_ROUTING_KEY from litellm.exceptions import ( validate_rate_limit_category, validate_rate_limit_type, @@ -214,17 +214,22 @@ def _get_proxy_llm_router() -> Router | None: return llm_router -def _bounded_requested_model_label(requested_model: str | None, router_originated: bool = False) -> str | None: +def _bounded_requested_model_label(requested_model: object, router_originated: bool = False) -> str | None: """ Bound ``requested_model`` label cardinality: names the router recognizes (model names, deployment ids, aliases, routing groups, team public model names) or matches via a global or team wildcard/pattern route keep their own label value; any other client-supplied string collapses into the - single ``other`` bucket. With no proxy router to vouch for the string, - client-supplied values collapse to ``other`` while ``router_originated`` - values (emitted by an SDK ``Router``'s own deployment failure and - fallback events, where the proxy router never exists) pass through. + single ``other`` bucket, as does any non-string request ``model`` value. + With no proxy router to vouch for the string, client-supplied values + collapse to ``other`` while ``router_originated`` values (emitted by an + SDK ``Router``'s own deployment failure and fallback events, where the + proxy router never exists) pass through. """ + if requested_model is None: + return None + if not isinstance(requested_model, str): + return UNRECOGNIZED_REQUESTED_MODEL_LABEL if not requested_model: return requested_model llm_router: Final = _get_proxy_llm_router() @@ -2832,7 +2837,7 @@ class PrometheusLogger(CustomLogger): # On LiteLLM-side rejects (no deployment picked), route request_kwargs["model"] # into requested_model and leave deployment-scoped labels empty. - deployment_selected: Final = bool(model_id) + deployment_selected: Final = bool(model_id) and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY) if deployment_selected: label_litellm_model_name = litellm_model_name label_model_id = model_id diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ea3ecb7f637..c7ce9d081d6 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -52,6 +52,7 @@ from litellm.constants import ( DEFAULT_MODEL_CREATED_AT_TIME, LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, MAX_TEAM_LIST_LIMIT, + PROXY_REJECTED_BEFORE_ROUTING_KEY, REDIS_SPEND_LOGS_BUFFER_DEQUEUE_COUNT, SPEND_LOG_QUEUE_MAX_BYTES, SPEND_LOG_WRITE_BATCH_MAX_BYTES, @@ -979,6 +980,92 @@ def _failure_usage_to_lift( _EMPTY_LIFT: Final = MappingProxyType({}) +def _stamp_deployment_attribution( + litellm_params: dict[str, object], model_group: str | None, team_id: str | None, dispatched: bool +) -> Mapping[str, object]: + """Stamp provider and logging-metadata attribution onto ``litellm_params`` and return it. + ``litellm_params["model_info"]`` stays unset: the router's cooldown and per-deployment rpm + callbacks key off it and must not count a proxy-side reject against the deployment. A failure + after the provider handoff keeps the metadata the router stamped; a request that never reached a + provider is flagged ``PROXY_REJECTED_BEFORE_ROUTING_KEY`` (deployment metrics key off it) whatever + its metadata says, since ``metadata.model_info`` can be caller supplied.""" + attribution: Final = _deployment_attribution_for_model_group(model_group, team_id) + if "custom_llm_provider" in attribution: + litellm_params["custom_llm_provider"] = attribution["custom_llm_provider"] + if dispatched: + return attribution + litellm_params[PROXY_REJECTED_BEFORE_ROUTING_KEY] = True + if "model_info" not in attribution: + return attribution + if litellm_params.get("metadata") is None: + litellm_params["metadata"] = {} # mutable-ok: legacy logging payload is populated in place + metadata: Final = litellm_params["metadata"] + if not isinstance(metadata, dict): + return attribution + metadata.setdefault("model_info", attribution["model_info"]) + metadata.setdefault("deployment", attribution["deployment"]) + return attribution + + +def _deployment_attribution_for_model_group(model_group: object, team_id: str | None) -> Mapping[str, object]: + """Provider fields the router would have stamped had it reached a deployment: + ``custom_llm_provider`` when every deployment in the group resolves to the same + provider, plus ``model_info`` and ``deployment`` when the group has exactly one. + ``team_id`` picks the key's team deployments over a global group of the same public name.""" + if not isinstance(model_group, str): + return _EMPTY_LIFT + + from litellm.proxy.proxy_server import llm_router + + if llm_router is None: + return _EMPTY_LIFT + deployments: Final = llm_router.get_model_list(model_name=model_group, team_id=team_id) + if not deployments: + return _EMPTY_LIFT + + def _provider_for_deployment(deployment: Mapping[str, object]) -> str | None: + litellm_params: Final = cast( # cast-ok: router deployment parameters are mapping-shaped + Mapping[str, object], deployment["litellm_params"] + ) + try: + provider: Final = litellm.get_llm_provider( + model=cast(str, litellm_params["model"]), # cast-ok: router deployment model is a string + custom_llm_provider=cast( # cast-ok: router deployment provider is optional + str | None, litellm_params.get("custom_llm_provider") + ), + )[1] + return cast(str | None, provider) # cast-ok: provider resolver returns an optional provider string + except Exception: # noqa: BLE001 # get_llm_provider raises for unmapped models + return None + + providers: Final = frozenset(_provider_for_deployment(deployment) for deployment in deployments) + shared_provider: Final = next(iter(providers)) if len(providers) == 1 else None + single_deployment: Final = deployments[0] if len(deployments) == 1 else None + single_deployment_params: Final = ( + cast( # cast-ok: router deployment parameters are mapping-shaped + Mapping[str, object], single_deployment["litellm_params"] + ) + if single_deployment is not None + else None + ) + return MappingProxyType( + { + # mutable-ok: frozen immediately by the outer MappingProxyType + **({"custom_llm_provider": shared_provider} if shared_provider is not None else {}), + **( + { # mutable-ok: frozen immediately by the outer MappingProxyType + "model_info": dict( # mutable-ok: preserve the router's mutable model-info payload + single_deployment.get("model_info") or {} + ), + "deployment": single_deployment_params["model"], + } + if single_deployment is not None and single_deployment_params is not None + else {} # mutable-ok: frozen immediately by the outer MappingProxyType + ), + } + ) + + def _call_type_for_route(route: str | None) -> str | None: """The route's call type when it maps to a single operation (its async and sync variants); None for routes shared by several operations, since the method is not known here.""" @@ -3227,11 +3314,25 @@ class ProxyLogging: elif k not in ("model", "user", "litellm_logging_obj"): _optional_params[k] = v + attribution: Final = _stamp_deployment_attribution( + _litellm_params, + request_data.get("model"), + user_api_key_dict.team_id, + dispatched=litellm_logging_obj.model_call_details.get("first_api_call_start_time") is not None, + ) + litellm_logging_obj.update_environment_variables( model=request_data.get("model", ""), user=request_data.get("user", ""), optional_params=_optional_params, litellm_params=_litellm_params, + **( + { # mutable-ok: frozen immediately by keyword expansion + "custom_llm_provider": attribution["custom_llm_provider"] + } + if "custom_llm_provider" in attribution + else {} # mutable-ok: frozen immediately by keyword expansion + ), ) input: list | str | dict = "" diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index 6ea75b806d0..ad1e0322787 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -216,7 +216,7 @@ quota_management... | isolates_per_model | isolates_per_member | isolates_per_group | enforced_across_keys | routes_to_fallback | reseed_matches_db | reports_spend | logs_cost | zero_cost | matches_sum_of_logs | loses_no_spend | attributes_spend | writes_own_rows - | writes_failure_row | returns_cost | keeps_total | joins_key | reports_alias_and_email + | writes_failure_row | attributes_provider | returns_cost | keeps_total | joins_key | reports_alias_and_email | health_rows_keep_service_account | retrieve_batch_cost_joins_retrieving_key | poller_batch_cost_joins_creating_key | bills_under_request_session e.g. quota_management.ratelimit.rpm.blocks_over_limit exercised_on=[chat_completions, messages] diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index 5432c5095a2..5ea48fdcd9d 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -51,6 +51,7 @@ - {id: quota_management.spend_tracking.end_user.attributes_spend, module: quota_management, tier: P1, behavior: spend_tracking, variant: end_user, assertions: [attributes_spend], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "user= attribution lands the end-user id on the spend row"} - {id: quota_management.spend_tracking.per_model.writes_own_rows, module: quota_management, tier: P2, behavior: spend_tracking, variant: per_model, assertions: [writes_own_rows], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Each model on a shared key gets its own spend row"} - {id: quota_management.spend_tracking.failure.writes_failure_row, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_failure_row], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_log_error_logger.py", rationale: "A failed call writes a failure-status spend row"} +- {id: quota_management.spend_tracking.failure.attributes_provider, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [attributes_provider], exercised_on: [chat_completions], source: "proxy/utils.py", rationale: "A request rejected in pre_call_hook (rate limit, guardrail) still lands its single deployment's provider and model_id on the failure spend row"} - {id: quota_management.spend_tracking.spend_calculate.returns_cost, module: quota_management, tier: P2, behavior: spend_tracking, variant: spend_calculate, assertions: [returns_cost], exercised_on: [spend_calculate], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "/spend/calculate prices a hypothetical request at nonzero cost"} - {id: quota_management.spend_tracking.pagination.keeps_total, module: quota_management, tier: P2, behavior: spend_tracking, variant: pagination, assertions: [keeps_total], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "Spend-logs v2 pagination caps page size without losing the total"} - {id: quota_management.spend_tracking.cache_write.bills_cache_creation_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_write, assertions: [bills_cache_creation_rate], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "OpenAI cache-write tokens land on the spend row as cache-creation tokens billed at the cache-creation rate, not silently at the input rate (#34046)"} diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 74cad3ad8df..bc81d1015ba 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -952,6 +952,7 @@ class SpendLogRow(BaseModel): cache_hit: str | None = None call_type: str | None = None custom_llm_provider: str | None = None + model_id: str | None = None team_id: str | None = None user: str | None = None end_user: str | None = None diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py index 8a91e53e7d7..6ca6f8cee55 100644 --- a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py @@ -21,9 +21,9 @@ from math import isclose from typing import Final import pytest -from e2e_http import Success +from e2e_http import RateLimitedError, Success from lifecycle import ResourceManager -from models import LiteLLMParamsBody, SpendLogs, SpendLogsParams +from models import KeyGenerateBody, LiteLLMParamsBody, SpendLogs, SpendLogsParams from spend_e2e_client import SpendClient, SpendLogRow, is_ok, unique_marker, unwrap pytestmark = pytest.mark.e2e @@ -43,6 +43,7 @@ def _summarize(rows: list[SpendLogRow]) -> list[dict[str, object]]: "cache_hit", "call_type", "custom_llm_provider", + "model_id", "prompt_tokens", "completion_tokens", "total_tokens", @@ -498,6 +499,47 @@ def test_failure_call_writes_failure_status_row( assert (failure_row.spend or 0) == 0.0, "failed call must not be charged" +@pytest.mark.covers("quota_management.spend_tracking.failure.attributes_provider") +def test_pre_call_rejection_row_attributes_provider_and_model_id( + client: SpendClient, resources: ResourceManager +) -> None: + """A request the proxy rejects before the router picks a deployment (here the + key's rpm limit, a pre_call_hook 429) never reaches the code that stamps the + deployment onto the log. The failure row must still carry the provider and + model_id of the model group's only deployment, so per-provider failure reports + can count it.""" + model = f"e2e-spend-precall-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model="openai/gpt-5.5", api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = client.proxy.generate_key(KeyGenerateBody(models=[model], rpm_limit=1)) + resources.defer(lambda: client.proxy.delete_key(key)) + + unwrap(client.chat(key, model, f"reply with one word {unique_marker()}", max_tokens=8)) + rejected = client.chat(key, model, f"over the rpm limit {unique_marker()}", max_tokens=8) + assert isinstance(rejected, RateLimitedError), ( + f"the second call on an rpm_limit=1 key must be rejected with 429 before routing, got {rejected}" + ) + + rows = client.poll_logs_for_key( + key, + min_rows=2, + predicate=lambda rs: {r.status for r in rs} >= {"success", "failure"}, + ) + success_row = _require_row(rows, lambda r: r.status == "success", "for the served call") + failure_row = _require_row(rows, lambda r: r.status == "failure", "for the rate-limited call") + + assert failure_row.custom_llm_provider == success_row.custom_llm_provider, ( + f"rejected call lost its provider: failure row {failure_row.custom_llm_provider!r} vs " + f"served row {success_row.custom_llm_provider!r}; {_summarize(rows)}" + ) + assert failure_row.model_id == model_id, ( + f"rejected call lost its deployment: failure row model_id {failure_row.model_id!r} vs " + f"registered {model_id!r}; {_summarize(rows)}" + ) + + @pytest.mark.covers("quota_management.spend_tracking.spend_calculate.returns_cost") def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None: cost = client.calculate_spend( diff --git a/tests/test_litellm/integrations/test_prometheus_deployment_state_proxy_rejects.py b/tests/test_litellm/integrations/test_prometheus_deployment_state_proxy_rejects.py new file mode 100644 index 00000000000..fedfc2b0848 --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_deployment_state_proxy_rejects.py @@ -0,0 +1,101 @@ +""" +LIT-7701 attributes a pre_call_hook rejection's failure log to the model +group's single deployment (``model_id`` and ``custom_llm_provider``) and flags +it with ``PROXY_REJECTED_BEFORE_ROUTING_KEY``. The deployment health metrics +must keep treating such rejects as "no deployment picked": a key rate limit or +guardrail block never reached the deployment, so it must not flip +``litellm_deployment_state`` to partial outage or count as a deployment failure +response. A failure raised after the router picked a deployment (a post-call +guardrail block, a provider error) carries no flag and keeps its deployment labels. +""" + +import pytest +from fastapi import HTTPException +from prometheus_client import REGISTRY + +from litellm.constants import PROXY_REJECTED_BEFORE_ROUTING_KEY +from litellm.exceptions import GuardrailRaisedException +from litellm.integrations.prometheus import PrometheusLogger +from litellm.proxy._types import ProxyException +from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + +@pytest.fixture(autouse=True) +def cleanup_prometheus_registry(): + for collector in list(REGISTRY._collector_to_names.keys()): + try: + REGISTRY.unregister(collector) + except Exception: + pass + + yield + + for collector in list(REGISTRY._collector_to_names.keys()): + try: + REGISTRY.unregister(collector) + except Exception: + pass + + +def _attributed_failure_kwargs(exception: Exception, rejected_before_routing: bool) -> dict: + return { + "model": "openai/gpt-4.1", + "litellm_params": { + "custom_llm_provider": "openai", + "metadata": {"model_info": {"id": "dep-1"}, "model_group": "internal-model"}, + **({PROXY_REJECTED_BEFORE_ROUTING_KEY: True} if rejected_before_routing else {}), + }, + "standard_logging_object": { + "model_id": "dep-1", + "model_group": "internal-model", + "api_base": "https://api.openai.com", + "metadata": {}, + }, + "exception": exception, + } + + +def _model_id_values(metric) -> set[str]: + index = metric._labelnames.index("model_id") + return {sample_key[index] for sample_key in metric._metrics} + + +class _ProviderError(Exception): + status_code = 500 + + +@pytest.mark.parametrize( + "rejection", + [ + HTTPException(status_code=403, detail="guardrail blocked"), + ProxyException(message="budget exceeded", type="budget_exceeded", param=None, code=400), + ProxyRateLimitError(detail={"error": "key rpm limit"}), + GuardrailRaisedException(guardrail_name="pii", message="blocked", status_code=403), + ], + ids=["http_exception", "proxy_exception", "proxy_rate_limit", "guardrail_raised"], +) +def test_attributed_proxy_reject_leaves_deployment_healthy(rejection: Exception): + logger = PrometheusLogger() + + logger.set_llm_deployment_failure_metrics(_attributed_failure_kwargs(rejection, rejected_before_routing=True)) + + assert logger.litellm_deployment_state._metrics == {} + assert _model_id_values(logger.litellm_deployment_failure_responses) == {""} + assert _model_id_values(logger.litellm_deployment_total_requests) == {""} + + +@pytest.mark.parametrize( + "failure", + [ + _ProviderError("upstream 500"), + GuardrailRaisedException(guardrail_name="pii", message="response blocked", status_code=400), + ], + ids=["provider_error", "post_call_guardrail"], +) +def test_failure_after_routing_still_marks_deployment_partial_outage(failure: Exception): + logger = PrometheusLogger() + + logger.set_llm_deployment_failure_metrics(_attributed_failure_kwargs(failure, rejected_before_routing=False)) + + assert _model_id_values(logger.litellm_deployment_state) == {"dep-1"} + assert _model_id_values(logger.litellm_deployment_failure_responses) == {"dep-1"} diff --git a/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py b/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py index 519a13751f1..f86f4460b28 100644 --- a/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py +++ b/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py @@ -116,6 +116,22 @@ async def test_unknown_models_collapse_to_one_series_on_proxy_request_metrics(ro assert _total_value(metric) == 25 +@pytest.mark.asyncio +@pytest.mark.parametrize("model", [["gpt-4o-mini"], {"name": "gpt-4o-mini"}, 123]) +async def test_non_string_models_collapse_to_other_on_proxy_request_metrics(router, model: object): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await logger.async_post_call_failure_hook( + request_data={"model": model, "metadata": {}, "proxy_server_request": {}}, + original_exception=_ClientSideError("'model' must be a string."), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key-1"), + ) + + assert _requested_model_values(logger.litellm_proxy_failed_requests_metric) == {UNRECOGNIZED_REQUESTED_MODEL_LABEL} + assert _total_value(logger.litellm_proxy_failed_requests_metric) == 1 + + @pytest.mark.asyncio async def test_known_alias_and_wildcard_models_keep_their_own_labels(router): logger = PrometheusLogger() diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py b/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py index a4bb7d63548..d7c52c528eb 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py @@ -5,16 +5,18 @@ from __future__ import annotations import asyncio from datetime import datetime -from typing import Any, Final +from types import MappingProxyType +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import HTTPException import litellm +from litellm.constants import PROXY_REJECTED_BEFORE_ROUTING_KEY from litellm.exceptions import GuardrailRaisedException from litellm.integrations.custom_logger import CustomLogger -from litellm.proxy._types import AlertType, ProxyErrorTypes, UserAPIKeyAuth +from litellm.proxy._types import ProxyErrorTypes, UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging @@ -99,6 +101,456 @@ async def test_post_call_failure_hook_no_callbacks_returns_none( } +@pytest.mark.asyncio +async def test_post_call_failure_hook_attributes_single_router_deployment( + proxy_logging, make_user_api_key_auth, monkeypatch +): + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "internal-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + "model_info": {"provider": "acme"}, + } + ] + ), + ) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={"model": "internal-model", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=HTTPException(status_code=403, detail="blocked"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + kwargs = recorded[0] + assert kwargs["custom_llm_provider"] == "openai" + assert kwargs["litellm_params"]["custom_llm_provider"] == "openai" + assert kwargs["litellm_params"]["metadata"]["model_info"]["provider"] == "acme" + assert kwargs["litellm_params"]["metadata"]["deployment"] == "openai/gpt-4.1" + assert kwargs["litellm_params"][PROXY_REJECTED_BEFORE_ROUTING_KEY] is True + assert kwargs["standard_logging_object"]["custom_llm_provider"] == "openai" + assert ( + kwargs["standard_logging_object"]["model_id"] == proxy_server.llm_router.get_model_list()[0]["model_info"]["id"] + ) + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_keeps_router_stamped_metadata_for_post_call_failures( + proxy_logging, make_user_api_key_auth, monkeypatch +): + """A post-call guardrail block arrives after the provider handoff with the router's own + ``model_info`` in the request metadata. The pre-routing flag must stay off so deployment + metrics keep attributing the failure to the deployment that actually served the call.""" + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "internal-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + "model_info": {"id": "routed-deployment"}, + } + ] + ), + ) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + request_data = { + "litellm_call_id": "post-call-guardrail", + "model": "internal-model", + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"model_info": {"id": "routed-deployment", "served": True}}, + } + logging_obj, request_data = litellm.utils.function_setup( + original_function="acompletion", rules_obj=litellm.utils.Rules(), start_time=datetime.now(), **request_data + ) + logging_obj.model_call_details["first_api_call_start_time"] = datetime.now() + request_data["litellm_logging_obj"] = logging_obj + + await proxy_logging.post_call_failure_hook( + request_data=request_data, + original_exception=GuardrailRaisedException(guardrail_name="g", message="response blocked"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + kwargs = recorded[0] + assert kwargs["litellm_params"]["metadata"]["model_info"] == {"id": "routed-deployment", "served": True} + assert PROXY_REJECTED_BEFORE_ROUTING_KEY not in kwargs["litellm_params"] + assert kwargs["standard_logging_object"]["model_id"] == "routed-deployment" + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_flags_pre_routing_reject_despite_caller_model_info( + proxy_logging, make_user_api_key_auth, monkeypatch +): + """A key allowed to override pricing keeps caller-supplied ``metadata.model_info``. A reject + before any provider handoff must still carry the pre-routing flag so deployment metrics do + not record an outage for a deployment the request never reached.""" + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "internal-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + "model_info": {"id": "real-deployment"}, + } + ] + ), + ) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={ + "model": "internal-model", + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"model_info": {"id": "spoofed-deployment"}}, + }, + original_exception=HTTPException(status_code=429, detail="key over limit"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + kwargs = recorded[0] + assert kwargs["litellm_params"][PROXY_REJECTED_BEFORE_ROUTING_KEY] is True + assert kwargs["litellm_params"]["custom_llm_provider"] == "openai" + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_attribution_does_not_count_against_the_deployment( + proxy_logging, make_user_api_key_auth, monkeypatch +): + """The router's failure callbacks run on this path too. A proxy-side reject must not + bump the deployment's failure or rpm counters, or a key hitting its own limit + could cool down the only deployment for everyone.""" + from litellm.proxy import proxy_server + + router = litellm.Router( + model_list=[ + { + "model_name": "internal-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test", "rpm": 100}, + } + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + proxy_logging.alert_types = [] + deployment_id = router.get_model_list()[0]["model_info"]["id"] + + for status in (403, 429): + await proxy_logging.post_call_failure_hook( + request_data={"model": "internal-model", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=HTTPException(status_code=status, detail="blocked"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + pending = asyncio.all_tasks() - {asyncio.current_task()} + await asyncio.gather(*pending, return_exceptions=True) + + deployment_keys = [key for key in router.cache.in_memory_cache.cache_dict if deployment_id in key] + assert deployment_keys == [], f"proxy reject was counted against the deployment: {deployment_keys}" + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_attributes_the_keys_team_deployment_over_the_global_group( + proxy_logging, make_user_api_key_auth, monkeypatch +): + """A team key requesting its team public model name must be attributed to the team's + deployment, not to a global group that happens to share the public name.""" + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "shared-name", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + "model_info": {"id": "global-deployment"}, + }, + { + "model_name": "shared-name_test-team_deadbeef", + "litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": "sk-test"}, + "model_info": { + "id": "team-deployment", + "team_id": "test-team", + "team_public_model_name": "shared-name", + }, + }, + ] + ), + ) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={"model": "shared-name", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=HTTPException(status_code=429, detail="rate limited"), + user_api_key_dict=make_user_api_key_auth(team_id="test-team", request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + kwargs = recorded[0] + assert kwargs["custom_llm_provider"] == "anthropic" + assert kwargs["litellm_params"]["metadata"]["deployment"] == "anthropic/claude-sonnet-4-5" + assert kwargs["standard_logging_object"]["model_id"] == "team-deployment" + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_omits_provider_for_mixed_router_deployments( + proxy_logging, make_user_api_key_auth, monkeypatch +): + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "internal-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + }, + { + "model_name": "internal-model", + "litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": "sk-test"}, + }, + ] + ), + ) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={"model": "internal-model", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=HTTPException(status_code=403, detail="blocked"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + kwargs = recorded[0] + assert kwargs.get("custom_llm_provider") is None + assert "model_info" not in (kwargs["litellm_params"].get("metadata") or {}) + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_omits_provider_when_a_deployment_does_not_resolve( + proxy_logging, make_user_api_key_auth, monkeypatch +): + """One deployment resolves to openai and its sibling resolves to nothing: the group + is not known to be single-provider, so no provider is stamped on the failure.""" + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + router = MagicMock() + router.get_model_list.return_value = [ + {"model_name": "internal-model", "litellm_params": {"model": "openai/gpt-4.1"}}, + {"model_name": "internal-model", "litellm_params": {"model": "unmapped-model-with-no-provider"}}, + ] + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={"model": "internal-model", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=HTTPException(status_code=403, detail="blocked"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + kwargs = recorded[0] + assert kwargs.get("custom_llm_provider") is None + assert kwargs["litellm_params"].get("custom_llm_provider") is None + + +@pytest.mark.asyncio +async def test_handle_logging_proxy_only_path_attributes_with_read_only_metadata( + proxy_logging, make_user_api_key_auth, monkeypatch +): + """With a logging object already on the request, its metadata is taken as given; + a read-only mapping there must not crash the stamp, and the failure handler + still receives the provider attribution.""" + from litellm.proxy import proxy_server + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "internal-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + "model_info": {"provider": "acme"}, + } + ] + ), + ) + logging_obj = MagicMock() + logging_obj.call_type = "acompletion" + logging_obj.model_call_details = {} + logging_obj.async_failure_handler = AsyncMock() + + await proxy_logging._handle_logging_proxy_only_error( + request_data={ + "litellm_logging_obj": logging_obj, + "model": "internal-model", + "messages": [{"role": "user", "content": "hi"}], + "metadata": MappingProxyType({"user_api_key_alias": "frozen"}), + }, + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + original_exception=HTTPException(status_code=403, detail="blocked"), + ) + + assert logging_obj.async_failure_handler.called + update_kwargs = logging_obj.update_environment_variables.call_args.kwargs + assert update_kwargs["custom_llm_provider"] == "openai" + assert update_kwargs["litellm_params"]["custom_llm_provider"] == "openai" + assert update_kwargs["litellm_params"]["metadata"] == {"user_api_key_alias": "frozen"} + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_fires_without_router_attribution( + proxy_logging, make_user_api_key_auth, monkeypatch +): + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "different-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + } + ] + ), + ) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={"model": "internal-model", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=HTTPException(status_code=403, detail="blocked"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + kwargs = recorded[0] + assert kwargs.get("custom_llm_provider") is None + assert "model_info" not in (kwargs["litellm_params"].get("metadata") or {}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", [123, ["internal-model"], {"name": "internal-model"}, None]) +async def test_post_call_failure_hook_fires_for_non_string_model( + proxy_logging, make_user_api_key_auth, monkeypatch, model: object +): + """A body whose ``model`` is not a string is rejected by the proxy before routing; its + failure callback must still fire, unattributed, instead of a TypeError escaping the hook.""" + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "internal-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + } + ] + ), + ) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={"model": model, "messages": [{"role": "user", "content": "hi"}]}, + original_exception=HTTPException(status_code=400, detail="'model' must be a string."), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + kwargs = recorded[0] + assert kwargs.get("custom_llm_provider") is None + assert "model_info" not in (kwargs["litellm_params"].get("metadata") or {}) + + @pytest.mark.asyncio async def test_post_call_failure_hook_callback_returns_http_exception( proxy_logging, make_user_api_key_auth, monkeypatch From 075536eca1a247bbab78d7992275a176875b96ae Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 21:19:26 +0000 Subject: [PATCH 47/48] chore(cost-map): remove models past their deprecation date (#42435) * chore(cost-map): remove models past their deprecation date Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(cost-calc): drop the empty parametrize left behind by the gemini web search removal Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost-map): drop merge base block left by conflict resolution Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(cost-calc): drop gemini image cost tests pinned on removed model Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 6095 +---------------- model_prices_and_context_window.json | 6095 +---------------- .../test_anthropic_cache_control_hook.py | 2 +- .../llm_cost_calc/test_llm_cost_calc_utils.py | 151 - .../test_tool_call_cost_tracking.py | 123 - .../test_fallback_generalizations.py | 4 - .../test_get_model_cost_map.py | 1 - .../test_litellm_logging.py | 15 - ...streaming_chunk_builder_server_tool_use.py | 26 - .../litellm_core_utils/test_token_counter.py | 3 - .../chat/test_converse_transformation.py | 37 - .../test_databricks_cost_calculator.py | 8 - .../llms/gemini/test_cost_calculator.py | 195 - .../test_github_copilot_transformation.py | 54 - .../llms/openai/test_gpt5_transformation.py | 16 - ...test_vertex_and_google_ai_studio_gemini.py | 4 +- ...test_vertex_passthrough_logging_handler.py | 28 - .../wandb/test_wandb_chat_transformation.py | 4 - .../llms/xai/test_xai_cost_calculator.py | 73 - .../xai/test_xai_redirected_slug_pricing.py | 102 - .../proxy/auth/test_auth_checks.py | 1 - ...t_anthropic_passthrough_logging_handler.py | 73 - .../test_vertex_ai_batch_passthrough.py | 232 - .../proxy/spend_tracking/test_savings.py | 19 - .../test_spend_management_endpoints.py | 172 - .../responses/test_metadata_codex_callback.py | 66 - .../rust_bridge/test_token_counter.py | 1 - tests/test_litellm/test_cost_calculator.py | 374 - .../test_count_tokens_public_api.py | 18 - .../test_gpt_image_cost_calculator.py | 21 - tests/test_litellm/test_router.py | 30 - .../test_together_ai_model_metadata.py | 9 - tests/test_litellm/test_utils.py | 6 - .../test_xai_responses_auto_routing.py | 77 - .../chat/test_azure_ai_transformation.py | 21 - .../test_fireworks_ai_chat_transformation.py | 15 - .../test_moonshot_chat_transformation.py | 2 +- 37 files changed, 18 insertions(+), 14155 deletions(-) delete mode 100644 tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 431e965524e..b48bdd6322c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -53,13 +53,6 @@ "mode": "image_generation", "output_cost_per_image": 0.04 }, - "1024-x-1024/dall-e-2": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 1.9e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "1024-x-1024/max-steps/stability.stable-diffusion-xl-v1": { "litellm_provider": "bedrock", "max_input_tokens": 77, @@ -67,13 +60,6 @@ "mode": "image_generation", "output_cost_per_image": 0.08 }, - "256-x-256/dall-e-2": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 2.4414e-07, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "512-x-512/50-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, @@ -81,13 +67,6 @@ "mode": "image_generation", "output_cost_per_image": 0.018 }, - "512-x-512/dall-e-2": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 6.86e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "512-x-512/max-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, @@ -961,23 +940,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 - }, "anthropic.claude-3-opus-20240229-v1:0": { "input_cost_per_token": 1.5e-05, "litellm_provider": "bedrock", @@ -993,23 +955,6 @@ "cache_read_input_token_cost": 1.5e-06, "cache_creation_input_token_cost": 1.875e-05 }, - "anthropic.claude-3-sonnet-20240229-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, "anthropic.claude-instant-v1": { "input_cost_per_token": 8e-07, "litellm_provider": "bedrock", @@ -2971,60 +2916,6 @@ "supports_vision": true, "supports_tool_choice": true }, - "apac.anthropic.claude-3-5-sonnet-20240620-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, - "apac.anthropic.claude-3-5-sonnet-20241022-v2:0": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "apac.anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 - }, "apac.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, "cache_read_input_token_cost": 1.1e-07, @@ -3052,23 +2943,6 @@ "input_cost_per_token_batches": 5.5e-07, "output_cost_per_token_batches": 2.75e-06 }, - "apac.anthropic.claude-3-sonnet-20240229-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, "apac.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, @@ -3446,29 +3320,6 @@ "supports_max_reasoning_effort": true, "prompt_cache_min_tokens": 1024 }, - "azure_ai/claude-opus-4-1": { - "deprecation_date": "2026-08-05", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "azure_ai", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, "azure_ai/claude-sonnet-4-5": { "deprecation_date": "2026-10-19", "cache_creation_input_token_cost": 3.75e-06, @@ -4503,43 +4354,6 @@ "output_cost_per_token_priority": 2.2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/eu/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.375e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.375e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/eu/gpt-5.1-codex": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.375e-07, @@ -4832,43 +4646,6 @@ "output_cost_per_token_priority": 2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/global/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/global/gpt-5.1-codex": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, @@ -4944,19 +4721,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "azure/gpt-3.5-turbo-0125": { - "deprecation_date": "2025-03-31", - "input_cost_per_token": 5e-07, - "litellm_provider": "azure", - "max_input_tokens": 16384, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true - }, "azure/gpt-3.5-turbo-instruct-0914": { "input_cost_per_token": 1.5e-06, "litellm_provider": "azure_text", @@ -4976,32 +4740,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "azure/gpt-35-turbo-0125": { - "deprecation_date": "2025-05-31", - "input_cost_per_token": 5e-07, - "litellm_provider": "azure", - "max_input_tokens": 16384, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true - }, - "azure/gpt-35-turbo-1106": { - "deprecation_date": "2025-03-31", - "input_cost_per_token": 1e-06, - "litellm_provider": "azure", - "max_input_tokens": 16384, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true - }, "azure/gpt-35-turbo-16k": { "input_cost_per_token": 3e-06, "litellm_provider": "azure", @@ -5769,40 +5507,6 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "azure/gpt-realtime-2": { - "cache_read_input_audio_token_cost": 4e-07, - "cache_read_input_token_cost": 4e-07, - "deprecation_date": "2026-08-31", - "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image_token": 5e-06, - "input_cost_per_token": 4e-06, - "litellm_provider": "azure", - "max_input_tokens": 32000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 6.4e-05, - "output_cost_per_token": 2.4e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", - "supported_endpoints": [ - "/v1/realtime" - ], - "supported_modalities": [ - "text", - "image", - "audio" - ], - "supported_output_modalities": [ - "text", - "audio" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "azure/gpt-realtime-2.1": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, @@ -6100,45 +5804,6 @@ "supports_minimal_reasoning_effort": true, "cache_read_input_token_cost_batches": 6.25e-08 }, - "azure/gpt-5.1-chat-2025-11-13": { - "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_priority": 2.5e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "input_cost_per_token_priority": 2.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "output_cost_per_token_priority": 2e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_native_streaming": true, - "supports_parallel_function_calling": false, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": false, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/gpt-5.1-codex-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, @@ -6289,73 +5954,6 @@ "supports_vision": true, "cache_read_input_token_cost_batches": 6.25e-08 }, - "azure/gpt-5-chat": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "azure/gpt-5-chat-latest": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "azure/gpt-5-codex": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2027-03-17", @@ -6617,43 +6215,6 @@ "output_cost_per_token_priority": 2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/gpt-5.1-codex": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, @@ -6832,78 +6393,6 @@ "supports_vision": true, "cache_read_input_token_cost_batches": 8.75e-08 }, - "azure/gpt-5.2-chat": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "azure/gpt-5.2-chat-2025-12-11": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-05-13", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "azure/gpt-5.2-codex": { "cache_read_input_token_cost": 1.75e-07, "deprecation_date": "2027-07-13", @@ -6936,42 +6425,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "azure/gpt-5.3-chat": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "azure/gpt-5.3-codex": { "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, @@ -10563,43 +10016,6 @@ "output_cost_per_token_priority": 2.2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/us/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.375e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.375e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/us/gpt-5.1-codex": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.375e-07, @@ -11201,18 +10617,6 @@ "/v1/images/edits" ] }, - "azure_ai/MAI-Image-2e": { - "deprecation_date": "2026-08-15", - "input_cost_per_token": 5e-06, - "litellm_provider": "azure_ai", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "output_cost_per_image_token": 1.95e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/images/generations" - ] - }, "azure_ai/MAI-Thinking-1": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 2e-06, @@ -11237,34 +10641,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "azure_ai/Llama-3.2-11B-Vision-Instruct": { - "deprecation_date": "2026-06-13", - "input_cost_per_token": 3.7e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 3.7e-07, - "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-11b-vision-instruct-offer?tab=Overview", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "azure_ai/Llama-3.2-90B-Vision-Instruct": { - "deprecation_date": "2026-06-13", - "input_cost_per_token": 2.04e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 2.04e-06, - "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-90b-vision-instruct-offer?tab=Overview", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "azure_ai/Llama-3.3-70B-Instruct": { "input_cost_per_token": 7.1e-07, "litellm_provider": "azure_ai", @@ -11313,18 +10689,6 @@ "output_cost_per_token": 3.7e-07, "supports_tool_choice": true }, - "azure_ai/Meta-Llama-3.1-405B-Instruct": { - "deprecation_date": "2026-06-13", - "input_cost_per_token": 5.33e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 1.6e-05, - "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-405b-instruct-offer?tab=PlansAndPrice", - "supports_tool_choice": true - }, "azure_ai/Meta-Llama-3.1-70B-Instruct": { "input_cost_per_token": 2.68e-06, "litellm_provider": "azure_ai", @@ -11336,18 +10700,6 @@ "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-70b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, - "azure_ai/Meta-Llama-3.1-8B-Instruct": { - "deprecation_date": "2026-06-13", - "input_cost_per_token": 3e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 6.1e-07, - "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-8b-instruct-offer?tab=PlansAndPrice", - "supports_tool_choice": true - }, "azure_ai/Phi-3-medium-128k-instruct": { "input_cost_per_token": 1.7e-07, "litellm_provider": "azure_ai", @@ -11518,16 +10870,6 @@ "supports_tool_choice": true, "supports_reasoning": true }, - "azure_ai/mistral-document-ai-2505": { - "deprecation_date": "2026-07-20", - "litellm_provider": "azure_ai", - "ocr_cost_per_page": 0.003, - "mode": "ocr", - "supported_endpoints": [ - "/v1/ocr" - ], - "source": "https://devblogs.microsoft.com/foundry/whats-new-in-azure-ai-foundry-august-2025/#mistral-document-ai-(ocr)-%E2%80%94-serverless-in-foundry" - }, "azure_ai/mistral-document-ai-2512": { "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.003, @@ -11628,17 +10970,6 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, - "azure_ai/cohere-rerank-v3.5": { - "deprecation_date": "2026-05-14", - "input_cost_per_query": 0.002, - "input_cost_per_token": 0.0, - "litellm_provider": "azure_ai", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "rerank", - "output_cost_per_token": 0.0 - }, "azure_ai/cohere-rerank-v4.0-pro": { "input_cost_per_query": 0.0025, "input_cost_per_token": 0.0, @@ -11691,19 +11022,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "azure_ai/deepseek-r1": { - "deprecation_date": "2026-08-13", - "input_cost_per_token": 1.35e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 5.4e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_reasoning": true, - "supports_tool_choice": true - }, "azure_ai/deepseek-v3": { "input_cost_per_token": 1.14e-06, "litellm_provider": "azure_ai", @@ -11715,33 +11033,6 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true }, - "azure_ai/deepseek-v3-0324": { - "deprecation_date": "2026-07-13", - "input_cost_per_token": 1.14e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4.56e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_tool_choice": true - }, - "azure_ai/deepseek-v3.1": { - "deprecation_date": "2026-07-13", - "input_cost_per_token": 1.23e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 4.94e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "azure_ai/deepseek-v4-pro": { "deprecation_date": "2028-02-20", "input_cost_per_token": 1.74e-06, @@ -11808,68 +11099,6 @@ ], "supports_embedding_image_input": true }, - "azure_ai/global/grok-3": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 3e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true - }, - "azure_ai/global/grok-3-mini": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.27e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true - }, - "azure_ai/grok-3": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 3e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true - }, - "azure_ai/grok-3-mini": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.27e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true - }, "azure_ai/grok-4": { "input_cost_per_token": 3e-06, "litellm_provider": "azure_ai", @@ -11963,36 +11192,6 @@ "supports_vision": true, "supports_web_search": true }, - "azure_ai/grok-4-fast-non-reasoning": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 2e-07, - "output_cost_per_token": 5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "azure_ai/grok-4-fast-reasoning": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 2e-07, - "output_cost_per_token": 5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_web_search": true - }, "azure_ai/grok-4-1-fast-non-reasoning": { "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, @@ -13516,40 +12715,6 @@ "mode": "chat", "output_cost_per_token": 1.5e-06 }, - "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3.6e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3.6e-07, - "cache_creation_input_token_cost": 4.5e-06 - }, - "bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 3e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-08, - "cache_creation_input_token_cost": 3.75e-07 - }, "bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, @@ -13727,61 +12892,6 @@ "mode": "chat", "output_cost_per_token": 1.5e-06 }, - "bedrock/us-gov-west-1/anthropic.claude-3-7-sonnet-20250219-v1:0": { - "cache_creation_input_token_cost": 4.5e-06, - "cache_read_input_token_cost": 3.6e-07, - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3.6e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3.6e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3.6e-07, - "cache_creation_input_token_cost": 4.5e-06 - }, - "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 3e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-08, - "cache_creation_input_token_cost": 3.75e-07 - }, "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, @@ -14219,34 +13329,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "cerebras/zai-glm-4.6": { - "deprecation_date": "2026-01-20", - "input_cost_per_token": 2.25e-06, - "litellm_provider": "cerebras", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.75e-06, - "source": "https://www.cerebras.ai/pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, - "cerebras/zai-glm-4.7": { - "deprecation_date": "2026-08-17", - "input_cost_per_token": 2.25e-06, - "litellm_provider": "cerebras", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.75e-06, - "source": "https://www.cerebras.ai/pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "cerebras/qwen-3.8-27b": { "input_cost_per_token": 9.9e-07, "litellm_provider": "cerebras", @@ -14272,23 +13354,6 @@ "mode": "chat", "output_cost_per_token": 5e-07 }, - "chatgpt-4o-latest": { - "deprecation_date": "2026-02-17", - "input_cost_per_token": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "gpt-4o-transcribe-diarize": { "input_cost_per_audio_token": 2.5e-06, "input_cost_per_token": 2.5e-06, @@ -14351,131 +13416,6 @@ "prompt_cache_min_tokens": 4096, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, - "claude-3-7-sonnet-20250219": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-02-19", - "input_cost_per_token": 3e-06, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, - "claude-3-haiku-20240307": { - "cache_creation_input_token_cost": 3e-07, - "cache_creation_input_token_cost_above_1hr": 5e-07, - "cache_read_input_token_cost": 3e-08, - "deprecation_date": "2026-04-20", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "claude-3-opus-20240229": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "deprecation_date": "2026-01-05", - "input_cost_per_token": 1.5e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "claude-4-opus-20250514": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_read_input_token_cost": 1.5e-06, - "deprecation_date": "2026-06-15", - "input_cost_per_token": 1.5e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "claude-4-sonnet-20250514": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost": 3e-07, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "deprecation_date": "2026-06-15", - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "litellm_provider": "anthropic", - "max_input_tokens": 1000000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "prompt_cache_min_tokens": 1024 - }, "claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -14654,92 +13594,6 @@ "input_cost_per_token_batches": 1.5e-06, "output_cost_per_token_batches": 7.5e-06 }, - "claude-opus-4-1": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_native_structured_output": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024, - "deprecation_date": "2026-08-05" - }, - "claude-opus-4-1-20250805": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "deprecation_date": "2026-08-05", - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_native_structured_output": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "claude-opus-4-20250514": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "deprecation_date": "2026-06-15", - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, "claude-opus-4-5-20251101": { "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -15167,38 +14021,6 @@ "prompt_cache_min_tokens": 1024, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, - "claude-sonnet-4-20250514": { - "deprecation_date": "2026-06-15", - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "litellm_provider": "anthropic", - "max_input_tokens": 1000000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { "input_cost_per_token": 1.923e-06, "litellm_provider": "cloudflare", @@ -15551,36 +14373,6 @@ "supports_assistant_prefill": true, "supports_tool_choice": true }, - "codex-mini-latest": { - "cache_read_input_token_cost": 3.75e-07, - "deprecation_date": "2026-02-12", - "input_cost_per_token": 1.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 6e-06, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "cohere.command-light-text-v14": { "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", @@ -15680,16 +14472,6 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, - "command": { - "input_cost_per_token": 1e-06, - "litellm_provider": "cohere", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "completion", - "output_cost_per_token": 2e-06, - "deprecation_date": "2025-09-15" - }, "command-a-03-2025": { "input_cost_per_token": 2.5e-06, "litellm_provider": "cohere_chat", @@ -15716,17 +14498,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "command-light": { - "input_cost_per_token": 3e-07, - "litellm_provider": "cohere_chat", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_tool_choice": true, - "deprecation_date": "2025-09-15" - }, "command-nightly": { "input_cost_per_token": 1e-06, "litellm_provider": "cohere", @@ -15736,18 +14507,6 @@ "mode": "completion", "output_cost_per_token": 2e-06 }, - "command-r": { - "input_cost_per_token": 1.5e-07, - "litellm_provider": "cohere_chat", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_function_calling": true, - "supports_tool_choice": true, - "deprecation_date": "2025-09-15" - }, "command-r-08-2024": { "input_cost_per_token": 1.5e-07, "litellm_provider": "cohere_chat", @@ -15759,18 +14518,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "command-r-plus": { - "input_cost_per_token": 2.5e-06, - "litellm_provider": "cohere_chat", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "deprecation_date": "2025-09-15" - }, "command-r-plus-08-2024": { "input_cost_per_token": 2.5e-06, "litellm_provider": "cohere_chat", @@ -15823,26 +14570,6 @@ "supports_vision": true, "source": "https://platform.openai.com/docs/models/computer-use-preview" }, - "dall-e-2": { - "deprecation_date": "2026-05-12", - "input_cost_per_image": 0.02, - "litellm_provider": "openai", - "mode": "image_generation", - "supported_endpoints": [ - "/v1/images/generations", - "/v1/images/edits", - "/v1/images/variations" - ] - }, - "dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_image": 0.04, - "litellm_provider": "openai", - "mode": "image_generation", - "supported_endpoints": [ - "/v1/images/generations" - ] - }, "deepseek-chat": { "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 2.8e-07, @@ -18839,30 +17566,6 @@ "output_vector_size": 1024, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, - "databricks/databricks-claude-3-7-sonnet": { - "cache_creation_input_token_cost": 3.74997e-06, - "cache_read_input_token_cost": 3.0002e-07, - "deprecation_date": "2026-04-12", - "input_cost_per_token": 2.9999900000000002e-06, - "input_dbu_cost_per_token": 4.2857e-05, - "litellm_provider": "databricks", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, - "output_dbu_cost_per_token": 0.000214286, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_anthropic_thinking_payload": true, - "supports_tool_choice": true - }, "databricks/databricks-claude-fable-5": { "cache_creation_input_token_cost": 1.250004e-05, "cache_read_input_token_cost": 1.00002e-06, @@ -19825,44 +18528,6 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, - "databricks/databricks-gpt-5-1-codex-max": { - "cache_creation_input_token_cost": 1.24999e-06, - "cache_read_input_token_cost": 1.2502e-07, - "deprecation_date": "2026-07-16", - "input_cost_per_token": 1.24999e-06, - "input_dbu_cost_per_token": 1.7857e-05, - "litellm_provider": "databricks", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 9.999990000000002e-06, - "output_dbu_cost_per_token": 0.000142857, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true - }, - "databricks/databricks-gpt-5-1-codex-mini": { - "cache_creation_input_token_cost": 2.4997e-07, - "cache_read_input_token_cost": 2.499e-08, - "deprecation_date": "2026-07-16", - "input_cost_per_token": 2.4997e-07, - "input_dbu_cost_per_token": 3.571e-06, - "litellm_provider": "databricks", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.99997e-06, - "output_dbu_cost_per_token": 2.8571e-05, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true - }, "databricks/databricks-gpt-5-2": { "cache_creation_input_token_cost": 1.75e-06, "cache_read_input_token_cost": 1.75e-07, @@ -19883,25 +18548,6 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, - "databricks/databricks-gpt-5-2-codex": { - "cache_creation_input_token_cost": 1.75e-06, - "cache_read_input_token_cost": 1.75e-07, - "deprecation_date": "2026-07-16", - "input_cost_per_token": 1.75e-06, - "input_dbu_cost_per_token": 2.5e-05, - "litellm_provider": "databricks", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_dbu_cost_per_token": 0.0002, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true - }, "databricks/databricks-gpt-5-3-codex": { "cache_creation_input_token_cost": 1.75e-06, "cache_read_input_token_cost": 1.75e-07, @@ -20328,25 +18974,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "databricks/databricks-llama-2-70b-chat": { - "cache_creation_input_token_cost": 5.0001e-07, - "cache_read_input_token_cost": 5.0001e-07, - "deprecation_date": "2024-10-30", - "input_cost_per_token": 5.0001e-07, - "input_dbu_cost_per_token": 7.143e-06, - "litellm_provider": "databricks", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.5000300000000002e-06, - "output_dbu_cost_per_token": 2.1429e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, "databricks/databricks-llama-4-maverick": { "cache_creation_input_token_cost": 5.0001e-07, "cache_read_input_token_cost": 5.0001e-07, @@ -20365,25 +18992,6 @@ "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true }, - "databricks/databricks-meta-llama-3-1-405b-instruct": { - "cache_creation_input_token_cost": 5.00003e-06, - "cache_read_input_token_cost": 5.00003e-06, - "deprecation_date": "2026-02-15", - "input_cost_per_token": 5.00003e-06, - "input_dbu_cost_per_token": 7.1429e-05, - "litellm_provider": "databricks", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, - "output_dbu_cost_per_token": 0.000214286, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, "databricks/databricks-meta-llama-3-1-8b-instruct": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, @@ -20419,82 +19027,6 @@ "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true }, - "databricks/databricks-meta-llama-3-70b-instruct": { - "cache_creation_input_token_cost": 1.00002e-06, - "cache_read_input_token_cost": 1.00002e-06, - "deprecation_date": "2024-07-23", - "input_cost_per_token": 1.00002e-06, - "input_dbu_cost_per_token": 1.4286e-05, - "litellm_provider": "databricks", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 2.9999900000000002e-06, - "output_dbu_cost_per_token": 4.2857e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, - "databricks/databricks-mixtral-8x7b-instruct": { - "cache_creation_input_token_cost": 5.0001e-07, - "cache_read_input_token_cost": 5.0001e-07, - "deprecation_date": "2025-04-30", - "input_cost_per_token": 5.0001e-07, - "input_dbu_cost_per_token": 7.143e-06, - "litellm_provider": "databricks", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.00002e-06, - "output_dbu_cost_per_token": 1.4286e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, - "databricks/databricks-mpt-30b-instruct": { - "cache_creation_input_token_cost": 1.00002e-06, - "cache_read_input_token_cost": 1.00002e-06, - "deprecation_date": "2024-08-30", - "input_cost_per_token": 1.00002e-06, - "input_dbu_cost_per_token": 1.4286e-05, - "litellm_provider": "databricks", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.00002e-06, - "output_dbu_cost_per_token": 1.4286e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, - "databricks/databricks-mpt-7b-instruct": { - "cache_creation_input_token_cost": 5.0001e-07, - "cache_read_input_token_cost": 5.0001e-07, - "deprecation_date": "2024-08-30", - "input_cost_per_token": 5.0001e-07, - "input_dbu_cost_per_token": 7.143e-06, - "litellm_provider": "databricks", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 0.0, - "output_dbu_cost_per_token": 0.0, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, "databricks/databricks-qwen35-122b-a10b": { "cache_creation_input_token_cost": 2.2001e-07, "cache_read_input_token_cost": 2.2001e-07, @@ -21559,18 +20091,6 @@ "supports_tool_choice": true, "supports_function_calling": true }, - "deepinfra/google/gemini-2.0-flash-001": { - "deprecation_date": "2026-06-01", - "max_tokens": 1000000, - "max_input_tokens": 1000000, - "max_output_tokens": 1000000, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 4e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true, - "supports_function_calling": true - }, "deepinfra/google/gemini-2.5-flash": { "max_tokens": 1000000, "max_input_tokens": 1000000, @@ -22429,15 +20949,6 @@ "/v1/audio/speech" ] }, - "embed-english-light-v2.0": { - "deprecation_date": "2026-04-04", - "input_cost_per_token": 1e-07, - "litellm_provider": "cohere", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, "embed-english-light-v3.0": { "input_cost_per_token": 1e-07, "litellm_provider": "cohere", @@ -22446,15 +20957,6 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, - "embed-english-v2.0": { - "deprecation_date": "2026-04-04", - "input_cost_per_token": 1e-07, - "litellm_provider": "cohere", - "max_input_tokens": 4096, - "max_tokens": 4096, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, "embed-english-v3.0": { "input_cost_per_image": 0.0001, "input_cost_per_token": 1e-07, @@ -22469,15 +20971,6 @@ "supports_embedding_image_input": true, "supports_image_input": true }, - "embed-multilingual-v2.0": { - "deprecation_date": "2026-04-04", - "input_cost_per_token": 1e-07, - "litellm_provider": "cohere", - "max_input_tokens": 768, - "max_tokens": 768, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, "embed-multilingual-v3.0": { "input_cost_per_token": 1e-07, "litellm_provider": "cohere", @@ -22645,23 +21138,6 @@ "cache_read_input_token_cost": 3e-07, "cache_creation_input_token_cost": 3.75e-06 }, - "eu.anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 - }, "eu.anthropic.claude-3-opus-20240229-v1:0": { "input_cost_per_token": 1.5e-05, "litellm_provider": "bedrock", @@ -22677,23 +21153,6 @@ "cache_read_input_token_cost": 1.5e-06, "cache_creation_input_token_cost": 1.875e-05 }, - "eu.anthropic.claude-3-sonnet-20240229-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, "eu.anthropic.claude-opus-4-1-20250805-v1:0": { "cache_creation_input_token_cost": 1.875e-05, "cache_read_input_token_cost": 1.5e-06, @@ -25265,26 +23724,6 @@ "supports_tool_choice": true, "supports_vision": false }, - "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro": { - "cache_read_input_token_cost": 6e-07, - "cache_read_input_token_cost_priority": 6e-07, - "deprecation_date": "2026-08-27", - "input_cost_per_token": 1.2e-06, - "input_cost_per_token_priority": 1.2e-06, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_priority": 1.2e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false - }, "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813": { "cache_read_input_token_cost": 4.4e-08, "cache_read_input_token_cost_priority": 5.5e-08, @@ -25672,26 +24111,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "fireworks_ai/accounts/fireworks/models/minimax-m2p7": { - "cache_read_input_token_cost": 6e-08, - "cache_read_input_token_cost_priority": 6e-07, - "deprecation_date": "2026-08-27", - "input_cost_per_token": 3e-07, - "input_cost_per_token_priority": 1.2e-06, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 196608, - "max_output_tokens": 196608, - "max_tokens": 196608, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_priority": 1.2e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false - }, "fireworks_ai/accounts/fireworks/models/minimax-m3": { "cache_read_input_token_cost": 6e-08, "cache_read_input_token_cost_priority": 9e-08, @@ -25779,26 +24198,6 @@ "supports_tool_choice": true, "supports_vision": false }, - "fireworks_ai/deepseek-v4-pro": { - "cache_read_input_token_cost": 6e-07, - "cache_read_input_token_cost_priority": 6e-07, - "deprecation_date": "2026-08-27", - "input_cost_per_token": 1.2e-06, - "input_cost_per_token_priority": 1.2e-06, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_priority": 1.2e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false - }, "fireworks_ai/glm-4p7": { "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 6e-07, @@ -25998,26 +24397,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "fireworks_ai/minimax-m2p7": { - "cache_read_input_token_cost": 6e-08, - "cache_read_input_token_cost_priority": 6e-07, - "deprecation_date": "2026-08-27", - "input_cost_per_token": 3e-07, - "input_cost_per_token_priority": 1.2e-06, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 196608, - "max_output_tokens": 196608, - "max_tokens": 196608, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_priority": 1.2e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false - }, "fireworks_ai/minimax-m3": { "cache_read_input_token_cost": 6e-08, "cache_read_input_token_cost_priority": 9e-08, @@ -26195,31 +24574,6 @@ "comment": "Open flagship GLM for long-horizon coding agents and million-token context work", "source": "https://api.friendli.ai/serverless/v1/models" }, - "friendliai/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B": { - "litellm_provider": "friendliai", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "input_cost_per_token": 6e-07, - "output_cost_per_token": 2.4e-06, - "cache_read_input_token_cost": 1.2e-07, - "supports_prompt_caching": true, - "supports_reasoning": true, - "reasoning_effort_levels": [], - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_native_structured_output": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_image_input": false, - "supports_video_input": false, - "mode": "chat", - "comment": "Frontier-scale multilingual language model developed by LG AI Research", - "deprecation_date": "2026-09-06", - "source": "https://api.friendli.ai/serverless/v1/models" - }, "friendliai/deepseek-ai/DeepSeek-V3.2": { "litellm_provider": "friendliai", "max_input_tokens": 163840, @@ -26525,160 +24879,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "gemini-2.0-flash": { - "cache_read_input_token_cost": 2.5e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 1e-06, - "input_cost_per_audio_token_batches": 5e-07, - "input_cost_per_character": 3.75e-08, - "input_cost_per_token": 1.5e-07, - "input_cost_per_token_batches": 7.5e-08, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-07, - "output_cost_per_token_batches": 3e-07, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini-2.0-flash-001": { - "cache_read_input_token_cost": 3.75e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 1e-06, - "input_cost_per_token": 1.5e-07, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini-2.0-flash-lite": { - "cache_read_input_token_cost": 1.875e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7.5e-08, - "input_cost_per_audio_token_batches": 3.75e-08, - "input_cost_per_character": 1.875e-08, - "input_cost_per_token": 7.5e-08, - "input_cost_per_token_batches": 3.75e-08, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "output_cost_per_token_batches": 1.5e-07, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini-2.0-flash-lite-001": { - "cache_read_input_token_cost": 1.875e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7.5e-08, - "input_cost_per_token": 7.5e-08, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, "gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, "deprecation_date": "2026-10-20", @@ -27503,53 +25703,6 @@ }, "gemini_native_audio": true }, - "gemini-2.5-flash-lite-preview-06-17": { - "deprecation_date": "2025-11-18", - "cache_read_input_token_cost": 1e-08, - "input_cost_per_audio_token": 5e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_reasoning_token": 4e-07, - "output_cost_per_token": 4e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "google_maps_grounding_cost_per_query": 0.025, - "supports_image_size": false - }, "gemini-2.5-pro": { "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 1.25e-07, @@ -27608,62 +25761,6 @@ "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 1.8e-05 }, - "gemini-3-pro-preview": { - "deprecation_date": "2026-03-26", - "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, - "input_cost_per_token": 2e-06, - "input_cost_per_token_above_200k_tokens": 4e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "output_cost_per_token_above_200k_tokens": 1.8e-05, - "output_cost_per_token_batches": 6e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_video_input": true, - "supports_vision": true, - "supports_web_search": true, - "supports_native_streaming": true, - "input_cost_per_token_priority": 3.6e-06, - "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, - "output_cost_per_token_priority": 2.16e-05, - "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, - "cache_read_input_token_cost_priority": 3.6e-07, - "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query" - }, "gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, @@ -28322,51 +26419,6 @@ "supports_url_context": true, "supports_vision": true }, - "gemini/gemini-robotics-er-1.5-preview": { - "cache_read_input_token_cost": 0, - "deprecation_date": "2026-04-30", - "input_cost_per_token": 3e-07, - "input_cost_per_audio_token": 1e-06, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "output_cost_per_reasoning_token": 2.5e-06, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-robotics-er-1-5-preview", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions" - ], - "supported_modalities": [ - "text", - "image", - "video", - "audio" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": false, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000, - "rpm": 10, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, "gemini/gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_batches": 5e-08, @@ -28417,53 +26469,6 @@ "supports_web_search": true, "web_search_billing_unit": "per_query" }, - "gemini/gemini-robotics-er-1.6-preview": { - "deprecation_date": "2026-08-31", - "input_cost_per_audio_token": 2e-06, - "input_cost_per_token": 1e-06, - "litellm_provider": "gemini", - "max_input_tokens": 131072, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_reasoning_token": 5e-06, - "output_cost_per_token": 5e-06, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-robotics-er", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_video_input": true, - "supports_vision": true, - "supports_web_search": true, - "web_search_billing_unit": "per_query" - }, "gemini-2.5-computer-use-preview-10-2025": { "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, @@ -28600,27 +26605,6 @@ "source": "https://ai.google.dev/gemini-api/docs/embeddings#model-versions", "tpm": 10000000 }, - "gemini/gemini-embedding-2-preview": { - "deprecation_date": "2026-08-10", - "input_cost_per_audio_token": 6.5e-06, - "input_cost_per_audio_token_batches": 3.25e-06, - "input_cost_per_image_token": 4.5e-07, - "input_cost_per_image_token_batches": 2.25e-07, - "input_cost_per_token": 2e-07, - "input_cost_per_token_batches": 1e-07, - "input_cost_per_video_token": 1.2e-05, - "input_cost_per_video_token_batches": 6e-06, - "litellm_provider": "gemini", - "max_input_tokens": 8192, - "max_tokens": 8192, - "mode": "embedding", - "output_cost_per_token": 0, - "output_vector_size": 3072, - "rpm": 10000, - "source": "https://ai.google.dev/gemini-api/docs/pricing", - "supports_multimodal": true, - "tpm": 10000000 - }, "gemini/gemini-embedding-2": { "input_cost_per_audio_token": 6.5e-06, "input_cost_per_audio_token_batches": 3.25e-06, @@ -28643,135 +26627,6 @@ "supports_vision": true, "tpm": 10000000 }, - "gemini/gemini-1.5-flash": { - "deprecation_date": "2025-09-29", - "input_cost_per_token": 7.5e-08, - "input_cost_per_token_above_128k_tokens": 1.5e-07, - "litellm_provider": "gemini", - "max_input_tokens": 8192, - "max_tokens": 8192, - "mode": "embedding", - "output_cost_per_token": 0, - "output_vector_size": 3072, - "rpm": 10000, - "source": "https://ai.google.dev/gemini-api/docs/embeddings#multimodal", - "supports_multimodal": true, - "tpm": 10000000 - }, - "gemini/gemini-2.0-flash": { - "cache_read_input_token_cost": 2.5e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4e-07, - "rpm": 10000, - "source": "https://ai.google.dev/pricing#2_0flash", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 10000000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini/gemini-2.0-flash-001": { - "cache_read_input_token_cost": 2.5e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4e-07, - "rpm": 10000, - "source": "https://ai.google.dev/pricing#2_0flash", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 10000000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini/gemini-2.0-flash-lite": { - "cache_read_input_token_cost": 1.875e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7.5e-08, - "input_cost_per_token": 7.5e-08, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "rpm": 4000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.0-flash-lite", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 4000000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, "gemini/gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, "cache_read_input_token_cost": 3e-08, @@ -28934,50 +26789,6 @@ "web_search_billing_unit": "per_query", "supports_reasoning": false }, - "gemini/gemini-3-pro-image-preview": { - "deprecation_date": "2026-06-25", - "input_cost_per_image": 0.0011, - "input_cost_per_token": 2e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "gemini", - "max_input_tokens": 65536, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "image_generation", - "output_cost_per_image": 0.134, - "output_cost_per_image_token": 0.00012, - "output_cost_per_token": 1.2e-05, - "rpm": 1000, - "tpm": 4000000, - "output_cost_per_token_batches": 6e-06, - "source": "https://ai.google.dev/gemini-api/docs/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_prompt_caching": true, - "supports_reasoning": false, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query" - }, "gemini/nano-banana-pro-preview": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -29063,49 +26874,6 @@ }, "web_search_billing_unit": "per_query" }, - "gemini/gemini-3.1-flash-image-preview": { - "deprecation_date": "2026-06-25", - "input_cost_per_token": 5e-07, - "input_cost_per_token_batches": 2.5e-07, - "litellm_provider": "gemini", - "max_input_tokens": 65536, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "image_generation", - "output_cost_per_image": 0.045, - "output_cost_per_image_token": 6e-05, - "output_cost_per_token": 3e-06, - "output_cost_per_token_batches": 1.5e-06, - "rpm": 1000, - "tpm": 4000000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-image-preview", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_prompt_caching": true, - "supports_reasoning": false, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query" - }, "gemini/gemini-3.1-flash-lite-image": { "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -29244,104 +27012,6 @@ "supports_audio_input": true, "supports_image_size": false }, - "gemini/gemini-2.5-flash-lite-preview-09-2025": { - "cache_read_input_token_cost": 1e-08, - "deprecation_date": "2026-03-31", - "input_cost_per_audio_token": 3e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_reasoning_token": 4e-07, - "output_cost_per_token": 4e-07, - "rpm": 15, - "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "google_maps_grounding_cost_per_query": 0.025, - "supports_image_size": false - }, - "gemini/gemini-2.5-flash-preview-09-2025": { - "cache_read_input_token_cost": 3e-08, - "deprecation_date": "2026-02-17", - "input_cost_per_audio_token": 1e-06, - "input_cost_per_token": 3e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_reasoning_token": 2.5e-06, - "output_cost_per_token": 2.5e-06, - "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "google_maps_grounding_cost_per_query": 0.025, - "supports_image_size": false - }, "gemini/gemini-flash-latest": { "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, @@ -29459,55 +27129,6 @@ "supports_video_input": true, "web_search_billing_unit": "per_query" }, - "gemini/gemini-2.5-flash-lite-preview-06-17": { - "deprecation_date": "2025-11-18", - "cache_read_input_token_cost": 1e-08, - "input_cost_per_audio_token": 5e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_reasoning_token": 4e-07, - "output_cost_per_token": 4e-07, - "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "google_maps_grounding_cost_per_query": 0.025, - "supports_image_size": false - }, "gemini/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, @@ -29618,114 +27239,6 @@ "supports_vision": true, "tpm": 800000 }, - "gemini/gemini-3-pro-preview": { - "deprecation_date": "2026-03-09", - "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "input_cost_per_token": 2e-06, - "input_cost_per_token_above_200k_tokens": 4e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "output_cost_per_token_above_200k_tokens": 1.8e-05, - "output_cost_per_token_batches": 6e-06, - "rpm": 2000, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_video_input": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 800000, - "input_cost_per_token_priority": 3.6e-06, - "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, - "output_cost_per_token_priority": 2.16e-05, - "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, - "cache_read_input_token_cost_priority": 3.6e-07, - "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query" - }, - "gemini/gemini-3.1-flash-lite-preview": { - "cache_read_input_token_cost": 2.5e-08, - "deprecation_date": "2026-05-25", - "input_cost_per_audio_token": 5e-07, - "input_cost_per_token": 2.5e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_reasoning_token": 1.5e-06, - "output_cost_per_token": 1.5e-06, - "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/models", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_video_input": true, - "supports_vision": true, - "supports_web_search": true, - "supports_native_streaming": true, - "tpm": 250000, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 - }, "gemini/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, "cache_read_input_token_cost": 2.5e-08, @@ -30826,34 +28339,6 @@ "output_cost_per_image": 0.04, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, - "gemini/imagen-3.0-generate-002": { - "deprecation_date": "2025-11-10", - "litellm_provider": "gemini", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "gemini/imagen-4.0-fast-generate-001": { - "deprecation_date": "2026-08-17", - "litellm_provider": "gemini", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "gemini/imagen-4.0-generate-001": { - "deprecation_date": "2026-08-17", - "litellm_provider": "gemini", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "gemini/imagen-4.0-ultra-generate-001": { - "deprecation_date": "2026-08-17", - "litellm_provider": "gemini", - "mode": "image_generation", - "output_cost_per_image": 0.06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, "gemini/learnlm-1.5-pro-experimental": { "input_cost_per_audio_per_second": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -30932,21 +28417,6 @@ "supports_web_search": false, "output_cost_per_image": 0.08 }, - "gemini/veo-2.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "gemini", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "video_generation", - "output_cost_per_second": 0.35, - "source": "https://ai.google.dev/gemini-api/docs/video", - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "video" - ] - }, "gemini/veo-3.1-fast-generate-preview": { "litellm_provider": "gemini", "max_input_tokens": 1024, @@ -32156,33 +29626,6 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "gpt-4-0125-preview": { - "deprecation_date": "2026-03-26", - "input_cost_per_token": 1e-05, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 3e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, - "gpt-4-0314": { - "deprecation_date": "2026-03-26", - "input_cost_per_token": 3e-05, - "litellm_provider": "openai", - "max_input_tokens": 8192, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-05, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4-0613": { "deprecation_date": "2026-10-23", "input_cost_per_token": 3e-05, @@ -32252,22 +29695,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "gpt-4-turbo-preview": { - "deprecation_date": "2026-03-26", - "input_cost_per_token": 1e-05, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 3e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4.1": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_priority": 8.75e-07, @@ -32610,24 +30037,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "gpt-4o-audio-preview": { - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 4e-05, - "input_cost_per_token": 2.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_audio_token": 8e-05, - "output_cost_per_token": 1e-05, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4o-audio-preview-2024-12-17": { "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 4e-05, @@ -32812,44 +30221,6 @@ "supports_tool_choice": true, "supports_vision": false }, - "gpt-audio-mini-2025-10-06": { - "deprecation_date": "2026-07-23", - "input_cost_per_audio_token": 1e-05, - "input_cost_per_token": 6e-07, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 2.4e-06, - "source": "https://developers.openai.com/api/docs/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses", - "/v1/realtime", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "audio" - ], - "supported_output_modalities": [ - "text", - "audio" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": false, - "supports_reasoning": false, - "supports_response_schema": false, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": false - }, "gpt-audio-mini-2025-12-15": { "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, @@ -32946,24 +30317,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "gpt-4o-mini-audio-preview": { - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 1e-05, - "input_cost_per_token": 1.5e-07, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 6e-07, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4o-mini-audio-preview-2024-12-17": { "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 1e-05, @@ -32982,26 +30335,6 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "gpt-4o-mini-realtime-preview": { - "cache_creation_input_audio_token_cost": 3e-07, - "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 1e-05, - "input_cost_per_token": 6e-07, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 2.4e-06, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4o-mini-realtime-preview-2024-12-17": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 3e-07, @@ -33048,32 +30381,6 @@ "supports_vision": true, "supports_web_search": true }, - "gpt-4o-mini-search-preview-2025-03-11": { - "cache_read_input_token_cost": 7.5e-08, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.5e-07, - "input_cost_per_token_batches": 7.5e-08, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 6e-07, - "output_cost_per_token_batches": 3e-07, - "search_context_cost_per_query": { - "search_context_size_high": 0.025, - "search_context_size_low": 0.025, - "search_context_size_medium": 0.025 - }, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "gpt-4o-mini-transcribe": { "input_cost_per_audio_token": 1.25e-06, "input_cost_per_token": 1.25e-06, @@ -33108,63 +30415,6 @@ "audio" ] }, - "gpt-4o-realtime-preview": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 4e-05, - "input_cost_per_token": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 8e-05, - "output_cost_per_token": 2e-05, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, - "gpt-4o-realtime-preview-2024-12-17": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 4e-05, - "input_cost_per_token": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 8e-05, - "output_cost_per_token": 2e-05, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, - "gpt-4o-realtime-preview-2025-06-03": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 4e-05, - "input_cost_per_token": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 8e-05, - "output_cost_per_token": 2e-05, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4o-search-preview": { "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, @@ -33191,32 +30441,6 @@ "supports_vision": true, "supports_web_search": true }, - "gpt-4o-search-preview-2025-03-11": { - "cache_read_input_token_cost": 1.25e-06, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 2.5e-06, - "input_cost_per_token_batches": 1.25e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "output_cost_per_token_batches": 5e-06, - "search_context_cost_per_query": { - "search_context_size_high": 0.025, - "search_context_size_low": 0.025, - "search_context_size_medium": 0.025 - }, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "gpt-4o-transcribe": { "input_cost_per_audio_token": 2.5e-06, "input_cost_per_token": 2.5e-06, @@ -33879,52 +31103,6 @@ "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": false }, - "gpt-5.1-chat-latest": { - "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_priority": 2.5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "input_cost_per_token_priority": 2.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "output_cost_per_token_priority": 2e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_native_streaming": true, - "supports_parallel_function_calling": false, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": false, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, "gpt-5.2": { "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_batches": 8.75e-08, @@ -34031,94 +31209,6 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, - "gpt-5.2-chat-latest": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-08-10", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.3-chat-latest": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-08-10", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, "gpt-5.2-pro": { "input_cost_per_token": 2.1e-05, "input_cost_per_token_batches": 1.05e-05, @@ -35783,251 +32873,6 @@ "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": true }, - "gpt-5-chat-latest": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": false, - "supports_native_streaming": true, - "supports_parallel_function_calling": false, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": false, - "supports_vision": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5-codex": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 1e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.1-codex": { - "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_priority": 2.5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "input_cost_per_token_priority": 2.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 1e-05, - "output_cost_per_token_priority": 2e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.1-codex-max": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 1e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.1-codex-mini": { - "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_priority": 4.5e-08, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 2.5e-07, - "input_cost_per_token_priority": 4.5e-07, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 2e-06, - "output_cost_per_token_priority": 3.6e-06, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.2-codex": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true - }, "gpt-5.3-codex": { "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, @@ -36870,32 +33715,6 @@ "supports_response_schema": true, "supports_vision": true }, - "groq/llama-3.1-8b-instant": { - "deprecation_date": "2026-08-16", - "input_cost_per_token": 5e-08, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 8e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.3-70b-versatile": { - "deprecation_date": "2026-08-16", - "input_cost_per_token": 5.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 7.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, "groq/llama-guard-3-8b": { "input_cost_per_token": 2e-07, "litellm_provider": "groq", @@ -36905,19 +33724,6 @@ "output_cost_per_token": 2e-07, "source": "https://console.groq.com/docs/model/llama-guard-3-8b" }, - "groq/gemma-7b-it": { - "deprecation_date": "2024-12-18", - "input_cost_per_token": 5e-08, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 8e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, "groq/meta-llama/llama-prompt-guard-2-22m": { "input_cost_per_token": 3e-08, "litellm_provider": "groq", @@ -36938,58 +33744,6 @@ "output_cost_per_token": 4e-08, "source": "https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m" }, - "groq/meta-llama/llama-guard-4-12b": { - "deprecation_date": "2026-03-05", - "input_cost_per_token": 2e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-07 - }, - "groq/meta-llama/llama-4-maverick-17b-128e-instruct": { - "deprecation_date": "2026-03-09", - "input_cost_per_token": 2e-07, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/meta-llama/llama-4-scout-17b-16e-instruct": { - "deprecation_date": "2026-07-17", - "input_cost_per_token": 1.1e-07, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3.4e-07, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/moonshotai/kimi-k2-instruct-0905": { - "deprecation_date": "2026-04-15", - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3e-06, - "cache_read_input_token_cost": 5e-07, - "litellm_provider": "groq", - "max_input_tokens": 262144, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "groq/openai/gpt-oss-120b": { "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.5e-07, @@ -37070,45 +33824,6 @@ "mode": "audio_speech", "source": "https://console.groq.com/docs/models" }, - "groq/playai-tts": { - "deprecation_date": "2025-12-31", - "input_cost_per_character": 5e-05, - "litellm_provider": "groq", - "max_input_tokens": 10000, - "max_output_tokens": 10000, - "max_tokens": 10000, - "mode": "audio_speech" - }, - "groq/qwen/qwen3.6-27b": { - "input_cost_per_token": 6e-07, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://console.groq.com/docs/model/qwen/qwen3.6-27b", - "deprecation_date": "2026-09-14", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/qwen/qwen3-32b": { - "deprecation_date": "2026-07-17", - "input_cost_per_token": 2.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 131000, - "max_output_tokens": 131000, - "max_tokens": 131000, - "mode": "chat", - "output_cost_per_token": 5.9e-07, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, "groq/whisper-large-v3": { "input_cost_per_second": 3.083e-05, "litellm_provider": "groq", @@ -37121,27 +33836,6 @@ "mode": "audio_transcription", "output_cost_per_second": 0.0 }, - "hd/1024-x-1024/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 7.629e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, - "hd/1024-x-1792/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 6.539e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, - "hd/1792-x-1024/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 6.539e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "heroku/claude-3-5-haiku": { "litellm_provider": "heroku", "max_tokens": 8192, @@ -38968,19 +35662,6 @@ "supports_system_messages": true, "supports_native_structured_output": true }, - "mistral/codestral-2405": { - "deprecation_date": "2025-06-16", - "input_cost_per_token": 1e-06, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 3e-06, - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/codestral-2508": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, @@ -39024,51 +35705,6 @@ "supports_assistant_prefill": true, "supports_tool_choice": true }, - "mistral/devstral-medium-2507": { - "deprecation_date": "2026-05-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://mistral.ai/news/devstral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/devstral-small-2505": { - "deprecation_date": "2025-11-30", - "input_cost_per_token": 1e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://mistral.ai/news/devstral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/devstral-small-2507": { - "deprecation_date": "2026-05-31", - "input_cost_per_token": 1e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://mistral.ai/news/devstral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/devstral-small-latest": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, @@ -39084,21 +35720,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/labs-devstral-small-2512": { - "deprecation_date": "2026-03-31", - "input_cost_per_token": 1e-07, - "litellm_provider": "mistral", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://docs.mistral.ai/models/devstral-small-2-25-12", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/devstral-latest": { "cache_read_input_token_cost": 4e-08, "input_cost_per_token": 4e-07, @@ -39129,21 +35750,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/devstral-2512": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://mistral.ai/news/devstral-2-vibe-cli", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/ministral-14b-2512": { "input_cost_per_token": 2e-07, "litellm_provider": "mistral", @@ -39419,54 +36025,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/magistral-medium-2506": { - "deprecation_date": "2025-11-30", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/magistral-medium-2509": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/magistral-medium-1-2-2509": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/mistral-ocr-latest": { "litellm_provider": "mistral", "ocr_cost_per_page": 0.004, @@ -39506,20 +36064,6 @@ "/v1/batch" ] }, - "mistral/mistral-ocr-2505-completion": { - "deprecation_date": "2026-05-31", - "litellm_provider": "mistral", - "ocr_cost_per_page": 0.001, - "ocr_cost_per_page_batches": 0.0005, - "annotation_cost_per_page": 0.003, - "annotation_cost_per_page_batches": 0.0015, - "mode": "ocr", - "supported_endpoints": [ - "/v1/ocr", - "/v1/batch" - ], - "source": "https://mistral.ai/pricing#api-pricing" - }, "mistral/mistral-ocr-2512": { "litellm_provider": "mistral", "ocr_cost_per_page": 0.002, @@ -39550,22 +36094,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/magistral-small-2506": { - "deprecation_date": "2025-11-30", - "input_cost_per_token": 5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://mistral.ai/pricing#api-pricing", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/magistral-small-latest": { "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, @@ -39583,22 +36111,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/magistral-small-1-2-2509": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://mistral.ai/pricing#api-pricing", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/mistral-embed": { "input_cost_per_token": 1e-07, "litellm_provider": "mistral", @@ -39622,48 +36134,6 @@ "max_tokens": 8192, "mode": "embedding" }, - "mistral/mistral-large-2402": { - "deprecation_date": "2025-06-16", - "input_cost_per_token": 4e-06, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/mistral-large-2407": { - "deprecation_date": "2025-03-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 9e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/mistral-large-2411": { - "deprecation_date": "2026-05-31", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 6e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/mistral-large-latest": { "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 5e-07, @@ -39733,49 +36203,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/mistral-medium-2312": { - "deprecation_date": "2025-06-16", - "input_cost_per_token": 2.7e-06, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 8.1e-06, - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/mistral-medium-2505": { - "deprecation_date": "2026-08-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 2e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/mistral-medium-2508": { - "deprecation_date": "2026-08-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://mistral.ai/news/mistral-medium-3", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "mistral/mistral-medium-2604": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, @@ -39818,22 +36245,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/mistral-medium-3-1-2508": { - "deprecation_date": "2026-08-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://mistral.ai/news/mistral-medium-3", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "mistral/mistral-medium-3-5": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, @@ -39890,22 +36301,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/mistral-small-3-2-2506": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 6e-08, - "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "source": "https://mistral.ai/pricing", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "mistral/ministral-3-3b-2512": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, @@ -39999,32 +36394,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/open-codestral-mamba": { - "deprecation_date": "2025-06-06", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2.5e-07, - "source": "https://mistral.ai/technology/", - "supports_assistant_prefill": true, - "supports_tool_choice": true - }, - "mistral/open-mistral-7b": { - "deprecation_date": "2025-03-30", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 2.5e-07, - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/open-mistral-nemo": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, @@ -40039,78 +36408,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/open-mistral-nemo-2407": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 3e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://mistral.ai/technology/", - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/open-mixtral-8x22b": { - "deprecation_date": "2025-03-30", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 65336, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 6e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/open-mixtral-8x7b": { - "deprecation_date": "2025-03-30", - "input_cost_per_token": 7e-07, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 7e-07, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/pixtral-12b-2409": { - "deprecation_date": "2025-12-31", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.5e-07, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "mistral/pixtral-large-2411": { - "deprecation_date": "2026-05-31", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 6e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "mistral/pixtral-large-latest": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 2e-06, @@ -40159,36 +36456,6 @@ "supports_audio_input": false, "supports_response_schema": true }, - "moonshot/kimi-k2-0711-preview": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 6e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "moonshot/kimi-k2-0905-preview": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 6e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, "moonshot/kimi-k2.7-code": { "cache_read_input_token_cost": 1.9e-07, "input_cost_per_token": 9.5e-07, @@ -40207,21 +36474,6 @@ "supports_video_input": true, "supports_vision": true }, - "moonshot/kimi-k2-turbo-preview": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 1.15e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 8e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, "moonshot/kimi-k2.5": { "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 6e-07, @@ -40278,111 +36530,6 @@ "supports_video_input": true, "supports_vision": true }, - "moonshot/kimi-latest": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-01-28", - "input_cost_per_token": 2e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "moonshot/kimi-latest-128k": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-01-28", - "input_cost_per_token": 2e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "moonshot/kimi-latest-32k": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-01-28", - "input_cost_per_token": 1e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "moonshot/kimi-latest-8k": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-01-28", - "input_cost_per_token": 2e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "moonshot/kimi-thinking-preview": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2025-11-11", - "input_cost_per_token": 6e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_vision": true - }, - "moonshot/kimi-k2-thinking": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 6e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "moonshot/kimi-k2-thinking-turbo": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 1.15e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 8e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_web_search": true - }, "moonshot/moonshot-v1-128k": { "input_cost_per_token": 2e-06, "litellm_provider": "moonshot", @@ -40396,19 +36543,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "moonshot/moonshot-v1-128k-0430": { - "deprecation_date": "2024-04-30", - "input_cost_per_token": 2e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true - }, "moonshot/moonshot-v1-128k-vision-preview": { "input_cost_per_token": 2e-06, "litellm_provider": "moonshot", @@ -40436,19 +36570,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "moonshot/moonshot-v1-32k-0430": { - "deprecation_date": "2024-04-30", - "input_cost_per_token": 1e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true - }, "moonshot/moonshot-v1-32k-vision-preview": { "input_cost_per_token": 1e-06, "litellm_provider": "moonshot", @@ -40476,19 +36597,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "moonshot/moonshot-v1-8k-0430": { - "deprecation_date": "2024-04-30", - "input_cost_per_token": 2e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true - }, "moonshot/moonshot-v1-8k-vision-preview": { "input_cost_per_token": 2e-07, "litellm_provider": "moonshot", @@ -41679,88 +37787,6 @@ "supports_vision": true, "supports_web_search": true }, - "o3-deep-research": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1e-05, - "input_cost_per_token_batches": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 4e-05, - "output_cost_per_token_batches": 2e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, - "o3-deep-research-2025-06-26": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1e-05, - "input_cost_per_token_batches": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 4e-05, - "output_cost_per_token_batches": 2e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, "o3-mini": { "cache_read_input_token_cost": 5.5e-07, "deprecation_date": "2026-10-23", @@ -41946,88 +37972,6 @@ "supports_vision": true, "supports_web_search": true }, - "o4-mini-deep-research": { - "cache_read_input_token_cost": 5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 2e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 8e-06, - "output_cost_per_token_batches": 4e-06, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, - "o4-mini-deep-research-2025-06-26": { - "cache_read_input_token_cost": 5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 2e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 8e-06, - "output_cost_per_token_batches": 4e-06, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, "oci/meta.llama-3.1-8b-instruct": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", @@ -43509,23 +39453,6 @@ "supports_vision": false, "supports_web_search": false }, - "openrouter/google/gemini-2.0-flash-001": { - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4e-07, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/google/gemini-2.5-flash": { "cache_creation_input_token_cost": 8.33333333333333e-08, "cache_read_input_audio_token_cost": 1e-07, @@ -46699,17 +42626,6 @@ "supports_reasoning": true, "supports_system_messages": true }, - "rerank-english-v2.0": { - "input_cost_per_query": 0.002, - "input_cost_per_token": 0.0, - "litellm_provider": "cohere", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "rerank", - "output_cost_per_token": 0.0, - "deprecation_date": "2025-04-30" - }, "rerank-english-v3.0": { "input_cost_per_query": 0.002, "input_cost_per_token": 0.0, @@ -46720,17 +42636,6 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, - "rerank-multilingual-v2.0": { - "input_cost_per_query": 0.002, - "input_cost_per_token": 0.0, - "litellm_provider": "cohere", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "rerank", - "output_cost_per_token": 0.0, - "deprecation_date": "2025-04-30" - }, "rerank-multilingual-v3.0": { "input_cost_per_query": 0.002, "input_cost_per_token": 0.0, @@ -46871,31 +42776,6 @@ "output_cost_per_token": 7e-06, "source": "https://cloud.sambanova.ai/plans/pricing" }, - "sambanova/DeepSeek-R1-Distill-Llama-70B": { - "deprecation_date": "2026-03-20", - "input_cost_per_token": 7e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.4e-06, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, - "sambanova/DeepSeek-V3-0324": { - "deprecation_date": "2026-04-14", - "input_cost_per_token": 3e-06, - "litellm_provider": "sambanova", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 4.5e-06, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "sambanova/Llama-4-Maverick-17B-128E-Instruct": { "input_cost_per_token": 6.3e-07, "litellm_provider": "sambanova", @@ -46913,73 +42793,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "sambanova/Llama-4-Scout-17B-16E-Instruct": { - "deprecation_date": "2025-06-19", - "input_cost_per_token": 4e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "metadata": { - "notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount" - }, - "mode": "chat", - "output_cost_per_token": 7e-07, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "sambanova/Meta-Llama-3.1-405B-Instruct": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 5e-06, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "sambanova/Meta-Llama-3.1-8B-Instruct": { - "deprecation_date": "2026-04-14", - "input_cost_per_token": 1e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "sambanova/Meta-Llama-3.2-1B-Instruct": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 4e-08, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 8e-08, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, - "sambanova/Meta-Llama-3.2-3B-Instruct": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 8e-08, - "litellm_provider": "sambanova", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.6e-07, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, "sambanova/Meta-Llama-3.3-70B-Instruct": { "input_cost_per_token": 6e-07, "litellm_provider": "sambanova", @@ -46993,54 +42806,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "sambanova/Meta-Llama-Guard-3-8B": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 3e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, - "sambanova/QwQ-32B": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 5e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-06, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, - "sambanova/Qwen2-Audio-7B-Instruct": { - "deprecation_date": "2025-06-19", - "input_cost_per_token": 5e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 0.0001, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_audio_input": true - }, - "sambanova/Qwen3-32B": { - "deprecation_date": "2026-04-06", - "input_cost_per_token": 4e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "sambanova/DeepSeek-V3.1": { "max_tokens": 131072, "max_input_tokens": 131072, @@ -47633,27 +43398,6 @@ "mode": "image_generation", "output_cost_per_image": 0.14 }, - "standard/1024-x-1024/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 3.81469e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, - "standard/1024-x-1792/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 4.359e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, - "standard/1792-x-1024/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 4.359e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "linkup/search": { "input_cost_per_query": 0.00587, "litellm_provider": "linkup", @@ -47788,36 +43532,6 @@ "output_vector_size": 768, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, - "text-moderation-007": { - "deprecation_date": "2025-10-27", - "input_cost_per_token": 0.0, - "litellm_provider": "openai", - "max_input_tokens": 32768, - "max_output_tokens": 0, - "max_tokens": 0, - "mode": "moderation", - "output_cost_per_token": 0.0 - }, - "text-moderation-latest": { - "deprecation_date": "2025-10-27", - "input_cost_per_token": 0.0, - "litellm_provider": "openai", - "max_input_tokens": 32768, - "max_output_tokens": 0, - "max_tokens": 0, - "mode": "moderation", - "output_cost_per_token": 0.0 - }, - "text-moderation-stable": { - "deprecation_date": "2025-10-27", - "input_cost_per_token": 0.0, - "litellm_provider": "openai", - "max_input_tokens": 32768, - "max_output_tokens": 0, - "max_tokens": 0, - "mode": "moderation", - "output_cost_per_token": 0.0 - }, "text-multilingual-embedding-002": { "deprecation_date": "2027-04-01", "input_cost_per_character": 2.5e-08, @@ -47915,19 +43629,6 @@ "mode": "chat", "output_cost_per_token": 1e-07 }, - "together_ai/Qwen/Qwen2.5-72B-Instruct-Turbo": { - "deprecation_date": "2026-02-06", - "litellm_provider": "together_ai", - "mode": "chat", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "input_cost_per_token": 1.2e-06, - "output_cost_per_token": 1.2e-06, - "max_input_tokens": 131072, - "source": "https://api.together.ai/v1/models" - }, "together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo": { "litellm_provider": "together_ai", "mode": "chat", @@ -47940,87 +43641,6 @@ "max_input_tokens": 32768, "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { - "deprecation_date": "2026-07-10", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262000, - "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { - "deprecation_date": "2026-04-16", - "input_cost_per_token": 6.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": { - "deprecation_date": "2026-02-06", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://www.together.ai/models/qwen3-235b-a22b-fp8-tput", - "supports_function_calling": false, - "supports_parallel_function_calling": false, - "supports_tool_choice": false - }, - "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { - "deprecation_date": "2026-06-04", - "input_cost_per_token": 2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/deepseek-ai/DeepSeek-R1": { - "deprecation_date": "2026-05-14", - "input_cost_per_token": 3e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 128000, - "max_output_tokens": 20480, - "max_tokens": 20480, - "metadata": { - "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" - }, - "mode": "chat", - "output_cost_per_token": 7e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/deepseek-ai/DeepSeek-R1-0528-tput": { - "deprecation_date": "2026-02-03", - "input_cost_per_token": 5.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.19e-06, - "source": "https://www.together.ai/models/deepseek-r1-0528-throughput", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/deepseek-ai/DeepSeek-V3": { "input_cost_per_token": 1.25e-06, "litellm_provider": "together_ai", @@ -48037,33 +43657,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/deepseek-ai/DeepSeek-V3.1": { - "deprecation_date": "2026-05-14", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "max_tokens": 16384, - "metadata": { - "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" - }, - "mode": "chat", - "output_cost_per_token": 1.7e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "max_input_tokens": 131072, - "max_output_tokens": 16384 - }, - "together_ai/meta-llama/Llama-3.2-3B-Instruct-Turbo": { - "deprecation_date": "2026-03-06", - "litellm_provider": "together_ai", - "mode": "chat", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "input_cost_per_token": 1.04e-06, "litellm_provider": "together_ai", @@ -48077,112 +43670,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo-Free": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 0, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 0, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { - "deprecation_date": "2026-03-31", - "input_cost_per_token": 2.7e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8.5e-07, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Llama-4-Scout-17B-16E-Instruct": { - "deprecation_date": "2026-02-06", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 5.9e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo": { - "deprecation_date": "2026-02-06", - "input_cost_per_token": 3.5e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 3.5e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { - "deprecation_date": "2026-02-25", - "input_cost_per_token": 8.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8.8e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { - "deprecation_date": "2026-03-06", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/mistralai/Mistral-7B-Instruct-v0.1": { - "deprecation_date": "2025-11-13", - "litellm_provider": "together_ai", - "mode": "chat", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "input_cost_per_token": 2e-07, - "output_cost_per_token": 2e-07, - "max_input_tokens": 32768, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": { - "deprecation_date": "2026-04-02", - "litellm_provider": "together_ai", - "mode": "chat", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 3e-07, - "max_input_tokens": 32768, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1": { - "deprecation_date": "2026-04-16", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/moonshotai/Kimi-K2-Instruct": { "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", @@ -48211,19 +43698,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/openai/gpt-oss-20b": { - "deprecation_date": "2026-09-14", - "input_cost_per_token": 5e-08, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/togethercomputer/CodeLlama-34b-Instruct": { "litellm_provider": "together_ai", "mode": "chat", @@ -48231,19 +43705,6 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true }, - "together_ai/zai-org/GLM-4.5-Air-FP8": { - "deprecation_date": "2026-04-02", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.1e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/zai-org/GLM-4.6": { "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", @@ -48260,102 +43721,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "together_ai/zai-org/GLM-4.7": { - "deprecation_date": "2026-04-02", - "input_cost_per_token": 4.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 202752, - "max_tokens": 202752, - "metadata": { - "successor": "together_ai/zai-org/GLM-5.2" - }, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, - "together_ai/moonshotai/Kimi-K2.5": { - "deprecation_date": "2026-05-21", - "input_cost_per_token": 5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 256000, - "max_tokens": 256000, - "metadata": { - "successor": "together_ai/moonshotai/Kimi-K3" - }, - "mode": "chat", - "output_cost_per_token": 2.8e-06, - "source": "https://www.together.ai/models/kimi-k2-5", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_reasoning": true - }, - "together_ai/moonshotai/Kimi-K2-Instruct-0905": { - "deprecation_date": "2026-03-06", - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "metadata": { - "successor": "together_ai/moonshotai/Kimi-K3" - }, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://www.together.ai/models/kimi-k2-0905", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3-Next-80B-A3B-Instruct": { - "deprecation_date": "2026-04-02", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "metadata": { - "successor": "together_ai/Qwen/Qwen3.7-Plus" - }, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking": { - "deprecation_date": "2026-02-25", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "metadata": { - "successor": "together_ai/Qwen/Qwen3.6-Plus" - }, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3.5-397B-A17B": { - "cache_read_input_token_cost": 3.5e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 3.6e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/MiniMaxAI/MiniMax-M3": { "cache_read_input_token_cost": 6e-08, "input_cost_per_token": 3e-07, @@ -48477,23 +43842,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/deepseek-ai/DeepSeek-V4-Pro": { - "deprecation_date": "2026-08-27", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.74e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 512000, - "max_tokens": 512000, - "mode": "chat", - "output_cost_per_token": 3.48e-06, - "source": "https://docs.together.ai/docs/serverless-models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": { "cache_read_input_token_cost": 1.3e-07, "deprecation_date": "2026-09-29", @@ -48510,52 +43858,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/google/gemma-3n-E4B-it": { - "deprecation_date": "2026-08-25", - "input_cost_per_token": 6e-08, - "litellm_provider": "together_ai", - "max_input_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 1.2e-07, - "source": "https://docs.together.ai/docs/serverless-models" - }, - "together_ai/google/gemma-4-31B-it": { - "deprecation_date": "2026-09-14", - "input_cost_per_token": 3.9e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 9.7e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "together_ai/intfloat/multilingual-e5-large-instruct": { - "deprecation_date": "2026-09-14", - "input_cost_per_token": 2e-08, - "litellm_provider": "together_ai", - "max_input_tokens": 514, - "max_tokens": 514, - "mode": "embedding", - "output_cost_per_token": 2e-08, - "output_vector_size": 1024, - "source": "https://docs.together.ai/docs/serverless-models" - }, - "together_ai/meta-llama/Llama-Guard-4-12B": { - "deprecation_date": "2026-08-25", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 1048576, - "max_tokens": 1048576, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://docs.together.ai/docs/serverless-models" - }, "together_ai/meta-models/Muse-Glimmer-30B": { "cache_read_input_token_cost": 4e-08, "input_cost_per_token": 3.5e-07, @@ -48567,23 +43869,6 @@ "source": "https://api.together.ai/v1/models", "supports_prompt_caching": true }, - "together_ai/moonshotai/Kimi-K2.7-Code": { - "deprecation_date": "2026-08-27", - "cache_read_input_token_cost": 1.9e-07, - "input_cost_per_token": 9.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 4e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "together_ai/moonshotai/Kimi-K3": { "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, @@ -48606,33 +43891,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "together_ai/nvidia/nemotron-3-ultra-550b-a55b": { - "deprecation_date": "2026-08-27", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 512288, - "max_tokens": 512288, - "mode": "chat", - "output_cost_per_token": 3.6e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/pearl-ai/gemma-4-31b-it": { - "deprecation_date": "2026-08-27", - "input_cost_per_token": 2.8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 8.6e-07, - "source": "https://docs.together.ai/docs/serverless-models" - }, "together_ai/thinkingmachines/Inkling": { "cache_read_input_token_cost": 1.7e-07, "input_cost_per_token": 1e-06, @@ -48648,18 +43906,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/thinkingmachines/Inkling-Small": { - "deprecation_date": "2026-09-14", - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 524288, - "max_tokens": 524288, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://api.together.ai/v1/models", - "supports_prompt_caching": true - }, "together_ai/zai-org/GLM-5.2": { "cache_read_input_token_cost": 2.6e-07, "input_cost_per_token": 1.4e-06, @@ -48923,23 +44169,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "us.anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 - }, "us.anthropic.claude-3-opus-20240229-v1:0": { "input_cost_per_token": 1.5e-05, "litellm_provider": "bedrock", @@ -48955,23 +44184,6 @@ "cache_read_input_token_cost": 1.5e-06, "cache_creation_input_token_cost": 1.875e-05 }, - "us.anthropic.claude-3-sonnet-20240229-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, "us.anthropic.claude-opus-4-1-20250805-v1:0": { "cache_creation_input_token_cost": 1.875e-05, "cache_read_input_token_cost": 1.5e-06, @@ -49036,23 +44248,6 @@ "input_cost_per_token_batches": 1.65e-06, "output_cost_per_token_batches": 8.25e-06 }, - "us-gov.anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 3e-07, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-08, - "cache_creation_input_token_cost": 3.75e-07 - }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, @@ -50180,34 +45375,6 @@ "output_cost_per_token": 9e-07, "supports_tool_choice": true }, - "vercel_ai_gateway/google/gemini-2.0-flash": { - "deprecation_date": "2026-06-01", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "vercel_ai_gateway", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_vision": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true - }, - "vercel_ai_gateway/google/gemini-2.0-flash-lite": { - "deprecation_date": "2026-06-01", - "input_cost_per_token": 7.5e-08, - "litellm_provider": "vercel_ai_gateway", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "supports_vision": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true - }, "vercel_ai_gateway/google/gemini-2.5-flash": { "input_cost_per_token": 3e-07, "litellm_provider": "vercel_ai_gateway", @@ -51091,28 +46258,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "vertex_ai/claude-3-7-sonnet@20250219": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-05-11", - "input_cost_per_token": 3e-06, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "vertex_ai/claude-3-haiku": { "input_cost_per_token": 2.5e-07, "litellm_provider": "vertex_ai-anthropic_models", @@ -51191,72 +46336,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "vertex_ai/claude-opus-4": { - "deprecation_date": "2026-05-14", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "vertex_ai/claude-opus-4-1": { - "deprecation_date": "2026-08-05", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "input_cost_per_token_batches": 7.5e-06, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "output_cost_per_token_batches": 3.75e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "vertex_ai/claude-opus-4-1@20250805": { - "deprecation_date": "2026-08-05", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "input_cost_per_token_batches": 7.5e-06, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "output_cost_per_token_batches": 3.75e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "vertex_ai/claude-opus-4-5": { "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, @@ -51857,98 +46936,6 @@ "supports_native_streaming": true, "prompt_cache_min_tokens": 1024 }, - "vertex_ai/claude-opus-4@20250514": { - "deprecation_date": "2026-05-14", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "vertex_ai/claude-sonnet-4": { - "deprecation_date": "2026-05-14", - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 1000000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "vertex_ai/claude-sonnet-4@20250514": { - "deprecation_date": "2026-05-14", - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 1000000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, "vertex_ai/mistralai/codestral-2@001": { "input_cost_per_token": 3e-07, "litellm_provider": "vertex_ai-mistral_models", @@ -52451,62 +47438,6 @@ "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "cache_read_input_token_cost_batches": 1e-07 }, - "vertex_ai/imagegeneration@006": { - "deprecation_date": "2025-09-24", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-3.0-fast-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-3.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-3.0-generate-002": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-3.0-capability-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/image/edit-insert-objects" - }, - "vertex_ai/imagen-4.0-fast-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-4.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-4.0-ultra-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, "vertex_ai/jamba-1.5": { "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai-ai21_models", @@ -53251,51 +48182,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "vertex_ai/veo-2.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-video-models", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "video_generation", - "output_cost_per_second": 0.35, - "source": "https://ai.google.dev/gemini-api/docs/video", - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "video" - ] - }, - "vertex_ai/veo-3.0-fast-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-video-models", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "video_generation", - "output_cost_per_second": 0.15, - "source": "https://ai.google.dev/gemini-api/docs/video", - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "video" - ] - }, - "vertex_ai/veo-3.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-video-models", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "video_generation", - "output_cost_per_second": 0.4, - "source": "https://ai.google.dev/gemini-api/docs/video", - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "video" - ] - }, "vertex_ai/veo-3.1-generate-preview": { "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, @@ -53584,59 +48470,6 @@ "mode": "chat", "source": "https://wandb.ai/site/pricing/tokens/" }, - "wandb/zai-org/GLM-4.5": { - "deprecation_date": "2026-03-04", - "supports_reasoning": true, - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 0.055, - "output_cost_per_token": 0.2, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/Qwen/Qwen3-235B-A22B-Instruct-2507": { - "deprecation_date": "2026-08-04", - "max_tokens": 262144, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 1e-07, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/Qwen/Qwen3-Coder-480B-A35B-Instruct": { - "deprecation_date": "2026-08-25", - "max_tokens": 262144, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 1e-06, - "output_cost_per_token": 1.5e-06, - "litellm_provider": "wandb", - "mode": "chat", - "source": "https://wandb.ai/site/pricing/tokens/" - }, - "wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": { - "deprecation_date": "2026-08-04", - "supports_reasoning": true, - "max_tokens": 262144, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 1e-07, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/moonshotai/Kimi-K2-Instruct": { - "deprecation_date": "2026-03-04", - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 6e-07, - "output_cost_per_token": 2.5e-06, - "litellm_provider": "wandb", - "mode": "chat" - }, "wandb/moonshotai/Kimi-K2.5": { "max_tokens": 262144, "max_input_tokens": 262144, @@ -53652,20 +48485,6 @@ "supports_response_schema": true, "supports_vision": true }, - "wandb/MiniMaxAI/MiniMax-M2.5": { - "deprecation_date": "2026-08-25", - "max_tokens": 197000, - "max_input_tokens": 197000, - "max_output_tokens": 197000, - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "wandb", - "mode": "chat", - "source": "https://wandb.ai/inference/coreweave/cw_MiniMaxAI_MiniMax-M2.5", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true - }, "wandb/meta-llama/Llama-3.1-8B-Instruct": { "max_tokens": 128000, "max_input_tokens": 131000, @@ -53687,27 +48506,6 @@ "mode": "chat", "source": "https://wandb.ai/site/pricing/tokens/" }, - "wandb/deepseek-ai/DeepSeek-R1-0528": { - "deprecation_date": "2026-03-04", - "supports_reasoning": true, - "max_tokens": 161000, - "max_input_tokens": 161000, - "max_output_tokens": 161000, - "input_cost_per_token": 1.35e-06, - "output_cost_per_token": 5.4e-06, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/deepseek-ai/DeepSeek-V3-0324": { - "deprecation_date": "2026-03-04", - "max_tokens": 161000, - "max_input_tokens": 161000, - "max_output_tokens": 161000, - "input_cost_per_token": 1.14e-06, - "output_cost_per_token": 2.75e-06, - "litellm_provider": "wandb", - "mode": "chat" - }, "wandb/meta-llama/Llama-3.3-70B-Instruct": { "max_tokens": 128000, "max_input_tokens": 128000, @@ -53718,26 +48516,6 @@ "mode": "chat", "source": "https://wandb.ai/site/pricing/tokens/" }, - "wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": { - "deprecation_date": "2026-04-21", - "max_tokens": 64000, - "max_input_tokens": 64000, - "max_output_tokens": 64000, - "input_cost_per_token": 1.7e-07, - "output_cost_per_token": 6.6e-07, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/microsoft/Phi-4-mini-instruct": { - "deprecation_date": "2026-08-04", - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 0.008, - "output_cost_per_token": 0.035, - "litellm_provider": "wandb", - "mode": "chat" - }, "watsonx/ibm/granite-3-8b-instruct": { "input_cost_per_token": 2e-07, "litellm_provider": "watsonx", @@ -54138,440 +48916,6 @@ "deprecation_date": "2027-02-26", "source": "https://developers.openai.com/api/docs/pricing" }, - "xai/grok-3": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-beta": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-fast-beta": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-fast-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini": { - "cache_read_input_token_cost": 2e-07, - "deprecation_date": "2026-02-28", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-beta": { - "cache_read_input_token_cost": 2e-07, - "deprecation_date": "2026-02-28", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-fast": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-02-28", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-fast-beta": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-02-28", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-fast-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-02-28", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-02-28", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4": { - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-fast-reasoning": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-fast-non-reasoning": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-0709": { - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-latest": { - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast-reasoning": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast-reasoning-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast-non-reasoning": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast-non-reasoning-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, "xai/grok-4.20-multi-agent-beta-0309": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.25e-06, @@ -54816,72 +49160,6 @@ "supports_vision": true, "supports_web_search": true }, - "xai/grok-code-fast": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.x.ai/v1/language-models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "input_cost_per_token_above_200k_tokens": 2e-06, - "output_cost_per_token_above_200k_tokens": 4e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "supports_response_schema": true, - "supports_vision": true, - "deprecation_date": "2026-05-15", - "input_cost_per_image_token": 1e-06 - }, - "xai/grok-code-fast-1": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.x.ai/v1/language-models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "input_cost_per_token_above_200k_tokens": 2e-06, - "output_cost_per_token_above_200k_tokens": 4e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "supports_response_schema": true, - "supports_vision": true, - "deprecation_date": "2026-05-15", - "input_cost_per_image_token": 1e-06 - }, - "xai/grok-code-fast-1-0825": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.x.ai/v1/language-models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "input_cost_per_token_above_200k_tokens": 2e-06, - "output_cost_per_token_above_200k_tokens": 4e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "supports_response_schema": true, - "supports_vision": true, - "deprecation_date": "2026-05-15", - "input_cost_per_image_token": 1e-06 - }, "zai.glm-4.7": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", @@ -57624,30 +51902,6 @@ "supports_reasoning": true, "supports_vision": true }, - "scaleway/google/gemma-3-27b-it": { - "input_cost_per_token": 2.5e-07, - "litellm_provider": "scaleway", - "max_input_tokens": 40000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_function_calling": true, - "supports_vision": true, - "deprecation_date": "2026-08-01" - }, - "scaleway/hcompany/holo2-30b-a3b": { - "input_cost_per_token": 3e-07, - "litellm_provider": "scaleway", - "max_input_tokens": 22000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 7e-07, - "supports_reasoning": true, - "supports_vision": true, - "deprecation_date": "2026-08-09" - }, "scaleway/mistralai/mistral-medium-3.5-128b": { "input_cost_per_token": 1.5e-06, "litellm_provider": "scaleway", @@ -57661,29 +51915,6 @@ "supports_vision": true, "supports_tool_choice": true }, - "scaleway/mistralai/devstral-2-123b-instruct-2512": { - "input_cost_per_token": 4e-07, - "litellm_provider": "scaleway", - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 2e-06, - "supports_function_calling": true, - "deprecation_date": "2026-08-01" - }, - "scaleway/mistralai/voxtral-small-24b-2507": { - "input_cost_per_audio_token": 1.5e-07, - "input_cost_per_token": 1.5e-07, - "litellm_provider": "scaleway", - "max_input_tokens": 32000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 3.5e-07, - "supports_audio_input": true, - "deprecation_date": "2026-08-01" - }, "scaleway/mistralai/mistral-small-3.2-24b-instruct-2506": { "input_cost_per_token": 1.5e-07, "litellm_provider": "scaleway", @@ -59243,26 +53474,6 @@ "/v1/audio/speech" ] }, - "gpt-4o-mini-tts-2025-03-20": { - "deprecation_date": "2026-07-23", - "input_cost_per_token": 6e-07, - "litellm_provider": "openai", - "mode": "audio_speech", - "output_cost_per_audio_token": 1.2e-05, - "output_cost_per_second": 0.00025, - "output_cost_per_token": 1e-05, - "source": "https://developers.openai.com/api/docs/pricing", - "supported_endpoints": [ - "/v1/audio/speech" - ], - "supported_modalities": [ - "text", - "audio" - ], - "supported_output_modalities": [ - "audio" - ] - }, "gpt-4o-mini-tts-2025-12-15": { "input_cost_per_token": 6e-07, "litellm_provider": "openai", @@ -59366,41 +53577,6 @@ "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false }, - "gpt-realtime-mini-2025-10-06": { - "cache_creation_input_audio_token_cost": 3e-07, - "cache_read_input_audio_token_cost": 3e-07, - "cache_read_input_token_cost": 6e-08, - "deprecation_date": "2026-07-23", - "input_cost_per_audio_token": 1e-05, - "input_cost_per_image_token": 8e-07, - "input_cost_per_token": 6e-07, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 2.4e-06, - "source": "https://developers.openai.com/api/docs/pricing", - "supported_endpoints": [ - "/v1/realtime" - ], - "supported_modalities": [ - "text", - "image", - "audio" - ], - "supported_output_modalities": [ - "text", - "audio" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-realtime-mini-2025-12-15": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, @@ -59555,42 +53731,6 @@ "tpm": 250000, "rpm": 10 }, - "gemini/gemini-2.0-flash-lite-001": { - "cache_read_input_token_cost": 1.875e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7.5e-08, - "input_cost_per_token": 7.5e-08, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "rpm": 4000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.0-flash-lite", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 4000000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, "gemini-2.5-flash-native-audio-latest": { "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, @@ -65750,23 +59890,6 @@ "image" ] }, - "xai/grok-imagine-image-pro": { - "input_cost_per_image": 0.05, - "litellm_provider": "xai", - "mode": "image_generation", - "source": "https://docs.x.ai/docs/models", - "supported_endpoints": [ - "/v1/images/generations" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "image" - ], - "deprecation_date": "2026-05-15" - }, "xai/grok-imagine-image-2.0": { "input_cost_per_image": 0.06, "litellm_provider": "xai", @@ -66384,16 +60507,6 @@ "output_cost_per_token": 2.82e-07, "source": "https://api.together.ai/v1/models" }, - "together_ai/moonshotai/Kimi-K2.6": { - "deprecation_date": "2026-08-19", - "input_cost_per_token": 1.2e-06, - "output_cost_per_token": 4.5e-06, - "cache_read_input_token_cost": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, "together_ai/moonshotai/Kimi-K2.5-fp4": { "input_cost_per_token": 5e-07, "output_cost_per_token": 2.8e-06, @@ -66411,25 +60524,6 @@ "mode": "chat", "source": "https://api.together.ai/v1/models" }, - "together_ai/zai-org/GLM-5": { - "deprecation_date": "2026-06-22", - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3.2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 202752, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, - "together_ai/zai-org/GLM-5.1": { - "deprecation_date": "2026-07-10", - "input_cost_per_token": 1.4e-06, - "output_cost_per_token": 4.4e-06, - "cache_read_input_token_cost": 2.6e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 202752, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, "together_ai/deepseek-ai/DeepSeek-R1-0528": { "input_cost_per_token": 3e-06, "output_cost_per_token": 7e-06, @@ -66438,33 +60532,6 @@ "mode": "chat", "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen3-Coder-Next-FP8": { - "deprecation_date": "2026-05-14", - "input_cost_per_token": 5e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen3-VL-32B-Instruct": { - "deprecation_date": "2026-02-25", - "input_cost_per_token": 5e-07, - "output_cost_per_token": 1.5e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen3-VL-8B-Instruct": { - "deprecation_date": "2026-04-16", - "input_cost_per_token": 1.8e-07, - "output_cost_per_token": 6.8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, "together_ai/mistralai/Ministral-3-14B-Instruct-2512": { "input_cost_per_token": 2e-07, "output_cost_per_token": 2e-07, @@ -66489,15 +60556,6 @@ "mode": "chat", "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/QwQ-32B": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 1.2e-06, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, "cerebras/gemma-4-31b": { "input_cost_per_token": 9.9e-07, "litellm_provider": "cerebras", @@ -71240,37 +65298,14 @@ "output_cost_per_token": 1.5e-07, "source": "https://api.together.ai/v1/models" }, - "together_ai/deepseek-ai/deepseek-coder-33b-instruct": { - "deprecation_date": "2024-08-22", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { - "deprecation_date": "2025-12-23", - "input_cost_per_token": 2e-06, - "litellm_provider": "together_ai", - "mode": "chat", + "vertex_ai/gemini-2.5-flash-native-audio": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai", + "mode": "realtime", + "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 1.6e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1.6e-06, - "source": "https://api.together.ai/v1/models" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 5e-07, @@ -71321,14 +65356,6 @@ "output_cost_per_token": 6e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, - "together_ai/google/gemma-2-27b-it": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, "gpt-5.5-cyber": { "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 1.25e-05, @@ -71346,14 +65373,6 @@ "output_cost_per_token": 2.5e-05, "source": "https://developers.openai.com/api/docs/pricing" }, - "together_ai/meta-llama/Llama-3-8b-chat-hf": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/meta-llama/Llama-3.1-405B-Instruct": { "input_cost_per_token": 3.5e-06, "litellm_provider": "together_ai", @@ -71375,38 +65394,6 @@ "output_cost_per_token": 6e-08, "source": "https://api.together.ai/v1/models" }, - "together_ai/meta-llama/Meta-Llama-3-70B-Instruct-Turbo": { - "deprecation_date": "2025-12-23", - "input_cost_per_token": 8.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8.8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/meta-llama/Meta-Llama-3-8B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 8.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8.8e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/Qwen/Qwen2-1.5B-Instruct": { "input_cost_per_token": 2e-08, "litellm_provider": "together_ai", @@ -71414,22 +65401,6 @@ "output_cost_per_token": 2e-08, "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen2-72B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 9e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 9e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen2-VL-72B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 1.2e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://api.together.ai/v1/models" - }, "together_ai/Qwen/Qwen2.5-14B-Instruct": { "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", @@ -71444,22 +65415,6 @@ "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen2.5-Coder-32B-Instruct": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen2.5-VL-72B-Instruct": { - "deprecation_date": "2026-01-05", - "input_cost_per_token": 1.95e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8e-06, - "source": "https://api.together.ai/v1/models" - }, "azure/eu/codex-mini": { "deprecation_date": "2026-11-15", "cache_read_input_token_cost": 4.13e-07, @@ -71610,15 +65565,6 @@ "output_cost_per_token_priority": 3.08e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/eu/gpt-5.2-chat": { - "deprecation_date": "2026-06-29", - "cache_read_input_token_cost": 1.925e-07, - "input_cost_per_token": 1.925e-06, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 1.54e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/eu/gpt-5.2-codex": { "deprecation_date": "2027-07-13", "cache_read_input_token_cost": 1.925e-07, @@ -71637,15 +65583,6 @@ "output_cost_per_token_batches": 9.24e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/eu/gpt-5.3-chat": { - "deprecation_date": "2026-06-29", - "cache_read_input_token_cost": 1.925e-07, - "input_cost_per_token": 1.925e-06, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 1.54e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/eu/gpt-5.3-codex": { "deprecation_date": "2027-08-24", "cache_read_input_token_cost": 1.925e-07, @@ -71964,15 +65901,6 @@ "output_cost_per_token_priority": 3.08e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/us/gpt-5.2-chat": { - "deprecation_date": "2026-06-29", - "cache_read_input_token_cost": 1.925e-07, - "input_cost_per_token": 1.925e-06, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 1.54e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/us/gpt-5.2-codex": { "deprecation_date": "2027-07-13", "cache_read_input_token_cost": 1.925e-07, @@ -71991,15 +65919,6 @@ "output_cost_per_token_batches": 9.24e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/us/gpt-5.3-chat": { - "deprecation_date": "2026-06-29", - "cache_read_input_token_cost": 1.925e-07, - "input_cost_per_token": 1.925e-06, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 1.54e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/us/gpt-5.3-codex": { "deprecation_date": "2027-08-24", "cache_read_input_token_cost": 1.925e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 431e965524e..b48bdd6322c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -53,13 +53,6 @@ "mode": "image_generation", "output_cost_per_image": 0.04 }, - "1024-x-1024/dall-e-2": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 1.9e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "1024-x-1024/max-steps/stability.stable-diffusion-xl-v1": { "litellm_provider": "bedrock", "max_input_tokens": 77, @@ -67,13 +60,6 @@ "mode": "image_generation", "output_cost_per_image": 0.08 }, - "256-x-256/dall-e-2": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 2.4414e-07, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "512-x-512/50-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, @@ -81,13 +67,6 @@ "mode": "image_generation", "output_cost_per_image": 0.018 }, - "512-x-512/dall-e-2": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 6.86e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "512-x-512/max-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, @@ -961,23 +940,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 - }, "anthropic.claude-3-opus-20240229-v1:0": { "input_cost_per_token": 1.5e-05, "litellm_provider": "bedrock", @@ -993,23 +955,6 @@ "cache_read_input_token_cost": 1.5e-06, "cache_creation_input_token_cost": 1.875e-05 }, - "anthropic.claude-3-sonnet-20240229-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, "anthropic.claude-instant-v1": { "input_cost_per_token": 8e-07, "litellm_provider": "bedrock", @@ -2971,60 +2916,6 @@ "supports_vision": true, "supports_tool_choice": true }, - "apac.anthropic.claude-3-5-sonnet-20240620-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, - "apac.anthropic.claude-3-5-sonnet-20241022-v2:0": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "apac.anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 - }, "apac.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, "cache_read_input_token_cost": 1.1e-07, @@ -3052,23 +2943,6 @@ "input_cost_per_token_batches": 5.5e-07, "output_cost_per_token_batches": 2.75e-06 }, - "apac.anthropic.claude-3-sonnet-20240229-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, "apac.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, @@ -3446,29 +3320,6 @@ "supports_max_reasoning_effort": true, "prompt_cache_min_tokens": 1024 }, - "azure_ai/claude-opus-4-1": { - "deprecation_date": "2026-08-05", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "azure_ai", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, "azure_ai/claude-sonnet-4-5": { "deprecation_date": "2026-10-19", "cache_creation_input_token_cost": 3.75e-06, @@ -4503,43 +4354,6 @@ "output_cost_per_token_priority": 2.2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/eu/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.375e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.375e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/eu/gpt-5.1-codex": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.375e-07, @@ -4832,43 +4646,6 @@ "output_cost_per_token_priority": 2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/global/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/global/gpt-5.1-codex": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, @@ -4944,19 +4721,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "azure/gpt-3.5-turbo-0125": { - "deprecation_date": "2025-03-31", - "input_cost_per_token": 5e-07, - "litellm_provider": "azure", - "max_input_tokens": 16384, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true - }, "azure/gpt-3.5-turbo-instruct-0914": { "input_cost_per_token": 1.5e-06, "litellm_provider": "azure_text", @@ -4976,32 +4740,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "azure/gpt-35-turbo-0125": { - "deprecation_date": "2025-05-31", - "input_cost_per_token": 5e-07, - "litellm_provider": "azure", - "max_input_tokens": 16384, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true - }, - "azure/gpt-35-turbo-1106": { - "deprecation_date": "2025-03-31", - "input_cost_per_token": 1e-06, - "litellm_provider": "azure", - "max_input_tokens": 16384, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true - }, "azure/gpt-35-turbo-16k": { "input_cost_per_token": 3e-06, "litellm_provider": "azure", @@ -5769,40 +5507,6 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "azure/gpt-realtime-2": { - "cache_read_input_audio_token_cost": 4e-07, - "cache_read_input_token_cost": 4e-07, - "deprecation_date": "2026-08-31", - "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image_token": 5e-06, - "input_cost_per_token": 4e-06, - "litellm_provider": "azure", - "max_input_tokens": 32000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 6.4e-05, - "output_cost_per_token": 2.4e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", - "supported_endpoints": [ - "/v1/realtime" - ], - "supported_modalities": [ - "text", - "image", - "audio" - ], - "supported_output_modalities": [ - "text", - "audio" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "azure/gpt-realtime-2.1": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, @@ -6100,45 +5804,6 @@ "supports_minimal_reasoning_effort": true, "cache_read_input_token_cost_batches": 6.25e-08 }, - "azure/gpt-5.1-chat-2025-11-13": { - "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_priority": 2.5e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "input_cost_per_token_priority": 2.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "output_cost_per_token_priority": 2e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_native_streaming": true, - "supports_parallel_function_calling": false, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": false, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/gpt-5.1-codex-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, @@ -6289,73 +5954,6 @@ "supports_vision": true, "cache_read_input_token_cost_batches": 6.25e-08 }, - "azure/gpt-5-chat": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "azure/gpt-5-chat-latest": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "azure/gpt-5-codex": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2027-03-17", @@ -6617,43 +6215,6 @@ "output_cost_per_token_priority": 2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/gpt-5.1-codex": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, @@ -6832,78 +6393,6 @@ "supports_vision": true, "cache_read_input_token_cost_batches": 8.75e-08 }, - "azure/gpt-5.2-chat": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "azure/gpt-5.2-chat-2025-12-11": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-05-13", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "azure/gpt-5.2-codex": { "cache_read_input_token_cost": 1.75e-07, "deprecation_date": "2027-07-13", @@ -6936,42 +6425,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "azure/gpt-5.3-chat": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "azure/gpt-5.3-codex": { "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, @@ -10563,43 +10016,6 @@ "output_cost_per_token_priority": 2.2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/us/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.375e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.375e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/us/gpt-5.1-codex": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.375e-07, @@ -11201,18 +10617,6 @@ "/v1/images/edits" ] }, - "azure_ai/MAI-Image-2e": { - "deprecation_date": "2026-08-15", - "input_cost_per_token": 5e-06, - "litellm_provider": "azure_ai", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "output_cost_per_image_token": 1.95e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/images/generations" - ] - }, "azure_ai/MAI-Thinking-1": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 2e-06, @@ -11237,34 +10641,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "azure_ai/Llama-3.2-11B-Vision-Instruct": { - "deprecation_date": "2026-06-13", - "input_cost_per_token": 3.7e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 3.7e-07, - "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-11b-vision-instruct-offer?tab=Overview", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "azure_ai/Llama-3.2-90B-Vision-Instruct": { - "deprecation_date": "2026-06-13", - "input_cost_per_token": 2.04e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 2.04e-06, - "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-90b-vision-instruct-offer?tab=Overview", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "azure_ai/Llama-3.3-70B-Instruct": { "input_cost_per_token": 7.1e-07, "litellm_provider": "azure_ai", @@ -11313,18 +10689,6 @@ "output_cost_per_token": 3.7e-07, "supports_tool_choice": true }, - "azure_ai/Meta-Llama-3.1-405B-Instruct": { - "deprecation_date": "2026-06-13", - "input_cost_per_token": 5.33e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 1.6e-05, - "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-405b-instruct-offer?tab=PlansAndPrice", - "supports_tool_choice": true - }, "azure_ai/Meta-Llama-3.1-70B-Instruct": { "input_cost_per_token": 2.68e-06, "litellm_provider": "azure_ai", @@ -11336,18 +10700,6 @@ "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-70b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, - "azure_ai/Meta-Llama-3.1-8B-Instruct": { - "deprecation_date": "2026-06-13", - "input_cost_per_token": 3e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 6.1e-07, - "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-8b-instruct-offer?tab=PlansAndPrice", - "supports_tool_choice": true - }, "azure_ai/Phi-3-medium-128k-instruct": { "input_cost_per_token": 1.7e-07, "litellm_provider": "azure_ai", @@ -11518,16 +10870,6 @@ "supports_tool_choice": true, "supports_reasoning": true }, - "azure_ai/mistral-document-ai-2505": { - "deprecation_date": "2026-07-20", - "litellm_provider": "azure_ai", - "ocr_cost_per_page": 0.003, - "mode": "ocr", - "supported_endpoints": [ - "/v1/ocr" - ], - "source": "https://devblogs.microsoft.com/foundry/whats-new-in-azure-ai-foundry-august-2025/#mistral-document-ai-(ocr)-%E2%80%94-serverless-in-foundry" - }, "azure_ai/mistral-document-ai-2512": { "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.003, @@ -11628,17 +10970,6 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, - "azure_ai/cohere-rerank-v3.5": { - "deprecation_date": "2026-05-14", - "input_cost_per_query": 0.002, - "input_cost_per_token": 0.0, - "litellm_provider": "azure_ai", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "rerank", - "output_cost_per_token": 0.0 - }, "azure_ai/cohere-rerank-v4.0-pro": { "input_cost_per_query": 0.0025, "input_cost_per_token": 0.0, @@ -11691,19 +11022,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "azure_ai/deepseek-r1": { - "deprecation_date": "2026-08-13", - "input_cost_per_token": 1.35e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 5.4e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_reasoning": true, - "supports_tool_choice": true - }, "azure_ai/deepseek-v3": { "input_cost_per_token": 1.14e-06, "litellm_provider": "azure_ai", @@ -11715,33 +11033,6 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true }, - "azure_ai/deepseek-v3-0324": { - "deprecation_date": "2026-07-13", - "input_cost_per_token": 1.14e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4.56e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_tool_choice": true - }, - "azure_ai/deepseek-v3.1": { - "deprecation_date": "2026-07-13", - "input_cost_per_token": 1.23e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 4.94e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "azure_ai/deepseek-v4-pro": { "deprecation_date": "2028-02-20", "input_cost_per_token": 1.74e-06, @@ -11808,68 +11099,6 @@ ], "supports_embedding_image_input": true }, - "azure_ai/global/grok-3": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 3e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true - }, - "azure_ai/global/grok-3-mini": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.27e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true - }, - "azure_ai/grok-3": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 3e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true - }, - "azure_ai/grok-3-mini": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.27e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true - }, "azure_ai/grok-4": { "input_cost_per_token": 3e-06, "litellm_provider": "azure_ai", @@ -11963,36 +11192,6 @@ "supports_vision": true, "supports_web_search": true }, - "azure_ai/grok-4-fast-non-reasoning": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 2e-07, - "output_cost_per_token": 5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "azure_ai/grok-4-fast-reasoning": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 2e-07, - "output_cost_per_token": 5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_web_search": true - }, "azure_ai/grok-4-1-fast-non-reasoning": { "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, @@ -13516,40 +12715,6 @@ "mode": "chat", "output_cost_per_token": 1.5e-06 }, - "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3.6e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3.6e-07, - "cache_creation_input_token_cost": 4.5e-06 - }, - "bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 3e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-08, - "cache_creation_input_token_cost": 3.75e-07 - }, "bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, @@ -13727,61 +12892,6 @@ "mode": "chat", "output_cost_per_token": 1.5e-06 }, - "bedrock/us-gov-west-1/anthropic.claude-3-7-sonnet-20250219-v1:0": { - "cache_creation_input_token_cost": 4.5e-06, - "cache_read_input_token_cost": 3.6e-07, - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3.6e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3.6e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3.6e-07, - "cache_creation_input_token_cost": 4.5e-06 - }, - "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 3e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-08, - "cache_creation_input_token_cost": 3.75e-07 - }, "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, @@ -14219,34 +13329,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "cerebras/zai-glm-4.6": { - "deprecation_date": "2026-01-20", - "input_cost_per_token": 2.25e-06, - "litellm_provider": "cerebras", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.75e-06, - "source": "https://www.cerebras.ai/pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, - "cerebras/zai-glm-4.7": { - "deprecation_date": "2026-08-17", - "input_cost_per_token": 2.25e-06, - "litellm_provider": "cerebras", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.75e-06, - "source": "https://www.cerebras.ai/pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "cerebras/qwen-3.8-27b": { "input_cost_per_token": 9.9e-07, "litellm_provider": "cerebras", @@ -14272,23 +13354,6 @@ "mode": "chat", "output_cost_per_token": 5e-07 }, - "chatgpt-4o-latest": { - "deprecation_date": "2026-02-17", - "input_cost_per_token": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "gpt-4o-transcribe-diarize": { "input_cost_per_audio_token": 2.5e-06, "input_cost_per_token": 2.5e-06, @@ -14351,131 +13416,6 @@ "prompt_cache_min_tokens": 4096, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, - "claude-3-7-sonnet-20250219": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-02-19", - "input_cost_per_token": 3e-06, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, - "claude-3-haiku-20240307": { - "cache_creation_input_token_cost": 3e-07, - "cache_creation_input_token_cost_above_1hr": 5e-07, - "cache_read_input_token_cost": 3e-08, - "deprecation_date": "2026-04-20", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "claude-3-opus-20240229": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "deprecation_date": "2026-01-05", - "input_cost_per_token": 1.5e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "claude-4-opus-20250514": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_read_input_token_cost": 1.5e-06, - "deprecation_date": "2026-06-15", - "input_cost_per_token": 1.5e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "claude-4-sonnet-20250514": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost": 3e-07, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "deprecation_date": "2026-06-15", - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "litellm_provider": "anthropic", - "max_input_tokens": 1000000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "prompt_cache_min_tokens": 1024 - }, "claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -14654,92 +13594,6 @@ "input_cost_per_token_batches": 1.5e-06, "output_cost_per_token_batches": 7.5e-06 }, - "claude-opus-4-1": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_native_structured_output": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024, - "deprecation_date": "2026-08-05" - }, - "claude-opus-4-1-20250805": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "deprecation_date": "2026-08-05", - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_native_structured_output": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "claude-opus-4-20250514": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "deprecation_date": "2026-06-15", - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, "claude-opus-4-5-20251101": { "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -15167,38 +14021,6 @@ "prompt_cache_min_tokens": 1024, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, - "claude-sonnet-4-20250514": { - "deprecation_date": "2026-06-15", - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "litellm_provider": "anthropic", - "max_input_tokens": 1000000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { "input_cost_per_token": 1.923e-06, "litellm_provider": "cloudflare", @@ -15551,36 +14373,6 @@ "supports_assistant_prefill": true, "supports_tool_choice": true }, - "codex-mini-latest": { - "cache_read_input_token_cost": 3.75e-07, - "deprecation_date": "2026-02-12", - "input_cost_per_token": 1.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 6e-06, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "cohere.command-light-text-v14": { "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", @@ -15680,16 +14472,6 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, - "command": { - "input_cost_per_token": 1e-06, - "litellm_provider": "cohere", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "completion", - "output_cost_per_token": 2e-06, - "deprecation_date": "2025-09-15" - }, "command-a-03-2025": { "input_cost_per_token": 2.5e-06, "litellm_provider": "cohere_chat", @@ -15716,17 +14498,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "command-light": { - "input_cost_per_token": 3e-07, - "litellm_provider": "cohere_chat", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_tool_choice": true, - "deprecation_date": "2025-09-15" - }, "command-nightly": { "input_cost_per_token": 1e-06, "litellm_provider": "cohere", @@ -15736,18 +14507,6 @@ "mode": "completion", "output_cost_per_token": 2e-06 }, - "command-r": { - "input_cost_per_token": 1.5e-07, - "litellm_provider": "cohere_chat", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_function_calling": true, - "supports_tool_choice": true, - "deprecation_date": "2025-09-15" - }, "command-r-08-2024": { "input_cost_per_token": 1.5e-07, "litellm_provider": "cohere_chat", @@ -15759,18 +14518,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "command-r-plus": { - "input_cost_per_token": 2.5e-06, - "litellm_provider": "cohere_chat", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "deprecation_date": "2025-09-15" - }, "command-r-plus-08-2024": { "input_cost_per_token": 2.5e-06, "litellm_provider": "cohere_chat", @@ -15823,26 +14570,6 @@ "supports_vision": true, "source": "https://platform.openai.com/docs/models/computer-use-preview" }, - "dall-e-2": { - "deprecation_date": "2026-05-12", - "input_cost_per_image": 0.02, - "litellm_provider": "openai", - "mode": "image_generation", - "supported_endpoints": [ - "/v1/images/generations", - "/v1/images/edits", - "/v1/images/variations" - ] - }, - "dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_image": 0.04, - "litellm_provider": "openai", - "mode": "image_generation", - "supported_endpoints": [ - "/v1/images/generations" - ] - }, "deepseek-chat": { "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 2.8e-07, @@ -18839,30 +17566,6 @@ "output_vector_size": 1024, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, - "databricks/databricks-claude-3-7-sonnet": { - "cache_creation_input_token_cost": 3.74997e-06, - "cache_read_input_token_cost": 3.0002e-07, - "deprecation_date": "2026-04-12", - "input_cost_per_token": 2.9999900000000002e-06, - "input_dbu_cost_per_token": 4.2857e-05, - "litellm_provider": "databricks", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, - "output_dbu_cost_per_token": 0.000214286, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_anthropic_thinking_payload": true, - "supports_tool_choice": true - }, "databricks/databricks-claude-fable-5": { "cache_creation_input_token_cost": 1.250004e-05, "cache_read_input_token_cost": 1.00002e-06, @@ -19825,44 +18528,6 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, - "databricks/databricks-gpt-5-1-codex-max": { - "cache_creation_input_token_cost": 1.24999e-06, - "cache_read_input_token_cost": 1.2502e-07, - "deprecation_date": "2026-07-16", - "input_cost_per_token": 1.24999e-06, - "input_dbu_cost_per_token": 1.7857e-05, - "litellm_provider": "databricks", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 9.999990000000002e-06, - "output_dbu_cost_per_token": 0.000142857, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true - }, - "databricks/databricks-gpt-5-1-codex-mini": { - "cache_creation_input_token_cost": 2.4997e-07, - "cache_read_input_token_cost": 2.499e-08, - "deprecation_date": "2026-07-16", - "input_cost_per_token": 2.4997e-07, - "input_dbu_cost_per_token": 3.571e-06, - "litellm_provider": "databricks", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.99997e-06, - "output_dbu_cost_per_token": 2.8571e-05, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true - }, "databricks/databricks-gpt-5-2": { "cache_creation_input_token_cost": 1.75e-06, "cache_read_input_token_cost": 1.75e-07, @@ -19883,25 +18548,6 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, - "databricks/databricks-gpt-5-2-codex": { - "cache_creation_input_token_cost": 1.75e-06, - "cache_read_input_token_cost": 1.75e-07, - "deprecation_date": "2026-07-16", - "input_cost_per_token": 1.75e-06, - "input_dbu_cost_per_token": 2.5e-05, - "litellm_provider": "databricks", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_dbu_cost_per_token": 0.0002, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true - }, "databricks/databricks-gpt-5-3-codex": { "cache_creation_input_token_cost": 1.75e-06, "cache_read_input_token_cost": 1.75e-07, @@ -20328,25 +18974,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "databricks/databricks-llama-2-70b-chat": { - "cache_creation_input_token_cost": 5.0001e-07, - "cache_read_input_token_cost": 5.0001e-07, - "deprecation_date": "2024-10-30", - "input_cost_per_token": 5.0001e-07, - "input_dbu_cost_per_token": 7.143e-06, - "litellm_provider": "databricks", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.5000300000000002e-06, - "output_dbu_cost_per_token": 2.1429e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, "databricks/databricks-llama-4-maverick": { "cache_creation_input_token_cost": 5.0001e-07, "cache_read_input_token_cost": 5.0001e-07, @@ -20365,25 +18992,6 @@ "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true }, - "databricks/databricks-meta-llama-3-1-405b-instruct": { - "cache_creation_input_token_cost": 5.00003e-06, - "cache_read_input_token_cost": 5.00003e-06, - "deprecation_date": "2026-02-15", - "input_cost_per_token": 5.00003e-06, - "input_dbu_cost_per_token": 7.1429e-05, - "litellm_provider": "databricks", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, - "output_dbu_cost_per_token": 0.000214286, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, "databricks/databricks-meta-llama-3-1-8b-instruct": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, @@ -20419,82 +19027,6 @@ "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true }, - "databricks/databricks-meta-llama-3-70b-instruct": { - "cache_creation_input_token_cost": 1.00002e-06, - "cache_read_input_token_cost": 1.00002e-06, - "deprecation_date": "2024-07-23", - "input_cost_per_token": 1.00002e-06, - "input_dbu_cost_per_token": 1.4286e-05, - "litellm_provider": "databricks", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 2.9999900000000002e-06, - "output_dbu_cost_per_token": 4.2857e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, - "databricks/databricks-mixtral-8x7b-instruct": { - "cache_creation_input_token_cost": 5.0001e-07, - "cache_read_input_token_cost": 5.0001e-07, - "deprecation_date": "2025-04-30", - "input_cost_per_token": 5.0001e-07, - "input_dbu_cost_per_token": 7.143e-06, - "litellm_provider": "databricks", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.00002e-06, - "output_dbu_cost_per_token": 1.4286e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, - "databricks/databricks-mpt-30b-instruct": { - "cache_creation_input_token_cost": 1.00002e-06, - "cache_read_input_token_cost": 1.00002e-06, - "deprecation_date": "2024-08-30", - "input_cost_per_token": 1.00002e-06, - "input_dbu_cost_per_token": 1.4286e-05, - "litellm_provider": "databricks", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.00002e-06, - "output_dbu_cost_per_token": 1.4286e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, - "databricks/databricks-mpt-7b-instruct": { - "cache_creation_input_token_cost": 5.0001e-07, - "cache_read_input_token_cost": 5.0001e-07, - "deprecation_date": "2024-08-30", - "input_cost_per_token": 5.0001e-07, - "input_dbu_cost_per_token": 7.143e-06, - "litellm_provider": "databricks", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 0.0, - "output_dbu_cost_per_token": 0.0, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, "databricks/databricks-qwen35-122b-a10b": { "cache_creation_input_token_cost": 2.2001e-07, "cache_read_input_token_cost": 2.2001e-07, @@ -21559,18 +20091,6 @@ "supports_tool_choice": true, "supports_function_calling": true }, - "deepinfra/google/gemini-2.0-flash-001": { - "deprecation_date": "2026-06-01", - "max_tokens": 1000000, - "max_input_tokens": 1000000, - "max_output_tokens": 1000000, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 4e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true, - "supports_function_calling": true - }, "deepinfra/google/gemini-2.5-flash": { "max_tokens": 1000000, "max_input_tokens": 1000000, @@ -22429,15 +20949,6 @@ "/v1/audio/speech" ] }, - "embed-english-light-v2.0": { - "deprecation_date": "2026-04-04", - "input_cost_per_token": 1e-07, - "litellm_provider": "cohere", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, "embed-english-light-v3.0": { "input_cost_per_token": 1e-07, "litellm_provider": "cohere", @@ -22446,15 +20957,6 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, - "embed-english-v2.0": { - "deprecation_date": "2026-04-04", - "input_cost_per_token": 1e-07, - "litellm_provider": "cohere", - "max_input_tokens": 4096, - "max_tokens": 4096, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, "embed-english-v3.0": { "input_cost_per_image": 0.0001, "input_cost_per_token": 1e-07, @@ -22469,15 +20971,6 @@ "supports_embedding_image_input": true, "supports_image_input": true }, - "embed-multilingual-v2.0": { - "deprecation_date": "2026-04-04", - "input_cost_per_token": 1e-07, - "litellm_provider": "cohere", - "max_input_tokens": 768, - "max_tokens": 768, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, "embed-multilingual-v3.0": { "input_cost_per_token": 1e-07, "litellm_provider": "cohere", @@ -22645,23 +21138,6 @@ "cache_read_input_token_cost": 3e-07, "cache_creation_input_token_cost": 3.75e-06 }, - "eu.anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 - }, "eu.anthropic.claude-3-opus-20240229-v1:0": { "input_cost_per_token": 1.5e-05, "litellm_provider": "bedrock", @@ -22677,23 +21153,6 @@ "cache_read_input_token_cost": 1.5e-06, "cache_creation_input_token_cost": 1.875e-05 }, - "eu.anthropic.claude-3-sonnet-20240229-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, "eu.anthropic.claude-opus-4-1-20250805-v1:0": { "cache_creation_input_token_cost": 1.875e-05, "cache_read_input_token_cost": 1.5e-06, @@ -25265,26 +23724,6 @@ "supports_tool_choice": true, "supports_vision": false }, - "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro": { - "cache_read_input_token_cost": 6e-07, - "cache_read_input_token_cost_priority": 6e-07, - "deprecation_date": "2026-08-27", - "input_cost_per_token": 1.2e-06, - "input_cost_per_token_priority": 1.2e-06, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_priority": 1.2e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false - }, "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813": { "cache_read_input_token_cost": 4.4e-08, "cache_read_input_token_cost_priority": 5.5e-08, @@ -25672,26 +24111,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "fireworks_ai/accounts/fireworks/models/minimax-m2p7": { - "cache_read_input_token_cost": 6e-08, - "cache_read_input_token_cost_priority": 6e-07, - "deprecation_date": "2026-08-27", - "input_cost_per_token": 3e-07, - "input_cost_per_token_priority": 1.2e-06, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 196608, - "max_output_tokens": 196608, - "max_tokens": 196608, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_priority": 1.2e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false - }, "fireworks_ai/accounts/fireworks/models/minimax-m3": { "cache_read_input_token_cost": 6e-08, "cache_read_input_token_cost_priority": 9e-08, @@ -25779,26 +24198,6 @@ "supports_tool_choice": true, "supports_vision": false }, - "fireworks_ai/deepseek-v4-pro": { - "cache_read_input_token_cost": 6e-07, - "cache_read_input_token_cost_priority": 6e-07, - "deprecation_date": "2026-08-27", - "input_cost_per_token": 1.2e-06, - "input_cost_per_token_priority": 1.2e-06, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_priority": 1.2e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false - }, "fireworks_ai/glm-4p7": { "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 6e-07, @@ -25998,26 +24397,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "fireworks_ai/minimax-m2p7": { - "cache_read_input_token_cost": 6e-08, - "cache_read_input_token_cost_priority": 6e-07, - "deprecation_date": "2026-08-27", - "input_cost_per_token": 3e-07, - "input_cost_per_token_priority": 1.2e-06, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 196608, - "max_output_tokens": 196608, - "max_tokens": 196608, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_priority": 1.2e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false - }, "fireworks_ai/minimax-m3": { "cache_read_input_token_cost": 6e-08, "cache_read_input_token_cost_priority": 9e-08, @@ -26195,31 +24574,6 @@ "comment": "Open flagship GLM for long-horizon coding agents and million-token context work", "source": "https://api.friendli.ai/serverless/v1/models" }, - "friendliai/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B": { - "litellm_provider": "friendliai", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "input_cost_per_token": 6e-07, - "output_cost_per_token": 2.4e-06, - "cache_read_input_token_cost": 1.2e-07, - "supports_prompt_caching": true, - "supports_reasoning": true, - "reasoning_effort_levels": [], - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_native_structured_output": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_image_input": false, - "supports_video_input": false, - "mode": "chat", - "comment": "Frontier-scale multilingual language model developed by LG AI Research", - "deprecation_date": "2026-09-06", - "source": "https://api.friendli.ai/serverless/v1/models" - }, "friendliai/deepseek-ai/DeepSeek-V3.2": { "litellm_provider": "friendliai", "max_input_tokens": 163840, @@ -26525,160 +24879,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "gemini-2.0-flash": { - "cache_read_input_token_cost": 2.5e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 1e-06, - "input_cost_per_audio_token_batches": 5e-07, - "input_cost_per_character": 3.75e-08, - "input_cost_per_token": 1.5e-07, - "input_cost_per_token_batches": 7.5e-08, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-07, - "output_cost_per_token_batches": 3e-07, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini-2.0-flash-001": { - "cache_read_input_token_cost": 3.75e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 1e-06, - "input_cost_per_token": 1.5e-07, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini-2.0-flash-lite": { - "cache_read_input_token_cost": 1.875e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7.5e-08, - "input_cost_per_audio_token_batches": 3.75e-08, - "input_cost_per_character": 1.875e-08, - "input_cost_per_token": 7.5e-08, - "input_cost_per_token_batches": 3.75e-08, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "output_cost_per_token_batches": 1.5e-07, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini-2.0-flash-lite-001": { - "cache_read_input_token_cost": 1.875e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7.5e-08, - "input_cost_per_token": 7.5e-08, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, "gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, "deprecation_date": "2026-10-20", @@ -27503,53 +25703,6 @@ }, "gemini_native_audio": true }, - "gemini-2.5-flash-lite-preview-06-17": { - "deprecation_date": "2025-11-18", - "cache_read_input_token_cost": 1e-08, - "input_cost_per_audio_token": 5e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_reasoning_token": 4e-07, - "output_cost_per_token": 4e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "google_maps_grounding_cost_per_query": 0.025, - "supports_image_size": false - }, "gemini-2.5-pro": { "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 1.25e-07, @@ -27608,62 +25761,6 @@ "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 1.8e-05 }, - "gemini-3-pro-preview": { - "deprecation_date": "2026-03-26", - "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, - "input_cost_per_token": 2e-06, - "input_cost_per_token_above_200k_tokens": 4e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "output_cost_per_token_above_200k_tokens": 1.8e-05, - "output_cost_per_token_batches": 6e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_video_input": true, - "supports_vision": true, - "supports_web_search": true, - "supports_native_streaming": true, - "input_cost_per_token_priority": 3.6e-06, - "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, - "output_cost_per_token_priority": 2.16e-05, - "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, - "cache_read_input_token_cost_priority": 3.6e-07, - "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query" - }, "gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, @@ -28322,51 +26419,6 @@ "supports_url_context": true, "supports_vision": true }, - "gemini/gemini-robotics-er-1.5-preview": { - "cache_read_input_token_cost": 0, - "deprecation_date": "2026-04-30", - "input_cost_per_token": 3e-07, - "input_cost_per_audio_token": 1e-06, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "output_cost_per_reasoning_token": 2.5e-06, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-robotics-er-1-5-preview", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions" - ], - "supported_modalities": [ - "text", - "image", - "video", - "audio" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": false, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000, - "rpm": 10, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, "gemini/gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_batches": 5e-08, @@ -28417,53 +26469,6 @@ "supports_web_search": true, "web_search_billing_unit": "per_query" }, - "gemini/gemini-robotics-er-1.6-preview": { - "deprecation_date": "2026-08-31", - "input_cost_per_audio_token": 2e-06, - "input_cost_per_token": 1e-06, - "litellm_provider": "gemini", - "max_input_tokens": 131072, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_reasoning_token": 5e-06, - "output_cost_per_token": 5e-06, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-robotics-er", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_video_input": true, - "supports_vision": true, - "supports_web_search": true, - "web_search_billing_unit": "per_query" - }, "gemini-2.5-computer-use-preview-10-2025": { "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, @@ -28600,27 +26605,6 @@ "source": "https://ai.google.dev/gemini-api/docs/embeddings#model-versions", "tpm": 10000000 }, - "gemini/gemini-embedding-2-preview": { - "deprecation_date": "2026-08-10", - "input_cost_per_audio_token": 6.5e-06, - "input_cost_per_audio_token_batches": 3.25e-06, - "input_cost_per_image_token": 4.5e-07, - "input_cost_per_image_token_batches": 2.25e-07, - "input_cost_per_token": 2e-07, - "input_cost_per_token_batches": 1e-07, - "input_cost_per_video_token": 1.2e-05, - "input_cost_per_video_token_batches": 6e-06, - "litellm_provider": "gemini", - "max_input_tokens": 8192, - "max_tokens": 8192, - "mode": "embedding", - "output_cost_per_token": 0, - "output_vector_size": 3072, - "rpm": 10000, - "source": "https://ai.google.dev/gemini-api/docs/pricing", - "supports_multimodal": true, - "tpm": 10000000 - }, "gemini/gemini-embedding-2": { "input_cost_per_audio_token": 6.5e-06, "input_cost_per_audio_token_batches": 3.25e-06, @@ -28643,135 +26627,6 @@ "supports_vision": true, "tpm": 10000000 }, - "gemini/gemini-1.5-flash": { - "deprecation_date": "2025-09-29", - "input_cost_per_token": 7.5e-08, - "input_cost_per_token_above_128k_tokens": 1.5e-07, - "litellm_provider": "gemini", - "max_input_tokens": 8192, - "max_tokens": 8192, - "mode": "embedding", - "output_cost_per_token": 0, - "output_vector_size": 3072, - "rpm": 10000, - "source": "https://ai.google.dev/gemini-api/docs/embeddings#multimodal", - "supports_multimodal": true, - "tpm": 10000000 - }, - "gemini/gemini-2.0-flash": { - "cache_read_input_token_cost": 2.5e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4e-07, - "rpm": 10000, - "source": "https://ai.google.dev/pricing#2_0flash", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 10000000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini/gemini-2.0-flash-001": { - "cache_read_input_token_cost": 2.5e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4e-07, - "rpm": 10000, - "source": "https://ai.google.dev/pricing#2_0flash", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 10000000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini/gemini-2.0-flash-lite": { - "cache_read_input_token_cost": 1.875e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7.5e-08, - "input_cost_per_token": 7.5e-08, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "rpm": 4000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.0-flash-lite", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 4000000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, "gemini/gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, "cache_read_input_token_cost": 3e-08, @@ -28934,50 +26789,6 @@ "web_search_billing_unit": "per_query", "supports_reasoning": false }, - "gemini/gemini-3-pro-image-preview": { - "deprecation_date": "2026-06-25", - "input_cost_per_image": 0.0011, - "input_cost_per_token": 2e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "gemini", - "max_input_tokens": 65536, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "image_generation", - "output_cost_per_image": 0.134, - "output_cost_per_image_token": 0.00012, - "output_cost_per_token": 1.2e-05, - "rpm": 1000, - "tpm": 4000000, - "output_cost_per_token_batches": 6e-06, - "source": "https://ai.google.dev/gemini-api/docs/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_prompt_caching": true, - "supports_reasoning": false, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query" - }, "gemini/nano-banana-pro-preview": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -29063,49 +26874,6 @@ }, "web_search_billing_unit": "per_query" }, - "gemini/gemini-3.1-flash-image-preview": { - "deprecation_date": "2026-06-25", - "input_cost_per_token": 5e-07, - "input_cost_per_token_batches": 2.5e-07, - "litellm_provider": "gemini", - "max_input_tokens": 65536, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "image_generation", - "output_cost_per_image": 0.045, - "output_cost_per_image_token": 6e-05, - "output_cost_per_token": 3e-06, - "output_cost_per_token_batches": 1.5e-06, - "rpm": 1000, - "tpm": 4000000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-image-preview", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_prompt_caching": true, - "supports_reasoning": false, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query" - }, "gemini/gemini-3.1-flash-lite-image": { "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -29244,104 +27012,6 @@ "supports_audio_input": true, "supports_image_size": false }, - "gemini/gemini-2.5-flash-lite-preview-09-2025": { - "cache_read_input_token_cost": 1e-08, - "deprecation_date": "2026-03-31", - "input_cost_per_audio_token": 3e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_reasoning_token": 4e-07, - "output_cost_per_token": 4e-07, - "rpm": 15, - "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "google_maps_grounding_cost_per_query": 0.025, - "supports_image_size": false - }, - "gemini/gemini-2.5-flash-preview-09-2025": { - "cache_read_input_token_cost": 3e-08, - "deprecation_date": "2026-02-17", - "input_cost_per_audio_token": 1e-06, - "input_cost_per_token": 3e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_reasoning_token": 2.5e-06, - "output_cost_per_token": 2.5e-06, - "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "google_maps_grounding_cost_per_query": 0.025, - "supports_image_size": false - }, "gemini/gemini-flash-latest": { "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, @@ -29459,55 +27129,6 @@ "supports_video_input": true, "web_search_billing_unit": "per_query" }, - "gemini/gemini-2.5-flash-lite-preview-06-17": { - "deprecation_date": "2025-11-18", - "cache_read_input_token_cost": 1e-08, - "input_cost_per_audio_token": 5e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_reasoning_token": 4e-07, - "output_cost_per_token": 4e-07, - "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "google_maps_grounding_cost_per_query": 0.025, - "supports_image_size": false - }, "gemini/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, @@ -29618,114 +27239,6 @@ "supports_vision": true, "tpm": 800000 }, - "gemini/gemini-3-pro-preview": { - "deprecation_date": "2026-03-09", - "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "input_cost_per_token": 2e-06, - "input_cost_per_token_above_200k_tokens": 4e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "output_cost_per_token_above_200k_tokens": 1.8e-05, - "output_cost_per_token_batches": 6e-06, - "rpm": 2000, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_video_input": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 800000, - "input_cost_per_token_priority": 3.6e-06, - "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, - "output_cost_per_token_priority": 2.16e-05, - "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, - "cache_read_input_token_cost_priority": 3.6e-07, - "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query" - }, - "gemini/gemini-3.1-flash-lite-preview": { - "cache_read_input_token_cost": 2.5e-08, - "deprecation_date": "2026-05-25", - "input_cost_per_audio_token": 5e-07, - "input_cost_per_token": 2.5e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_reasoning_token": 1.5e-06, - "output_cost_per_token": 1.5e-06, - "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/models", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_video_input": true, - "supports_vision": true, - "supports_web_search": true, - "supports_native_streaming": true, - "tpm": 250000, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 - }, "gemini/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, "cache_read_input_token_cost": 2.5e-08, @@ -30826,34 +28339,6 @@ "output_cost_per_image": 0.04, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, - "gemini/imagen-3.0-generate-002": { - "deprecation_date": "2025-11-10", - "litellm_provider": "gemini", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "gemini/imagen-4.0-fast-generate-001": { - "deprecation_date": "2026-08-17", - "litellm_provider": "gemini", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "gemini/imagen-4.0-generate-001": { - "deprecation_date": "2026-08-17", - "litellm_provider": "gemini", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "gemini/imagen-4.0-ultra-generate-001": { - "deprecation_date": "2026-08-17", - "litellm_provider": "gemini", - "mode": "image_generation", - "output_cost_per_image": 0.06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, "gemini/learnlm-1.5-pro-experimental": { "input_cost_per_audio_per_second": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -30932,21 +28417,6 @@ "supports_web_search": false, "output_cost_per_image": 0.08 }, - "gemini/veo-2.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "gemini", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "video_generation", - "output_cost_per_second": 0.35, - "source": "https://ai.google.dev/gemini-api/docs/video", - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "video" - ] - }, "gemini/veo-3.1-fast-generate-preview": { "litellm_provider": "gemini", "max_input_tokens": 1024, @@ -32156,33 +29626,6 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "gpt-4-0125-preview": { - "deprecation_date": "2026-03-26", - "input_cost_per_token": 1e-05, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 3e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, - "gpt-4-0314": { - "deprecation_date": "2026-03-26", - "input_cost_per_token": 3e-05, - "litellm_provider": "openai", - "max_input_tokens": 8192, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-05, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4-0613": { "deprecation_date": "2026-10-23", "input_cost_per_token": 3e-05, @@ -32252,22 +29695,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "gpt-4-turbo-preview": { - "deprecation_date": "2026-03-26", - "input_cost_per_token": 1e-05, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 3e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4.1": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_priority": 8.75e-07, @@ -32610,24 +30037,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "gpt-4o-audio-preview": { - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 4e-05, - "input_cost_per_token": 2.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_audio_token": 8e-05, - "output_cost_per_token": 1e-05, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4o-audio-preview-2024-12-17": { "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 4e-05, @@ -32812,44 +30221,6 @@ "supports_tool_choice": true, "supports_vision": false }, - "gpt-audio-mini-2025-10-06": { - "deprecation_date": "2026-07-23", - "input_cost_per_audio_token": 1e-05, - "input_cost_per_token": 6e-07, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 2.4e-06, - "source": "https://developers.openai.com/api/docs/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses", - "/v1/realtime", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "audio" - ], - "supported_output_modalities": [ - "text", - "audio" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": false, - "supports_reasoning": false, - "supports_response_schema": false, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": false - }, "gpt-audio-mini-2025-12-15": { "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, @@ -32946,24 +30317,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "gpt-4o-mini-audio-preview": { - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 1e-05, - "input_cost_per_token": 1.5e-07, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 6e-07, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4o-mini-audio-preview-2024-12-17": { "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 1e-05, @@ -32982,26 +30335,6 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "gpt-4o-mini-realtime-preview": { - "cache_creation_input_audio_token_cost": 3e-07, - "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 1e-05, - "input_cost_per_token": 6e-07, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 2.4e-06, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4o-mini-realtime-preview-2024-12-17": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 3e-07, @@ -33048,32 +30381,6 @@ "supports_vision": true, "supports_web_search": true }, - "gpt-4o-mini-search-preview-2025-03-11": { - "cache_read_input_token_cost": 7.5e-08, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.5e-07, - "input_cost_per_token_batches": 7.5e-08, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 6e-07, - "output_cost_per_token_batches": 3e-07, - "search_context_cost_per_query": { - "search_context_size_high": 0.025, - "search_context_size_low": 0.025, - "search_context_size_medium": 0.025 - }, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "gpt-4o-mini-transcribe": { "input_cost_per_audio_token": 1.25e-06, "input_cost_per_token": 1.25e-06, @@ -33108,63 +30415,6 @@ "audio" ] }, - "gpt-4o-realtime-preview": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 4e-05, - "input_cost_per_token": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 8e-05, - "output_cost_per_token": 2e-05, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, - "gpt-4o-realtime-preview-2024-12-17": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 4e-05, - "input_cost_per_token": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 8e-05, - "output_cost_per_token": 2e-05, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, - "gpt-4o-realtime-preview-2025-06-03": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 4e-05, - "input_cost_per_token": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 8e-05, - "output_cost_per_token": 2e-05, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4o-search-preview": { "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, @@ -33191,32 +30441,6 @@ "supports_vision": true, "supports_web_search": true }, - "gpt-4o-search-preview-2025-03-11": { - "cache_read_input_token_cost": 1.25e-06, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 2.5e-06, - "input_cost_per_token_batches": 1.25e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "output_cost_per_token_batches": 5e-06, - "search_context_cost_per_query": { - "search_context_size_high": 0.025, - "search_context_size_low": 0.025, - "search_context_size_medium": 0.025 - }, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "gpt-4o-transcribe": { "input_cost_per_audio_token": 2.5e-06, "input_cost_per_token": 2.5e-06, @@ -33879,52 +31103,6 @@ "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": false }, - "gpt-5.1-chat-latest": { - "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_priority": 2.5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "input_cost_per_token_priority": 2.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "output_cost_per_token_priority": 2e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_native_streaming": true, - "supports_parallel_function_calling": false, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": false, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, "gpt-5.2": { "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_batches": 8.75e-08, @@ -34031,94 +31209,6 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, - "gpt-5.2-chat-latest": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-08-10", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.3-chat-latest": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-08-10", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, "gpt-5.2-pro": { "input_cost_per_token": 2.1e-05, "input_cost_per_token_batches": 1.05e-05, @@ -35783,251 +32873,6 @@ "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": true }, - "gpt-5-chat-latest": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": false, - "supports_native_streaming": true, - "supports_parallel_function_calling": false, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": false, - "supports_vision": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5-codex": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 1e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.1-codex": { - "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_priority": 2.5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "input_cost_per_token_priority": 2.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 1e-05, - "output_cost_per_token_priority": 2e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.1-codex-max": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 1e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.1-codex-mini": { - "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_priority": 4.5e-08, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 2.5e-07, - "input_cost_per_token_priority": 4.5e-07, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 2e-06, - "output_cost_per_token_priority": 3.6e-06, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.2-codex": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true - }, "gpt-5.3-codex": { "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, @@ -36870,32 +33715,6 @@ "supports_response_schema": true, "supports_vision": true }, - "groq/llama-3.1-8b-instant": { - "deprecation_date": "2026-08-16", - "input_cost_per_token": 5e-08, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 8e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.3-70b-versatile": { - "deprecation_date": "2026-08-16", - "input_cost_per_token": 5.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 7.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, "groq/llama-guard-3-8b": { "input_cost_per_token": 2e-07, "litellm_provider": "groq", @@ -36905,19 +33724,6 @@ "output_cost_per_token": 2e-07, "source": "https://console.groq.com/docs/model/llama-guard-3-8b" }, - "groq/gemma-7b-it": { - "deprecation_date": "2024-12-18", - "input_cost_per_token": 5e-08, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 8e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, "groq/meta-llama/llama-prompt-guard-2-22m": { "input_cost_per_token": 3e-08, "litellm_provider": "groq", @@ -36938,58 +33744,6 @@ "output_cost_per_token": 4e-08, "source": "https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m" }, - "groq/meta-llama/llama-guard-4-12b": { - "deprecation_date": "2026-03-05", - "input_cost_per_token": 2e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-07 - }, - "groq/meta-llama/llama-4-maverick-17b-128e-instruct": { - "deprecation_date": "2026-03-09", - "input_cost_per_token": 2e-07, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/meta-llama/llama-4-scout-17b-16e-instruct": { - "deprecation_date": "2026-07-17", - "input_cost_per_token": 1.1e-07, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3.4e-07, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/moonshotai/kimi-k2-instruct-0905": { - "deprecation_date": "2026-04-15", - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3e-06, - "cache_read_input_token_cost": 5e-07, - "litellm_provider": "groq", - "max_input_tokens": 262144, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "groq/openai/gpt-oss-120b": { "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.5e-07, @@ -37070,45 +33824,6 @@ "mode": "audio_speech", "source": "https://console.groq.com/docs/models" }, - "groq/playai-tts": { - "deprecation_date": "2025-12-31", - "input_cost_per_character": 5e-05, - "litellm_provider": "groq", - "max_input_tokens": 10000, - "max_output_tokens": 10000, - "max_tokens": 10000, - "mode": "audio_speech" - }, - "groq/qwen/qwen3.6-27b": { - "input_cost_per_token": 6e-07, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://console.groq.com/docs/model/qwen/qwen3.6-27b", - "deprecation_date": "2026-09-14", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/qwen/qwen3-32b": { - "deprecation_date": "2026-07-17", - "input_cost_per_token": 2.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 131000, - "max_output_tokens": 131000, - "max_tokens": 131000, - "mode": "chat", - "output_cost_per_token": 5.9e-07, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, "groq/whisper-large-v3": { "input_cost_per_second": 3.083e-05, "litellm_provider": "groq", @@ -37121,27 +33836,6 @@ "mode": "audio_transcription", "output_cost_per_second": 0.0 }, - "hd/1024-x-1024/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 7.629e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, - "hd/1024-x-1792/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 6.539e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, - "hd/1792-x-1024/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 6.539e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "heroku/claude-3-5-haiku": { "litellm_provider": "heroku", "max_tokens": 8192, @@ -38968,19 +35662,6 @@ "supports_system_messages": true, "supports_native_structured_output": true }, - "mistral/codestral-2405": { - "deprecation_date": "2025-06-16", - "input_cost_per_token": 1e-06, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 3e-06, - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/codestral-2508": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, @@ -39024,51 +35705,6 @@ "supports_assistant_prefill": true, "supports_tool_choice": true }, - "mistral/devstral-medium-2507": { - "deprecation_date": "2026-05-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://mistral.ai/news/devstral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/devstral-small-2505": { - "deprecation_date": "2025-11-30", - "input_cost_per_token": 1e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://mistral.ai/news/devstral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/devstral-small-2507": { - "deprecation_date": "2026-05-31", - "input_cost_per_token": 1e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://mistral.ai/news/devstral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/devstral-small-latest": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, @@ -39084,21 +35720,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/labs-devstral-small-2512": { - "deprecation_date": "2026-03-31", - "input_cost_per_token": 1e-07, - "litellm_provider": "mistral", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://docs.mistral.ai/models/devstral-small-2-25-12", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/devstral-latest": { "cache_read_input_token_cost": 4e-08, "input_cost_per_token": 4e-07, @@ -39129,21 +35750,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/devstral-2512": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://mistral.ai/news/devstral-2-vibe-cli", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/ministral-14b-2512": { "input_cost_per_token": 2e-07, "litellm_provider": "mistral", @@ -39419,54 +36025,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/magistral-medium-2506": { - "deprecation_date": "2025-11-30", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/magistral-medium-2509": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/magistral-medium-1-2-2509": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/mistral-ocr-latest": { "litellm_provider": "mistral", "ocr_cost_per_page": 0.004, @@ -39506,20 +36064,6 @@ "/v1/batch" ] }, - "mistral/mistral-ocr-2505-completion": { - "deprecation_date": "2026-05-31", - "litellm_provider": "mistral", - "ocr_cost_per_page": 0.001, - "ocr_cost_per_page_batches": 0.0005, - "annotation_cost_per_page": 0.003, - "annotation_cost_per_page_batches": 0.0015, - "mode": "ocr", - "supported_endpoints": [ - "/v1/ocr", - "/v1/batch" - ], - "source": "https://mistral.ai/pricing#api-pricing" - }, "mistral/mistral-ocr-2512": { "litellm_provider": "mistral", "ocr_cost_per_page": 0.002, @@ -39550,22 +36094,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/magistral-small-2506": { - "deprecation_date": "2025-11-30", - "input_cost_per_token": 5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://mistral.ai/pricing#api-pricing", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/magistral-small-latest": { "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, @@ -39583,22 +36111,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/magistral-small-1-2-2509": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://mistral.ai/pricing#api-pricing", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/mistral-embed": { "input_cost_per_token": 1e-07, "litellm_provider": "mistral", @@ -39622,48 +36134,6 @@ "max_tokens": 8192, "mode": "embedding" }, - "mistral/mistral-large-2402": { - "deprecation_date": "2025-06-16", - "input_cost_per_token": 4e-06, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/mistral-large-2407": { - "deprecation_date": "2025-03-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 9e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/mistral-large-2411": { - "deprecation_date": "2026-05-31", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 6e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/mistral-large-latest": { "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 5e-07, @@ -39733,49 +36203,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/mistral-medium-2312": { - "deprecation_date": "2025-06-16", - "input_cost_per_token": 2.7e-06, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 8.1e-06, - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/mistral-medium-2505": { - "deprecation_date": "2026-08-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 2e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/mistral-medium-2508": { - "deprecation_date": "2026-08-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://mistral.ai/news/mistral-medium-3", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "mistral/mistral-medium-2604": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, @@ -39818,22 +36245,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/mistral-medium-3-1-2508": { - "deprecation_date": "2026-08-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://mistral.ai/news/mistral-medium-3", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "mistral/mistral-medium-3-5": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, @@ -39890,22 +36301,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/mistral-small-3-2-2506": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 6e-08, - "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "source": "https://mistral.ai/pricing", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "mistral/ministral-3-3b-2512": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, @@ -39999,32 +36394,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/open-codestral-mamba": { - "deprecation_date": "2025-06-06", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2.5e-07, - "source": "https://mistral.ai/technology/", - "supports_assistant_prefill": true, - "supports_tool_choice": true - }, - "mistral/open-mistral-7b": { - "deprecation_date": "2025-03-30", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 2.5e-07, - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/open-mistral-nemo": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, @@ -40039,78 +36408,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/open-mistral-nemo-2407": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 3e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://mistral.ai/technology/", - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/open-mixtral-8x22b": { - "deprecation_date": "2025-03-30", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 65336, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 6e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/open-mixtral-8x7b": { - "deprecation_date": "2025-03-30", - "input_cost_per_token": 7e-07, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 7e-07, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/pixtral-12b-2409": { - "deprecation_date": "2025-12-31", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.5e-07, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "mistral/pixtral-large-2411": { - "deprecation_date": "2026-05-31", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 6e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "mistral/pixtral-large-latest": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 2e-06, @@ -40159,36 +36456,6 @@ "supports_audio_input": false, "supports_response_schema": true }, - "moonshot/kimi-k2-0711-preview": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 6e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "moonshot/kimi-k2-0905-preview": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 6e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, "moonshot/kimi-k2.7-code": { "cache_read_input_token_cost": 1.9e-07, "input_cost_per_token": 9.5e-07, @@ -40207,21 +36474,6 @@ "supports_video_input": true, "supports_vision": true }, - "moonshot/kimi-k2-turbo-preview": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 1.15e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 8e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, "moonshot/kimi-k2.5": { "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 6e-07, @@ -40278,111 +36530,6 @@ "supports_video_input": true, "supports_vision": true }, - "moonshot/kimi-latest": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-01-28", - "input_cost_per_token": 2e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "moonshot/kimi-latest-128k": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-01-28", - "input_cost_per_token": 2e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "moonshot/kimi-latest-32k": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-01-28", - "input_cost_per_token": 1e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "moonshot/kimi-latest-8k": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-01-28", - "input_cost_per_token": 2e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "moonshot/kimi-thinking-preview": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2025-11-11", - "input_cost_per_token": 6e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_vision": true - }, - "moonshot/kimi-k2-thinking": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 6e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "moonshot/kimi-k2-thinking-turbo": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 1.15e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 8e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_web_search": true - }, "moonshot/moonshot-v1-128k": { "input_cost_per_token": 2e-06, "litellm_provider": "moonshot", @@ -40396,19 +36543,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "moonshot/moonshot-v1-128k-0430": { - "deprecation_date": "2024-04-30", - "input_cost_per_token": 2e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true - }, "moonshot/moonshot-v1-128k-vision-preview": { "input_cost_per_token": 2e-06, "litellm_provider": "moonshot", @@ -40436,19 +36570,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "moonshot/moonshot-v1-32k-0430": { - "deprecation_date": "2024-04-30", - "input_cost_per_token": 1e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true - }, "moonshot/moonshot-v1-32k-vision-preview": { "input_cost_per_token": 1e-06, "litellm_provider": "moonshot", @@ -40476,19 +36597,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "moonshot/moonshot-v1-8k-0430": { - "deprecation_date": "2024-04-30", - "input_cost_per_token": 2e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true - }, "moonshot/moonshot-v1-8k-vision-preview": { "input_cost_per_token": 2e-07, "litellm_provider": "moonshot", @@ -41679,88 +37787,6 @@ "supports_vision": true, "supports_web_search": true }, - "o3-deep-research": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1e-05, - "input_cost_per_token_batches": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 4e-05, - "output_cost_per_token_batches": 2e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, - "o3-deep-research-2025-06-26": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1e-05, - "input_cost_per_token_batches": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 4e-05, - "output_cost_per_token_batches": 2e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, "o3-mini": { "cache_read_input_token_cost": 5.5e-07, "deprecation_date": "2026-10-23", @@ -41946,88 +37972,6 @@ "supports_vision": true, "supports_web_search": true }, - "o4-mini-deep-research": { - "cache_read_input_token_cost": 5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 2e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 8e-06, - "output_cost_per_token_batches": 4e-06, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, - "o4-mini-deep-research-2025-06-26": { - "cache_read_input_token_cost": 5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 2e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 8e-06, - "output_cost_per_token_batches": 4e-06, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, "oci/meta.llama-3.1-8b-instruct": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", @@ -43509,23 +39453,6 @@ "supports_vision": false, "supports_web_search": false }, - "openrouter/google/gemini-2.0-flash-001": { - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4e-07, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/google/gemini-2.5-flash": { "cache_creation_input_token_cost": 8.33333333333333e-08, "cache_read_input_audio_token_cost": 1e-07, @@ -46699,17 +42626,6 @@ "supports_reasoning": true, "supports_system_messages": true }, - "rerank-english-v2.0": { - "input_cost_per_query": 0.002, - "input_cost_per_token": 0.0, - "litellm_provider": "cohere", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "rerank", - "output_cost_per_token": 0.0, - "deprecation_date": "2025-04-30" - }, "rerank-english-v3.0": { "input_cost_per_query": 0.002, "input_cost_per_token": 0.0, @@ -46720,17 +42636,6 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, - "rerank-multilingual-v2.0": { - "input_cost_per_query": 0.002, - "input_cost_per_token": 0.0, - "litellm_provider": "cohere", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "rerank", - "output_cost_per_token": 0.0, - "deprecation_date": "2025-04-30" - }, "rerank-multilingual-v3.0": { "input_cost_per_query": 0.002, "input_cost_per_token": 0.0, @@ -46871,31 +42776,6 @@ "output_cost_per_token": 7e-06, "source": "https://cloud.sambanova.ai/plans/pricing" }, - "sambanova/DeepSeek-R1-Distill-Llama-70B": { - "deprecation_date": "2026-03-20", - "input_cost_per_token": 7e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.4e-06, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, - "sambanova/DeepSeek-V3-0324": { - "deprecation_date": "2026-04-14", - "input_cost_per_token": 3e-06, - "litellm_provider": "sambanova", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 4.5e-06, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "sambanova/Llama-4-Maverick-17B-128E-Instruct": { "input_cost_per_token": 6.3e-07, "litellm_provider": "sambanova", @@ -46913,73 +42793,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "sambanova/Llama-4-Scout-17B-16E-Instruct": { - "deprecation_date": "2025-06-19", - "input_cost_per_token": 4e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "metadata": { - "notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount" - }, - "mode": "chat", - "output_cost_per_token": 7e-07, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "sambanova/Meta-Llama-3.1-405B-Instruct": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 5e-06, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "sambanova/Meta-Llama-3.1-8B-Instruct": { - "deprecation_date": "2026-04-14", - "input_cost_per_token": 1e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "sambanova/Meta-Llama-3.2-1B-Instruct": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 4e-08, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 8e-08, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, - "sambanova/Meta-Llama-3.2-3B-Instruct": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 8e-08, - "litellm_provider": "sambanova", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.6e-07, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, "sambanova/Meta-Llama-3.3-70B-Instruct": { "input_cost_per_token": 6e-07, "litellm_provider": "sambanova", @@ -46993,54 +42806,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "sambanova/Meta-Llama-Guard-3-8B": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 3e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, - "sambanova/QwQ-32B": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 5e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-06, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, - "sambanova/Qwen2-Audio-7B-Instruct": { - "deprecation_date": "2025-06-19", - "input_cost_per_token": 5e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 0.0001, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_audio_input": true - }, - "sambanova/Qwen3-32B": { - "deprecation_date": "2026-04-06", - "input_cost_per_token": 4e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "sambanova/DeepSeek-V3.1": { "max_tokens": 131072, "max_input_tokens": 131072, @@ -47633,27 +43398,6 @@ "mode": "image_generation", "output_cost_per_image": 0.14 }, - "standard/1024-x-1024/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 3.81469e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, - "standard/1024-x-1792/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 4.359e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, - "standard/1792-x-1024/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 4.359e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "linkup/search": { "input_cost_per_query": 0.00587, "litellm_provider": "linkup", @@ -47788,36 +43532,6 @@ "output_vector_size": 768, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, - "text-moderation-007": { - "deprecation_date": "2025-10-27", - "input_cost_per_token": 0.0, - "litellm_provider": "openai", - "max_input_tokens": 32768, - "max_output_tokens": 0, - "max_tokens": 0, - "mode": "moderation", - "output_cost_per_token": 0.0 - }, - "text-moderation-latest": { - "deprecation_date": "2025-10-27", - "input_cost_per_token": 0.0, - "litellm_provider": "openai", - "max_input_tokens": 32768, - "max_output_tokens": 0, - "max_tokens": 0, - "mode": "moderation", - "output_cost_per_token": 0.0 - }, - "text-moderation-stable": { - "deprecation_date": "2025-10-27", - "input_cost_per_token": 0.0, - "litellm_provider": "openai", - "max_input_tokens": 32768, - "max_output_tokens": 0, - "max_tokens": 0, - "mode": "moderation", - "output_cost_per_token": 0.0 - }, "text-multilingual-embedding-002": { "deprecation_date": "2027-04-01", "input_cost_per_character": 2.5e-08, @@ -47915,19 +43629,6 @@ "mode": "chat", "output_cost_per_token": 1e-07 }, - "together_ai/Qwen/Qwen2.5-72B-Instruct-Turbo": { - "deprecation_date": "2026-02-06", - "litellm_provider": "together_ai", - "mode": "chat", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "input_cost_per_token": 1.2e-06, - "output_cost_per_token": 1.2e-06, - "max_input_tokens": 131072, - "source": "https://api.together.ai/v1/models" - }, "together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo": { "litellm_provider": "together_ai", "mode": "chat", @@ -47940,87 +43641,6 @@ "max_input_tokens": 32768, "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { - "deprecation_date": "2026-07-10", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262000, - "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { - "deprecation_date": "2026-04-16", - "input_cost_per_token": 6.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": { - "deprecation_date": "2026-02-06", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://www.together.ai/models/qwen3-235b-a22b-fp8-tput", - "supports_function_calling": false, - "supports_parallel_function_calling": false, - "supports_tool_choice": false - }, - "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { - "deprecation_date": "2026-06-04", - "input_cost_per_token": 2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/deepseek-ai/DeepSeek-R1": { - "deprecation_date": "2026-05-14", - "input_cost_per_token": 3e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 128000, - "max_output_tokens": 20480, - "max_tokens": 20480, - "metadata": { - "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" - }, - "mode": "chat", - "output_cost_per_token": 7e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/deepseek-ai/DeepSeek-R1-0528-tput": { - "deprecation_date": "2026-02-03", - "input_cost_per_token": 5.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.19e-06, - "source": "https://www.together.ai/models/deepseek-r1-0528-throughput", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/deepseek-ai/DeepSeek-V3": { "input_cost_per_token": 1.25e-06, "litellm_provider": "together_ai", @@ -48037,33 +43657,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/deepseek-ai/DeepSeek-V3.1": { - "deprecation_date": "2026-05-14", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "max_tokens": 16384, - "metadata": { - "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" - }, - "mode": "chat", - "output_cost_per_token": 1.7e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "max_input_tokens": 131072, - "max_output_tokens": 16384 - }, - "together_ai/meta-llama/Llama-3.2-3B-Instruct-Turbo": { - "deprecation_date": "2026-03-06", - "litellm_provider": "together_ai", - "mode": "chat", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "input_cost_per_token": 1.04e-06, "litellm_provider": "together_ai", @@ -48077,112 +43670,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo-Free": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 0, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 0, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { - "deprecation_date": "2026-03-31", - "input_cost_per_token": 2.7e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8.5e-07, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Llama-4-Scout-17B-16E-Instruct": { - "deprecation_date": "2026-02-06", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 5.9e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo": { - "deprecation_date": "2026-02-06", - "input_cost_per_token": 3.5e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 3.5e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { - "deprecation_date": "2026-02-25", - "input_cost_per_token": 8.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8.8e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { - "deprecation_date": "2026-03-06", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/mistralai/Mistral-7B-Instruct-v0.1": { - "deprecation_date": "2025-11-13", - "litellm_provider": "together_ai", - "mode": "chat", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "input_cost_per_token": 2e-07, - "output_cost_per_token": 2e-07, - "max_input_tokens": 32768, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": { - "deprecation_date": "2026-04-02", - "litellm_provider": "together_ai", - "mode": "chat", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 3e-07, - "max_input_tokens": 32768, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1": { - "deprecation_date": "2026-04-16", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/moonshotai/Kimi-K2-Instruct": { "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", @@ -48211,19 +43698,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/openai/gpt-oss-20b": { - "deprecation_date": "2026-09-14", - "input_cost_per_token": 5e-08, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/togethercomputer/CodeLlama-34b-Instruct": { "litellm_provider": "together_ai", "mode": "chat", @@ -48231,19 +43705,6 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true }, - "together_ai/zai-org/GLM-4.5-Air-FP8": { - "deprecation_date": "2026-04-02", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.1e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/zai-org/GLM-4.6": { "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", @@ -48260,102 +43721,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "together_ai/zai-org/GLM-4.7": { - "deprecation_date": "2026-04-02", - "input_cost_per_token": 4.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 202752, - "max_tokens": 202752, - "metadata": { - "successor": "together_ai/zai-org/GLM-5.2" - }, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, - "together_ai/moonshotai/Kimi-K2.5": { - "deprecation_date": "2026-05-21", - "input_cost_per_token": 5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 256000, - "max_tokens": 256000, - "metadata": { - "successor": "together_ai/moonshotai/Kimi-K3" - }, - "mode": "chat", - "output_cost_per_token": 2.8e-06, - "source": "https://www.together.ai/models/kimi-k2-5", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_reasoning": true - }, - "together_ai/moonshotai/Kimi-K2-Instruct-0905": { - "deprecation_date": "2026-03-06", - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "metadata": { - "successor": "together_ai/moonshotai/Kimi-K3" - }, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://www.together.ai/models/kimi-k2-0905", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3-Next-80B-A3B-Instruct": { - "deprecation_date": "2026-04-02", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "metadata": { - "successor": "together_ai/Qwen/Qwen3.7-Plus" - }, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking": { - "deprecation_date": "2026-02-25", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "metadata": { - "successor": "together_ai/Qwen/Qwen3.6-Plus" - }, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3.5-397B-A17B": { - "cache_read_input_token_cost": 3.5e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 3.6e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/MiniMaxAI/MiniMax-M3": { "cache_read_input_token_cost": 6e-08, "input_cost_per_token": 3e-07, @@ -48477,23 +43842,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/deepseek-ai/DeepSeek-V4-Pro": { - "deprecation_date": "2026-08-27", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.74e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 512000, - "max_tokens": 512000, - "mode": "chat", - "output_cost_per_token": 3.48e-06, - "source": "https://docs.together.ai/docs/serverless-models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": { "cache_read_input_token_cost": 1.3e-07, "deprecation_date": "2026-09-29", @@ -48510,52 +43858,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/google/gemma-3n-E4B-it": { - "deprecation_date": "2026-08-25", - "input_cost_per_token": 6e-08, - "litellm_provider": "together_ai", - "max_input_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 1.2e-07, - "source": "https://docs.together.ai/docs/serverless-models" - }, - "together_ai/google/gemma-4-31B-it": { - "deprecation_date": "2026-09-14", - "input_cost_per_token": 3.9e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 9.7e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "together_ai/intfloat/multilingual-e5-large-instruct": { - "deprecation_date": "2026-09-14", - "input_cost_per_token": 2e-08, - "litellm_provider": "together_ai", - "max_input_tokens": 514, - "max_tokens": 514, - "mode": "embedding", - "output_cost_per_token": 2e-08, - "output_vector_size": 1024, - "source": "https://docs.together.ai/docs/serverless-models" - }, - "together_ai/meta-llama/Llama-Guard-4-12B": { - "deprecation_date": "2026-08-25", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 1048576, - "max_tokens": 1048576, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://docs.together.ai/docs/serverless-models" - }, "together_ai/meta-models/Muse-Glimmer-30B": { "cache_read_input_token_cost": 4e-08, "input_cost_per_token": 3.5e-07, @@ -48567,23 +43869,6 @@ "source": "https://api.together.ai/v1/models", "supports_prompt_caching": true }, - "together_ai/moonshotai/Kimi-K2.7-Code": { - "deprecation_date": "2026-08-27", - "cache_read_input_token_cost": 1.9e-07, - "input_cost_per_token": 9.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 4e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "together_ai/moonshotai/Kimi-K3": { "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, @@ -48606,33 +43891,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "together_ai/nvidia/nemotron-3-ultra-550b-a55b": { - "deprecation_date": "2026-08-27", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 512288, - "max_tokens": 512288, - "mode": "chat", - "output_cost_per_token": 3.6e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/pearl-ai/gemma-4-31b-it": { - "deprecation_date": "2026-08-27", - "input_cost_per_token": 2.8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 8.6e-07, - "source": "https://docs.together.ai/docs/serverless-models" - }, "together_ai/thinkingmachines/Inkling": { "cache_read_input_token_cost": 1.7e-07, "input_cost_per_token": 1e-06, @@ -48648,18 +43906,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/thinkingmachines/Inkling-Small": { - "deprecation_date": "2026-09-14", - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 524288, - "max_tokens": 524288, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://api.together.ai/v1/models", - "supports_prompt_caching": true - }, "together_ai/zai-org/GLM-5.2": { "cache_read_input_token_cost": 2.6e-07, "input_cost_per_token": 1.4e-06, @@ -48923,23 +44169,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "us.anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 - }, "us.anthropic.claude-3-opus-20240229-v1:0": { "input_cost_per_token": 1.5e-05, "litellm_provider": "bedrock", @@ -48955,23 +44184,6 @@ "cache_read_input_token_cost": 1.5e-06, "cache_creation_input_token_cost": 1.875e-05 }, - "us.anthropic.claude-3-sonnet-20240229-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, "us.anthropic.claude-opus-4-1-20250805-v1:0": { "cache_creation_input_token_cost": 1.875e-05, "cache_read_input_token_cost": 1.5e-06, @@ -49036,23 +44248,6 @@ "input_cost_per_token_batches": 1.65e-06, "output_cost_per_token_batches": 8.25e-06 }, - "us-gov.anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 3e-07, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-08, - "cache_creation_input_token_cost": 3.75e-07 - }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, @@ -50180,34 +45375,6 @@ "output_cost_per_token": 9e-07, "supports_tool_choice": true }, - "vercel_ai_gateway/google/gemini-2.0-flash": { - "deprecation_date": "2026-06-01", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "vercel_ai_gateway", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_vision": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true - }, - "vercel_ai_gateway/google/gemini-2.0-flash-lite": { - "deprecation_date": "2026-06-01", - "input_cost_per_token": 7.5e-08, - "litellm_provider": "vercel_ai_gateway", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "supports_vision": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true - }, "vercel_ai_gateway/google/gemini-2.5-flash": { "input_cost_per_token": 3e-07, "litellm_provider": "vercel_ai_gateway", @@ -51091,28 +46258,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "vertex_ai/claude-3-7-sonnet@20250219": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-05-11", - "input_cost_per_token": 3e-06, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "vertex_ai/claude-3-haiku": { "input_cost_per_token": 2.5e-07, "litellm_provider": "vertex_ai-anthropic_models", @@ -51191,72 +46336,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "vertex_ai/claude-opus-4": { - "deprecation_date": "2026-05-14", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "vertex_ai/claude-opus-4-1": { - "deprecation_date": "2026-08-05", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "input_cost_per_token_batches": 7.5e-06, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "output_cost_per_token_batches": 3.75e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "vertex_ai/claude-opus-4-1@20250805": { - "deprecation_date": "2026-08-05", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "input_cost_per_token_batches": 7.5e-06, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "output_cost_per_token_batches": 3.75e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "vertex_ai/claude-opus-4-5": { "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, @@ -51857,98 +46936,6 @@ "supports_native_streaming": true, "prompt_cache_min_tokens": 1024 }, - "vertex_ai/claude-opus-4@20250514": { - "deprecation_date": "2026-05-14", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "vertex_ai/claude-sonnet-4": { - "deprecation_date": "2026-05-14", - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 1000000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "vertex_ai/claude-sonnet-4@20250514": { - "deprecation_date": "2026-05-14", - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 1000000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, "vertex_ai/mistralai/codestral-2@001": { "input_cost_per_token": 3e-07, "litellm_provider": "vertex_ai-mistral_models", @@ -52451,62 +47438,6 @@ "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "cache_read_input_token_cost_batches": 1e-07 }, - "vertex_ai/imagegeneration@006": { - "deprecation_date": "2025-09-24", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-3.0-fast-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-3.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-3.0-generate-002": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-3.0-capability-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/image/edit-insert-objects" - }, - "vertex_ai/imagen-4.0-fast-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-4.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-4.0-ultra-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, "vertex_ai/jamba-1.5": { "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai-ai21_models", @@ -53251,51 +48182,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "vertex_ai/veo-2.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-video-models", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "video_generation", - "output_cost_per_second": 0.35, - "source": "https://ai.google.dev/gemini-api/docs/video", - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "video" - ] - }, - "vertex_ai/veo-3.0-fast-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-video-models", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "video_generation", - "output_cost_per_second": 0.15, - "source": "https://ai.google.dev/gemini-api/docs/video", - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "video" - ] - }, - "vertex_ai/veo-3.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-video-models", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "video_generation", - "output_cost_per_second": 0.4, - "source": "https://ai.google.dev/gemini-api/docs/video", - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "video" - ] - }, "vertex_ai/veo-3.1-generate-preview": { "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, @@ -53584,59 +48470,6 @@ "mode": "chat", "source": "https://wandb.ai/site/pricing/tokens/" }, - "wandb/zai-org/GLM-4.5": { - "deprecation_date": "2026-03-04", - "supports_reasoning": true, - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 0.055, - "output_cost_per_token": 0.2, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/Qwen/Qwen3-235B-A22B-Instruct-2507": { - "deprecation_date": "2026-08-04", - "max_tokens": 262144, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 1e-07, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/Qwen/Qwen3-Coder-480B-A35B-Instruct": { - "deprecation_date": "2026-08-25", - "max_tokens": 262144, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 1e-06, - "output_cost_per_token": 1.5e-06, - "litellm_provider": "wandb", - "mode": "chat", - "source": "https://wandb.ai/site/pricing/tokens/" - }, - "wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": { - "deprecation_date": "2026-08-04", - "supports_reasoning": true, - "max_tokens": 262144, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 1e-07, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/moonshotai/Kimi-K2-Instruct": { - "deprecation_date": "2026-03-04", - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 6e-07, - "output_cost_per_token": 2.5e-06, - "litellm_provider": "wandb", - "mode": "chat" - }, "wandb/moonshotai/Kimi-K2.5": { "max_tokens": 262144, "max_input_tokens": 262144, @@ -53652,20 +48485,6 @@ "supports_response_schema": true, "supports_vision": true }, - "wandb/MiniMaxAI/MiniMax-M2.5": { - "deprecation_date": "2026-08-25", - "max_tokens": 197000, - "max_input_tokens": 197000, - "max_output_tokens": 197000, - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "wandb", - "mode": "chat", - "source": "https://wandb.ai/inference/coreweave/cw_MiniMaxAI_MiniMax-M2.5", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true - }, "wandb/meta-llama/Llama-3.1-8B-Instruct": { "max_tokens": 128000, "max_input_tokens": 131000, @@ -53687,27 +48506,6 @@ "mode": "chat", "source": "https://wandb.ai/site/pricing/tokens/" }, - "wandb/deepseek-ai/DeepSeek-R1-0528": { - "deprecation_date": "2026-03-04", - "supports_reasoning": true, - "max_tokens": 161000, - "max_input_tokens": 161000, - "max_output_tokens": 161000, - "input_cost_per_token": 1.35e-06, - "output_cost_per_token": 5.4e-06, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/deepseek-ai/DeepSeek-V3-0324": { - "deprecation_date": "2026-03-04", - "max_tokens": 161000, - "max_input_tokens": 161000, - "max_output_tokens": 161000, - "input_cost_per_token": 1.14e-06, - "output_cost_per_token": 2.75e-06, - "litellm_provider": "wandb", - "mode": "chat" - }, "wandb/meta-llama/Llama-3.3-70B-Instruct": { "max_tokens": 128000, "max_input_tokens": 128000, @@ -53718,26 +48516,6 @@ "mode": "chat", "source": "https://wandb.ai/site/pricing/tokens/" }, - "wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": { - "deprecation_date": "2026-04-21", - "max_tokens": 64000, - "max_input_tokens": 64000, - "max_output_tokens": 64000, - "input_cost_per_token": 1.7e-07, - "output_cost_per_token": 6.6e-07, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/microsoft/Phi-4-mini-instruct": { - "deprecation_date": "2026-08-04", - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 0.008, - "output_cost_per_token": 0.035, - "litellm_provider": "wandb", - "mode": "chat" - }, "watsonx/ibm/granite-3-8b-instruct": { "input_cost_per_token": 2e-07, "litellm_provider": "watsonx", @@ -54138,440 +48916,6 @@ "deprecation_date": "2027-02-26", "source": "https://developers.openai.com/api/docs/pricing" }, - "xai/grok-3": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-beta": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-fast-beta": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-fast-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini": { - "cache_read_input_token_cost": 2e-07, - "deprecation_date": "2026-02-28", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-beta": { - "cache_read_input_token_cost": 2e-07, - "deprecation_date": "2026-02-28", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-fast": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-02-28", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-fast-beta": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-02-28", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-fast-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-02-28", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-02-28", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4": { - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-fast-reasoning": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-fast-non-reasoning": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-0709": { - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-latest": { - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast-reasoning": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast-reasoning-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast-non-reasoning": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast-non-reasoning-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, "xai/grok-4.20-multi-agent-beta-0309": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.25e-06, @@ -54816,72 +49160,6 @@ "supports_vision": true, "supports_web_search": true }, - "xai/grok-code-fast": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.x.ai/v1/language-models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "input_cost_per_token_above_200k_tokens": 2e-06, - "output_cost_per_token_above_200k_tokens": 4e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "supports_response_schema": true, - "supports_vision": true, - "deprecation_date": "2026-05-15", - "input_cost_per_image_token": 1e-06 - }, - "xai/grok-code-fast-1": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.x.ai/v1/language-models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "input_cost_per_token_above_200k_tokens": 2e-06, - "output_cost_per_token_above_200k_tokens": 4e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "supports_response_schema": true, - "supports_vision": true, - "deprecation_date": "2026-05-15", - "input_cost_per_image_token": 1e-06 - }, - "xai/grok-code-fast-1-0825": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.x.ai/v1/language-models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "input_cost_per_token_above_200k_tokens": 2e-06, - "output_cost_per_token_above_200k_tokens": 4e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "supports_response_schema": true, - "supports_vision": true, - "deprecation_date": "2026-05-15", - "input_cost_per_image_token": 1e-06 - }, "zai.glm-4.7": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", @@ -57624,30 +51902,6 @@ "supports_reasoning": true, "supports_vision": true }, - "scaleway/google/gemma-3-27b-it": { - "input_cost_per_token": 2.5e-07, - "litellm_provider": "scaleway", - "max_input_tokens": 40000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_function_calling": true, - "supports_vision": true, - "deprecation_date": "2026-08-01" - }, - "scaleway/hcompany/holo2-30b-a3b": { - "input_cost_per_token": 3e-07, - "litellm_provider": "scaleway", - "max_input_tokens": 22000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 7e-07, - "supports_reasoning": true, - "supports_vision": true, - "deprecation_date": "2026-08-09" - }, "scaleway/mistralai/mistral-medium-3.5-128b": { "input_cost_per_token": 1.5e-06, "litellm_provider": "scaleway", @@ -57661,29 +51915,6 @@ "supports_vision": true, "supports_tool_choice": true }, - "scaleway/mistralai/devstral-2-123b-instruct-2512": { - "input_cost_per_token": 4e-07, - "litellm_provider": "scaleway", - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 2e-06, - "supports_function_calling": true, - "deprecation_date": "2026-08-01" - }, - "scaleway/mistralai/voxtral-small-24b-2507": { - "input_cost_per_audio_token": 1.5e-07, - "input_cost_per_token": 1.5e-07, - "litellm_provider": "scaleway", - "max_input_tokens": 32000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 3.5e-07, - "supports_audio_input": true, - "deprecation_date": "2026-08-01" - }, "scaleway/mistralai/mistral-small-3.2-24b-instruct-2506": { "input_cost_per_token": 1.5e-07, "litellm_provider": "scaleway", @@ -59243,26 +53474,6 @@ "/v1/audio/speech" ] }, - "gpt-4o-mini-tts-2025-03-20": { - "deprecation_date": "2026-07-23", - "input_cost_per_token": 6e-07, - "litellm_provider": "openai", - "mode": "audio_speech", - "output_cost_per_audio_token": 1.2e-05, - "output_cost_per_second": 0.00025, - "output_cost_per_token": 1e-05, - "source": "https://developers.openai.com/api/docs/pricing", - "supported_endpoints": [ - "/v1/audio/speech" - ], - "supported_modalities": [ - "text", - "audio" - ], - "supported_output_modalities": [ - "audio" - ] - }, "gpt-4o-mini-tts-2025-12-15": { "input_cost_per_token": 6e-07, "litellm_provider": "openai", @@ -59366,41 +53577,6 @@ "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false }, - "gpt-realtime-mini-2025-10-06": { - "cache_creation_input_audio_token_cost": 3e-07, - "cache_read_input_audio_token_cost": 3e-07, - "cache_read_input_token_cost": 6e-08, - "deprecation_date": "2026-07-23", - "input_cost_per_audio_token": 1e-05, - "input_cost_per_image_token": 8e-07, - "input_cost_per_token": 6e-07, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 2.4e-06, - "source": "https://developers.openai.com/api/docs/pricing", - "supported_endpoints": [ - "/v1/realtime" - ], - "supported_modalities": [ - "text", - "image", - "audio" - ], - "supported_output_modalities": [ - "text", - "audio" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-realtime-mini-2025-12-15": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, @@ -59555,42 +53731,6 @@ "tpm": 250000, "rpm": 10 }, - "gemini/gemini-2.0-flash-lite-001": { - "cache_read_input_token_cost": 1.875e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7.5e-08, - "input_cost_per_token": 7.5e-08, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "rpm": 4000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.0-flash-lite", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 4000000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, "gemini-2.5-flash-native-audio-latest": { "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, @@ -65750,23 +59890,6 @@ "image" ] }, - "xai/grok-imagine-image-pro": { - "input_cost_per_image": 0.05, - "litellm_provider": "xai", - "mode": "image_generation", - "source": "https://docs.x.ai/docs/models", - "supported_endpoints": [ - "/v1/images/generations" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "image" - ], - "deprecation_date": "2026-05-15" - }, "xai/grok-imagine-image-2.0": { "input_cost_per_image": 0.06, "litellm_provider": "xai", @@ -66384,16 +60507,6 @@ "output_cost_per_token": 2.82e-07, "source": "https://api.together.ai/v1/models" }, - "together_ai/moonshotai/Kimi-K2.6": { - "deprecation_date": "2026-08-19", - "input_cost_per_token": 1.2e-06, - "output_cost_per_token": 4.5e-06, - "cache_read_input_token_cost": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, "together_ai/moonshotai/Kimi-K2.5-fp4": { "input_cost_per_token": 5e-07, "output_cost_per_token": 2.8e-06, @@ -66411,25 +60524,6 @@ "mode": "chat", "source": "https://api.together.ai/v1/models" }, - "together_ai/zai-org/GLM-5": { - "deprecation_date": "2026-06-22", - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3.2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 202752, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, - "together_ai/zai-org/GLM-5.1": { - "deprecation_date": "2026-07-10", - "input_cost_per_token": 1.4e-06, - "output_cost_per_token": 4.4e-06, - "cache_read_input_token_cost": 2.6e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 202752, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, "together_ai/deepseek-ai/DeepSeek-R1-0528": { "input_cost_per_token": 3e-06, "output_cost_per_token": 7e-06, @@ -66438,33 +60532,6 @@ "mode": "chat", "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen3-Coder-Next-FP8": { - "deprecation_date": "2026-05-14", - "input_cost_per_token": 5e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen3-VL-32B-Instruct": { - "deprecation_date": "2026-02-25", - "input_cost_per_token": 5e-07, - "output_cost_per_token": 1.5e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen3-VL-8B-Instruct": { - "deprecation_date": "2026-04-16", - "input_cost_per_token": 1.8e-07, - "output_cost_per_token": 6.8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, "together_ai/mistralai/Ministral-3-14B-Instruct-2512": { "input_cost_per_token": 2e-07, "output_cost_per_token": 2e-07, @@ -66489,15 +60556,6 @@ "mode": "chat", "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/QwQ-32B": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 1.2e-06, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, "cerebras/gemma-4-31b": { "input_cost_per_token": 9.9e-07, "litellm_provider": "cerebras", @@ -71240,37 +65298,14 @@ "output_cost_per_token": 1.5e-07, "source": "https://api.together.ai/v1/models" }, - "together_ai/deepseek-ai/deepseek-coder-33b-instruct": { - "deprecation_date": "2024-08-22", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { - "deprecation_date": "2025-12-23", - "input_cost_per_token": 2e-06, - "litellm_provider": "together_ai", - "mode": "chat", + "vertex_ai/gemini-2.5-flash-native-audio": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai", + "mode": "realtime", + "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 1.6e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1.6e-06, - "source": "https://api.together.ai/v1/models" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 5e-07, @@ -71321,14 +65356,6 @@ "output_cost_per_token": 6e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, - "together_ai/google/gemma-2-27b-it": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, "gpt-5.5-cyber": { "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 1.25e-05, @@ -71346,14 +65373,6 @@ "output_cost_per_token": 2.5e-05, "source": "https://developers.openai.com/api/docs/pricing" }, - "together_ai/meta-llama/Llama-3-8b-chat-hf": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/meta-llama/Llama-3.1-405B-Instruct": { "input_cost_per_token": 3.5e-06, "litellm_provider": "together_ai", @@ -71375,38 +65394,6 @@ "output_cost_per_token": 6e-08, "source": "https://api.together.ai/v1/models" }, - "together_ai/meta-llama/Meta-Llama-3-70B-Instruct-Turbo": { - "deprecation_date": "2025-12-23", - "input_cost_per_token": 8.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8.8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/meta-llama/Meta-Llama-3-8B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 8.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8.8e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/Qwen/Qwen2-1.5B-Instruct": { "input_cost_per_token": 2e-08, "litellm_provider": "together_ai", @@ -71414,22 +65401,6 @@ "output_cost_per_token": 2e-08, "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen2-72B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 9e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 9e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen2-VL-72B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 1.2e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://api.together.ai/v1/models" - }, "together_ai/Qwen/Qwen2.5-14B-Instruct": { "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", @@ -71444,22 +65415,6 @@ "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen2.5-Coder-32B-Instruct": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen2.5-VL-72B-Instruct": { - "deprecation_date": "2026-01-05", - "input_cost_per_token": 1.95e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8e-06, - "source": "https://api.together.ai/v1/models" - }, "azure/eu/codex-mini": { "deprecation_date": "2026-11-15", "cache_read_input_token_cost": 4.13e-07, @@ -71610,15 +65565,6 @@ "output_cost_per_token_priority": 3.08e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/eu/gpt-5.2-chat": { - "deprecation_date": "2026-06-29", - "cache_read_input_token_cost": 1.925e-07, - "input_cost_per_token": 1.925e-06, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 1.54e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/eu/gpt-5.2-codex": { "deprecation_date": "2027-07-13", "cache_read_input_token_cost": 1.925e-07, @@ -71637,15 +65583,6 @@ "output_cost_per_token_batches": 9.24e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/eu/gpt-5.3-chat": { - "deprecation_date": "2026-06-29", - "cache_read_input_token_cost": 1.925e-07, - "input_cost_per_token": 1.925e-06, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 1.54e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/eu/gpt-5.3-codex": { "deprecation_date": "2027-08-24", "cache_read_input_token_cost": 1.925e-07, @@ -71964,15 +65901,6 @@ "output_cost_per_token_priority": 3.08e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/us/gpt-5.2-chat": { - "deprecation_date": "2026-06-29", - "cache_read_input_token_cost": 1.925e-07, - "input_cost_per_token": 1.925e-06, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 1.54e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/us/gpt-5.2-codex": { "deprecation_date": "2027-07-13", "cache_read_input_token_cost": 1.925e-07, @@ -71991,15 +65919,6 @@ "output_cost_per_token_batches": 9.24e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/us/gpt-5.3-chat": { - "deprecation_date": "2026-06-29", - "cache_read_input_token_cost": 1.925e-07, - "input_cost_per_token": 1.925e-06, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 1.54e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/us/gpt-5.3-codex": { "deprecation_date": "2027-08-24", "cache_read_input_token_cost": 1.925e-07, diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 7bf4533979a..f787d370f04 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -1664,7 +1664,7 @@ class TestEnableAnthropicPromptCaching: points = self._points(model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", provider="bedrock") assert [p["index"] for p in points] == [None, -1] - @pytest.mark.parametrize("model, provider", [("gpt-4o", "openai"), ("gemini-2.0-flash", "gemini")]) + @pytest.mark.parametrize("model, provider", [("gpt-4o", "openai")]) def test_non_anthropic_providers_never_injected(self, monkeypatch, model, provider): """These report supports_prompt_caching=True but never consume cache_control markers.""" from litellm.utils import supports_prompt_caching diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 5b21684ca9c..76ccdec25d0 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -299,42 +299,6 @@ def test_reasoning_tokens_gemini(_local_model_cost_map): ) -def test_reasoning_tokens_gemini_3_1_flash_lite(_local_model_cost_map): - """Test cost calculation for gemini-3.1-flash-lite-preview with reasoning tokens""" - model = "gemini-3.1-flash-lite-preview" - custom_llm_provider = "gemini" - - usage = Usage( - completion_tokens=1000, - prompt_tokens=500, - total_tokens=1500, - completion_tokens_details=CompletionTokensDetailsWrapper( - accepted_prediction_tokens=None, - audio_tokens=None, - reasoning_tokens=400, - rejected_prediction_tokens=None, - text_tokens=600, - ), - prompt_tokens_details=PromptTokensDetailsWrapper( - audio_tokens=None, cached_tokens=None, text_tokens=500, image_tokens=None - ), - ) - model_cost_map = litellm.model_cost[model] - prompt_cost, completion_cost = generic_cost_per_token( - model=model, - usage=usage, - custom_llm_provider=custom_llm_provider, - ) - - assert round(prompt_cost, 10) == round( - model_cost_map["input_cost_per_token"] * usage.prompt_tokens, - 10, - ) - assert round(completion_cost, 10) == round( - (model_cost_map["output_cost_per_token"] * usage.completion_tokens_details.text_tokens) - + (model_cost_map["output_cost_per_reasoning_token"] * usage.completion_tokens_details.reasoning_tokens), - 10, - ) def test_image_tokens_with_custom_pricing(): @@ -2221,65 +2185,8 @@ def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(_local_mo assert round(cost, 10) == round(expected_cost, 10) -def test_gemini_image_generation_cost_prefers_token_usage_metadata(_local_model_cost_map): - """ - When usage metadata exists on image responses, Gemini image generation cost - should be calculated from token pricing, not flat output_cost_per_image. - """ - - model = "gemini/gemini-3-pro-image-preview" - model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") - - input_text_tokens = 20 - input_image_tokens = 1120 - output_image_tokens = 1120 - prompt_tokens = input_text_tokens + input_image_tokens - - image_response = ImageResponse( - data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")], - usage=ImageUsage( - input_tokens=prompt_tokens, - input_tokens_details=ImageUsageInputTokensDetails( - text_tokens=input_text_tokens, - image_tokens=input_image_tokens, - ), - output_tokens=output_image_tokens, - total_tokens=prompt_tokens + output_image_tokens, - ), - ) - - cost = gemini_image_generation_cost_calculator( - model=model, - image_response=image_response, - ) - - expected_prompt_cost = prompt_tokens * model_info["input_cost_per_token"] - expected_completion_cost = output_image_tokens * model_info["output_cost_per_image_token"] - expected_total_cost = expected_prompt_cost + expected_completion_cost - - assert round(cost, 10) == round(expected_total_cost, 10) - # Ensure this is not falling back to flat per-image pricing. - assert cost != len(image_response.data) * model_info["output_cost_per_image"] -def test_gemini_image_generation_cost_falls_back_to_flat_image_pricing(_local_model_cost_map): - """ - Without usage metadata, Gemini image generation cost should fall back to - output_cost_per_image * number_of_images. - """ - - model = "gemini/gemini-3-pro-image-preview" - model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") - - image_response = ImageResponse(data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")]) - - cost = gemini_image_generation_cost_calculator( - model=model, - image_response=image_response, - ) - - expected_cost = len(image_response.data) * model_info["output_cost_per_image"] - assert round(cost, 10) == round(expected_cost, 10) def test_query_count_is_free_without_a_per_query_price(_local_model_cost_map): @@ -2460,23 +2367,6 @@ def test_vertex_global_or_absent_location_no_uplift(vertex_location, _local_mode assert base == located -@pytest.mark.parametrize("model", ["claude-opus-4-1", "gemini-2.0-flash-001"]) -def test_vertex_location_no_uplift_for_uniformly_priced_model(model, _local_model_cost_map): - """Models Google prices uniformly across endpoints (Gemini 2.x, Claude Opus 4.1 - and older) carry no multiplier and must not move with the location.""" - from litellm.types.utils import Usage - - usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - - base = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="vertex_ai") - regional = generic_cost_per_token( - model=model, - usage=usage, - custom_llm_provider="vertex_ai", - vertex_location="us-east5", - ) - - assert base == regional, f"{model} should not have a regional-endpoint uplift" def test_vertex_uplift_invalid_multiplier_defaults_to_one(): @@ -3695,47 +3585,6 @@ def test_route_image_generation_cost_openai_honors_deployment_input_cost_per_ima assert cost == pytest.approx(0.07) -def test_route_image_generation_cost_gemini_adds_grounding_to_deployment_image_price( - _local_model_cost_map: None, -) -> None: - usage = ImageUsage( - input_tokens=0, - input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=0), - output_tokens=0, - total_tokens=0, - web_search_requests=3, - ) - - cost = CostCalculatorUtils.route_image_generation_cost_calculator( - model="gemini/gemini-3.1-flash-image-preview", - completion_response=_image_response(usage=usage), - custom_llm_provider="gemini", - call_type="image_generation", - model_info={"output_cost_per_image": 0.1}, - ) - - assert cost == pytest.approx(0.1 + 3 * 0.014) - - -def test_route_image_generation_cost_gemini_bills_tokens_when_no_image_returned( - _local_model_cost_map: None, -) -> None: - usage = ImageUsage( - input_tokens=10, - input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=10), - output_tokens=1290, - total_tokens=1300, - ) - - cost = CostCalculatorUtils.route_image_generation_cost_calculator( - model="gemini/gemini-3.1-flash-image-preview", - completion_response=ImageResponse(data=[], usage=usage), - custom_llm_provider="gemini", - call_type="image_generation", - model_info={"output_cost_per_image": 0.08}, - ) - - assert cost == pytest.approx(10 * 5e-07 + 1290 * 6e-05) @pytest.mark.parametrize( diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 7bae2eaa338..41a2d19b8ab 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -109,26 +109,6 @@ def test_get_cost_for_built_in_tools_file_search(): assert cost == 0.00 -def test_get_cost_for_anthropic_web_search(): - """ - Test that Anthropic web search cost is tracked when usage.server_tool_use.web_search_requests - is set. Use claude-3-7-sonnet-20250219 (has search_context_cost_per_query) and - custom_llm_provider=anthropic so get_cost_for_anthropic_web_search is invoked. - """ - from litellm.types.utils import ServerToolUse, Usage - - model = "claude-3-7-sonnet-20250219" - usage = Usage(server_tool_use=ServerToolUse(web_search_requests=1)) - cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( - model=model, - usage=usage, - response_object=None, - standard_built_in_tools_params=None, - custom_llm_provider="anthropic", - ) - assert cost > 0.0 - - def test_get_cost_for_anthropic_web_search_with_server_tool_use_dict(): """ Anthropic-compatible passthrough responses can construct Usage from a raw @@ -145,88 +125,6 @@ def test_get_cost_for_anthropic_web_search_with_server_tool_use_dict(): ) -def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_drops_server_tool_use(): - """ - Regression: on the Anthropic /v1/messages sync cost path the response is the raw - Anthropic dict while the reconstructed OpenAI-shape Usage drops server_tool_use. - The web-search fee must still be charged by reading the count off the raw dict, - and the passed-in Usage must not be mutated. - """ - from litellm.types.utils import Usage - - model = "claude-3-7-sonnet-20250219" - web_search_requests = 3 - raw_response = { - "id": "msg_1", - "type": "message", - "role": "assistant", - "model": model, - "content": [{"type": "text", "text": "hi"}], - "stop_reason": "end_turn", - "stop_sequence": None, - "usage": { - "input_tokens": 100, - "output_tokens": 50, - "server_tool_use": {"web_search_requests": web_search_requests}, - }, - } - usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) - assert getattr(usage, "server_tool_use", None) is None - - cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( - model=model, - usage=usage, - response_object=raw_response, - custom_llm_provider="anthropic", - standard_built_in_tools_params=None, - ) - - per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][ - "search_context_size_medium" - ] - assert cost == per_query_cost * web_search_requests - assert cost > 0.0 - assert getattr(usage, "server_tool_use", None) is None - - -def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_is_none(): - """ - Regression: when a caller hands the cost tracker a raw Anthropic dict without a - parallel Usage object, the web-search fee must still be priced per request from - usage.server_tool_use.web_search_requests on the dict instead of falling back to - the flat search_context_size_medium tier. - """ - model = "claude-3-7-sonnet-20250219" - web_search_requests = 4 - raw_response = { - "id": "msg_1", - "type": "message", - "role": "assistant", - "model": model, - "content": [{"type": "text", "text": "hi"}], - "stop_reason": "end_turn", - "stop_sequence": None, - "usage": { - "input_tokens": 100, - "output_tokens": 50, - "server_tool_use": {"web_search_requests": web_search_requests}, - }, - } - - cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( - model=model, - usage=None, - response_object=raw_response, - custom_llm_provider="anthropic", - standard_built_in_tools_params=None, - ) - - per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][ - "search_context_size_medium" - ] - assert cost == per_query_cost * web_search_requests - - def test_anthropic_web_search_zero_requests_from_raw_response_charges_zero(): """ Regression: a raw Anthropic dict reporting zero web search requests must price @@ -287,27 +185,6 @@ def test_anthropic_response_usage_block_preserves_server_tool_use(): assert dumped_usage["server_tool_use"] == {"web_search_requests": 2} -@pytest.mark.parametrize( - "model", ["gemini/gemini-2.0-flash-001", "gemini-2.0-flash-001"] -) -def test_get_cost_for_gemini_web_search(model): - """ - Test that the cost for a web search is 0.00 when no response object is provided - """ - from litellm.types.utils import PromptTokensDetailsWrapper, Usage - - usage = Usage( - prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=1) - ) - cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( - model=model, - usage=usage, - response_object=None, - standard_built_in_tools_params=None, - ) - assert cost > 0.0 - - def test_completion_cost_includes_web_search_without_standard_built_in_tools_params(): """ Test that completion_cost includes web search cost even when diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index 7928768b3bd..f70b52a7026 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -979,10 +979,6 @@ def test_shipped_tool_search_rule_fills_mapped_claude_entries_without_flag(shipp assert "supports_tool_search" not in litellm.model_cost[key] assert litellm.get_model_info(model, custom_llm_provider=provider)["supports_tool_search"] is True - assert "supports_tool_search" not in litellm.model_cost["claude-opus-4-1"] - opus_4_1_info = litellm.get_model_info("claude-opus-4-1", custom_llm_provider="anthropic") - assert opus_4_1_info.get("supports_tool_search") is None - assert "supports_tool_search" not in litellm.model_cost["azure_ai/claude-opus-5"] azure_opus_5_info = litellm.get_model_info("claude-opus-5", custom_llm_provider="azure_ai") assert azure_opus_5_info.get("supports_tool_search") is None diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index 262dabb7c1b..00977d9c3ee 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -218,7 +218,6 @@ def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0(): assert backup[adaptive]["supports_adaptive_thinking"] is True, adaptive for non_adaptive in [ - "claude-opus-4-20250514", "us.anthropic.claude-opus-4-20250514-v1:0", "claude-opus-4-5", ]: diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 4541250e896..5bdc0e9f8b8 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -8389,21 +8389,6 @@ def test_get_assembled_streaming_response_bills_a_provider_reported_usage_cost() assert logging_obj._response_cost_calculator(result=assembled) == 0.0042 -def test_get_assembled_streaming_response_without_usage_cost_leaves_pricing_to_the_price_map(): - logging_obj = _responses_stream_logging_obj() - now = datetime.datetime.now() - - assembled = logging_obj._get_assembled_streaming_response( - result=_completed_responses_event(ResponseAPIUsage(input_tokens=12, output_tokens=2, total_tokens=14)), - start_time=now, - end_time=now, - is_async=True, - streaming_chunks=[], - ) - - assert "additional_headers" not in assembled._hidden_params - price_map_cost = logging_obj._response_cost_calculator(result=assembled) - assert price_map_cost is not None and 0 < price_map_cost != 0.0042 def test_response_cost_calculator_prices_terminal_responses_event_from_its_response(): diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_server_tool_use.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_server_tool_use.py index 75508917a1e..e8a7ef74bfa 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_server_tool_use.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_server_tool_use.py @@ -99,29 +99,3 @@ def test_stream_chunk_builder_coerces_server_tool_use_to_pydantic(): assert server_tool_use.web_search_requests == 3 -def test_completion_cost_does_not_raise_on_streaming_web_search_response(): - """ - Regression: completion_cost(...) must not raise AttributeError when the - response was reconstructed by stream_chunk_builder from a streaming - Anthropic web_search call. - """ - chunks = [ - _make_text_chunk("hello"), - _make_finish_chunk_with_usage_dict_server_tool_use(), - ] - - rebuilt = stream_chunk_builder(chunks) - assert rebuilt is not None - - # The exact dollar amount depends on the model-pricing table; what matters - # for this regression is that it does NOT raise AttributeError on - # `dict has no attribute 'web_search_requests'`. - try: - cost = completion_cost(completion_response=rebuilt) - except AttributeError as e: # pragma: no cover - regression guard - pytest.fail( - "completion_cost raised AttributeError after stream_chunk_builder " - f"(issue #26153 regression): {e}" - ) - - assert isinstance(cost, (int, float)) diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 5ce6a4b1ce9..eccf44a1bda 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -633,9 +633,6 @@ def test_openai_token_with_image_and_text(): "model, base_model, input_tokens, user_max_tokens, expected_value", [ ("random-model", "random-model", 1024, 1024, 1024), - ("command", "command", 1000000, None, None), # model max = 4096 - ("command", "command", 4000, 256, 96), # model max = 4096 - ("command", "command", 4000, 10, 10), # model max = 4096 ("gpt-3.5-turbo", "gpt-3.5-turbo", 4000, 5000, 4096), # model max output = 4096 ], ) diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index db5da28c024..f92c610df24 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -5580,43 +5580,6 @@ def test_tool_config_cachepoint_not_placed_or_credited_for_model_without_prompt_ assert "litellm_gateway_injected_cache" not in bucket -def test_translate_response_format_json_schema_still_injects_tool(): - """ - response_format with an explicit json_schema should still use the - synthetic tool call approach (for models that don't support native - structured outputs). - """ - config = AmazonConverseConfig() - - response_format = { - "type": "json_schema", - "json_schema": { - "name": "FactResult", - "schema": { - "type": "object", - "properties": { - "facts": { - "type": "array", - "items": {"type": "string"}, - }, - }, - "required": ["facts"], - }, - }, - } - - optional_params: dict = {} - result = config._translate_response_format_param( - value=response_format, - model="anthropic.claude-3-haiku-20240307-v1:0", - optional_params=optional_params, - non_default_params={"response_format": response_format}, - is_thinking_enabled=False, - ) - - assert result["json_mode"] is True - assert "tools" in result - assert "tool_choice" in result def test_transform_response_finish_reason_stop_when_json_mode_filters_all_tools(): diff --git a/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py b/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py index afac7b0bc1a..de0e547c0cd 100644 --- a/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py +++ b/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py @@ -153,14 +153,6 @@ def test_uncached_request_bills_every_prompt_token_at_the_input_rate(local_model assert completion_cost == pytest.approx(200 * info["output_cost_per_token"]) -def test_legacy_endpoint_names_still_resolve(local_model_cost_map: None) -> None: - info: Final = _model_info("databricks/databricks-mixtral-8x7b-instruct") - usage: Final = Usage(prompt_tokens=100, completion_tokens=100, total_tokens=200) - - prompt_cost, completion_cost = cost_per_token(model="databricks/mixtral-8x7b-instruct-v0.1", usage=usage) - - assert prompt_cost == pytest.approx(100 * info["input_cost_per_token"]) - assert completion_cost == pytest.approx(100 * info["output_cost_per_token"]) @pytest.mark.parametrize("model", NEW_MODELS) diff --git a/tests/test_litellm/llms/gemini/test_cost_calculator.py b/tests/test_litellm/llms/gemini/test_cost_calculator.py index 5eed11dff03..b2633c6091b 100644 --- a/tests/test_litellm/llms/gemini/test_cost_calculator.py +++ b/tests/test_litellm/llms/gemini/test_cost_calculator.py @@ -200,172 +200,12 @@ def test_maps_no_usage_details(): assert cost_per_google_maps_grounding_request(usage=usage, model_info=model_info) == 0.0 -def test_gemini_image_edit_cost_prefers_token_usage_metadata(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - model = "gemini/gemini-3-pro-image-preview" - model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") - - input_text_tokens = 20 - input_image_tokens = 1120 - output_image_tokens = 1120 - prompt_tokens = input_text_tokens + input_image_tokens - image_response = ImageResponse( - data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")], - usage=ImageUsage( - input_tokens=prompt_tokens, - input_tokens_details=ImageUsageInputTokensDetails( - text_tokens=input_text_tokens, - image_tokens=input_image_tokens, - ), - output_tokens=output_image_tokens, - total_tokens=prompt_tokens + output_image_tokens, - ), - ) - - cost = gemini_image_edit_cost_calculator( - model=model, - image_response=image_response, - ) - - expected_cost = ( - prompt_tokens * model_info["input_cost_per_token"] - + output_image_tokens * model_info["output_cost_per_image_token"] - ) - flat_image_cost = ( - len(image_response.data or []) * model_info["output_cost_per_image"] - ) - assert round(cost, 10) == round(expected_cost, 10) - assert cost != flat_image_cost -def test_gemini_image_edit_cost_uses_output_token_details(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - model = "gemini/gemini-3-pro-image-preview" - model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") - - input_text_tokens = 20 - output_text_tokens = 213 - output_image_tokens = 1120 - output_tokens = output_text_tokens + output_image_tokens - image_response = ImageResponse( - data=[ImageObject(b64_json="img1")], - usage=ImageUsage( - input_tokens=input_text_tokens, - input_tokens_details=ImageUsageInputTokensDetails( - text_tokens=input_text_tokens, - image_tokens=0, - ), - output_tokens=output_tokens, - total_tokens=input_text_tokens + output_tokens, - prompt_tokens=input_text_tokens, - completion_tokens=output_tokens, - prompt_tokens_details={ - "text_tokens": input_text_tokens, - "image_tokens": 0, - }, - completion_tokens_details={ - "text_tokens": output_text_tokens, - "image_tokens": output_image_tokens, - }, - output_tokens_details={ - "text_tokens": output_text_tokens, - "image_tokens": output_image_tokens, - }, - ), - ) - - cost = gemini_image_edit_cost_calculator( - model=model, - image_response=image_response, - ) - - expected_cost = ( - input_text_tokens * model_info["input_cost_per_token"] - + output_text_tokens * model_info["output_cost_per_token"] - + output_image_tokens * model_info["output_cost_per_image_token"] - ) - all_output_as_image_cost = ( - input_text_tokens * model_info["input_cost_per_token"] - + (output_text_tokens + output_image_tokens) - * model_info["output_cost_per_image_token"] - ) - assert round(cost, 10) == round(expected_cost, 10) - assert cost != all_output_as_image_cost -def test_gemini_image_generation_cost_uses_output_token_details(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - model = "gemini/gemini-3-pro-image-preview" - model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") - - input_text_tokens = 20 - output_text_tokens = 213 - output_image_tokens = 1120 - output_tokens = output_text_tokens + output_image_tokens - image_response = ImageResponse( - data=[ImageObject(b64_json="img1")], - usage=ImageUsage( - input_tokens=input_text_tokens, - input_tokens_details=ImageUsageInputTokensDetails( - text_tokens=input_text_tokens, - image_tokens=0, - ), - output_tokens=output_tokens, - total_tokens=input_text_tokens + output_tokens, - prompt_tokens=input_text_tokens, - completion_tokens=output_tokens, - prompt_tokens_details={ - "text_tokens": input_text_tokens, - "image_tokens": 0, - }, - completion_tokens_details={ - "text_tokens": output_text_tokens, - "image_tokens": output_image_tokens, - }, - output_tokens_details={ - "text_tokens": output_text_tokens, - "image_tokens": output_image_tokens, - }, - ), - ) - - cost = gemini_image_generation_cost_calculator( - model=model, - image_response=image_response, - ) - - expected_cost = ( - input_text_tokens * model_info["input_cost_per_token"] - + output_text_tokens * model_info["output_cost_per_token"] - + output_image_tokens * model_info["output_cost_per_image_token"] - ) - all_output_as_image_cost = ( - input_text_tokens * model_info["input_cost_per_token"] - + (output_text_tokens + output_image_tokens) - * model_info["output_cost_per_image_token"] - ) - assert round(cost, 10) == round(expected_cost, 10) - assert cost != all_output_as_image_cost -def test_gemini_image_edit_cost_falls_back_to_flat_image_pricing(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - model = "gemini/gemini-3-pro-image-preview" - model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") - image_response = ImageResponse( - data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")] - ) - - cost = gemini_image_edit_cost_calculator( - model=model, - image_response=image_response, - ) - - assert cost == len(image_response.data or []) * model_info["output_cost_per_image"] def _image_response_with_web_search(web_search_requests): @@ -383,43 +223,8 @@ def _image_response_with_web_search(web_search_requests): return ImageResponse(data=[ImageObject(b64_json="img1")], usage=usage) -def test_gemini_image_generation_cost_adds_web_search_grounding(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - model = "gemini/gemini-3-pro-image-preview" - model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") - - grounded = gemini_image_generation_cost_calculator( - model=model, - image_response=_image_response_with_web_search(2), - ) - ungrounded = gemini_image_generation_cost_calculator( - model=model, - image_response=_image_response_with_web_search(None), - ) - - expected_web_search_cost = cost_per_web_search_request( - usage=_make_usage(2), model_info=model_info - ) - assert expected_web_search_cost > 0 - assert round(grounded - ungrounded, 10) == round(expected_web_search_cost, 10) -def test_gemini_image_generation_cost_no_web_search_when_absent(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - model = "gemini/gemini-3-pro-image-preview" - - cost_zero = gemini_image_generation_cost_calculator( - model=model, - image_response=_image_response_with_web_search(0), - ) - cost_none = gemini_image_generation_cost_calculator( - model=model, - image_response=_image_response_with_web_search(None), - ) - - assert cost_zero == cost_none @pytest.mark.parametrize( diff --git a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py index f1f1978b06f..b1d7e49fbac 100644 --- a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py +++ b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py @@ -371,62 +371,8 @@ def test_x_initiator_header_system_only_messages(): assert headers["X-Initiator"] == "user" -def test_get_supported_openai_params_claude_model(): - """Test that Claude models with extended thinking support have thinking and reasoning parameters.""" - config = GithubCopilotConfig() - - # Test Claude 4 model supports thinking and reasoning_effort parameters - supported_params = config.get_supported_openai_params("claude-sonnet-4-20250514") - assert "thinking" in supported_params - assert "reasoning_effort" in supported_params - - # Test Claude 3-7 model supports thinking and reasoning_effort parameters - supported_params_claude37 = config.get_supported_openai_params( - "claude-3-7-sonnet-20250219" - ) - assert "thinking" in supported_params_claude37 - assert "reasoning_effort" in supported_params_claude37 - - # Test Claude 3.5 model does NOT support thinking parameters (no extended thinking) - supported_params_claude35 = config.get_supported_openai_params("claude-3.5-sonnet") - assert "thinking" not in supported_params_claude35 - assert "reasoning_effort" not in supported_params_claude35 - - # Test non-Claude model doesn't include thinking parameters but may include reasoning_effort - supported_params_gpt = config.get_supported_openai_params("gpt-4o") - assert "thinking" not in supported_params_gpt - # gpt-4o should NOT have reasoning_effort (not a reasoning model) - assert "reasoning_effort" not in supported_params_gpt - - # Test O-series reasoning models include reasoning_effort but not thinking - supported_params_o3 = config.get_supported_openai_params("o3-mini") - assert "thinking" not in supported_params_o3 - # o3-mini should have reasoning_effort (it's an O-series reasoning model) - assert "reasoning_effort" in supported_params_o3 -def test_get_supported_openai_params_case_insensitive(): - """Test that Claude model detection is case-insensitive for models with extended thinking.""" - config = GithubCopilotConfig() - - # Test uppercase Claude 4 model with full model name - supported_params_upper = config.get_supported_openai_params( - "CLAUDE-SONNET-4-20250514" - ) - assert "thinking" in supported_params_upper - assert "reasoning_effort" in supported_params_upper - - # Test mixed case Claude 3-7 model (has extended thinking) with full model name - supported_params_mixed = config.get_supported_openai_params( - "Claude-3-7-Sonnet-20250219" - ) - assert "thinking" in supported_params_mixed - assert "reasoning_effort" in supported_params_mixed - - # Test that Claude 3.5 models don't have thinking support (case insensitive) - supported_params_35 = config.get_supported_openai_params("CLAUDE-3.5-SONNET") - assert "thinking" not in supported_params_35 - assert "reasoning_effort" not in supported_params_35 def test_copilot_vision_request_header_with_image(): diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index 0adc7fa8d5f..41f5816600f 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -38,10 +38,6 @@ def test_gpt5_supports_reasoning_effort(config: OpenAIConfig): assert "reasoning_effort" in config.get_supported_openai_params(model="gpt-5-mini") -def test_gpt5_chat_does_not_support_reasoning_effort(config: OpenAIConfig): - assert "reasoning_effort" not in config.get_supported_openai_params( - model="gpt-5-chat-latest" - ) def test_gpt5_chat_supports_temperature(config: OpenAIConfig): @@ -174,10 +170,6 @@ def test_gpt5_codex_unsupported_params_drop(config: OpenAIConfig): assert param not in config.get_supported_openai_params(model="gpt-5-codex") -def test_gpt5_codex_supports_tool_choice(gpt5_config: OpenAIGPT5Config): - """Test that GPT-5-Codex supports tool_choice parameter.""" - supported_params = gpt5_config.get_supported_openai_params(model="gpt-5-codex") - assert "tool_choice" in supported_params def test_gpt5_codex_supports_function_calling(config: OpenAIConfig): @@ -246,14 +238,6 @@ def test_gpt5_1_reasoning_effort_none(config: OpenAIConfig): assert params["reasoning_effort"] == effort -def test_gpt5_1_codex_max_allows_reasoning_effort_xhigh(config: OpenAIConfig): - params = config.map_openai_params( - non_default_params={"reasoning_effort": "xhigh"}, - optional_params={}, - model="gpt-5.1-codex-max", - drop_params=False, - ) - assert params["reasoning_effort"] == "xhigh" def test_gpt5_rejects_reasoning_effort_xhigh_for_other_models(config: OpenAIConfig): diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index a796e2ac607..8b0662de3f9 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -5948,8 +5948,8 @@ def test_calculate_web_search_requests_counts_unique_queries(): @pytest.mark.parametrize("custom_llm_provider", ["gemini", "vertex_ai"]) @pytest.mark.parametrize( "model", - ["gemini-2.5-flash", "gemini-3-pro-preview"], - ids=["thinking_budget_mapper", "thinking_level_mapper"], + ["gemini-2.5-flash"], + ids=["thinking_budget_mapper"], ) @pytest.mark.parametrize("reasoning_effort", ["banana", "xhigh"]) def test_invalid_reasoning_effort_is_a_400_not_a_500(custom_llm_provider, model, reasoning_effort): diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py index 58e7529309a..4199e57d2f9 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py @@ -238,31 +238,3 @@ def test_audio_predict_response_supports_bytes_base64_encoded( assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.06) -def test_image_predict_response_is_not_billed_as_audio( - local_model_cost_map: None, -) -> None: - logging_obj = MagicMock() - logging_obj.model_call_details = {} - response = httpx.Response( - status_code=200, - json={"predictions": [{"bytesBase64Encoded": "frame", "mimeType": "image/png"}]}, - ) - - result = VertexPassthroughLoggingHandler.vertex_passthrough_handler( - httpx_response=response, - logging_obj=logging_obj, - url_route=( - "/v1/projects/test/locations/us-central1/publishers/google/models/imagen-4.0-generate-001:predict" - ), - result=response.text, - start_time=datetime.now(), - end_time=datetime.now(), - cache_hit=False, - request_body={"instances": [{"prompt": "a red cube"}]}, - ) - - assert isinstance(result["result"], litellm.ImageResponse) - assert logging_obj.call_type == PassthroughCallTypes.passthrough_image_generation.value - assert result["kwargs"]["response_cost"] == pytest.approx( - litellm.model_cost["vertex_ai/imagen-4.0-generate-001"]["output_cost_per_image"] - ) diff --git a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py index a25ed585ecc..5c168d84766 100644 --- a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py +++ b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py @@ -37,10 +37,6 @@ WANDB_REASONING_MODELS: Final = ( "Qwen/Qwen3.5-35B-A3B", "zai-org/GLM-5.2", "moonshotai/Kimi-K2.5", - "MiniMaxAI/MiniMax-M2.5", - "zai-org/GLM-4.5", - "Qwen/Qwen3-235B-A22B-Thinking-2507", - "deepseek-ai/DeepSeek-R1-0528", ) diff --git a/tests/test_litellm/llms/xai/test_xai_cost_calculator.py b/tests/test_litellm/llms/xai/test_xai_cost_calculator.py index cf3bc73a225..6065f1053e7 100644 --- a/tests/test_litellm/llms/xai/test_xai_cost_calculator.py +++ b/tests/test_litellm/llms/xai/test_xai_cost_calculator.py @@ -152,83 +152,10 @@ class TestXAICostCalculator: setattr(reported, "server_side_tool_usage_details", {"web_search_calls": 3}) assert get_cost_for_web_search_request("xai", reported, {}) == 0.0 - def test_no_reported_cost_falls_back_to_token_math(self): - """Absent the provider figure, nothing changes for existing callers.""" - usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300) - prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) - assert prompt_cost > 0.0 - assert completion_cost > 0.0 - def test_malformed_reported_cost_falls_back_to_token_math(self): - """A junk value must not fail the request, fall back to calculating.""" - usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300) - setattr(usage, "cost", "not-a-number") - prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) - - assert prompt_cost > 0.0 - assert completion_cost > 0.0 - - def test_boolean_reported_cost_falls_back_to_token_math(self): - """True is an int in python and would otherwise be billed as $1.""" - usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300) - setattr(usage, "cost", True) - - prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) - - assert prompt_cost > 0.0 - assert completion_cost > 0.0 - assert completion_cost != 1.0 - - def test_negative_reported_cost_is_rejected(self): - """A negative amount must never reach spend tracking. - - A caller who can set api_base controls the response body, so trusting a - negative figure would let them subtract from their own recorded spend and - slip past a budget. Fall back to token pricing instead, and keep charging - the web search surcharge, since no trustworthy total was reported. - """ - usage = Usage( - prompt_tokens=100, - completion_tokens=200, - total_tokens=300, - cost=-0.0037756, - ) - setattr(usage, "server_side_tool_usage_details", {"web_search_calls": 3}) - - prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) - - assert prompt_cost > 0.0 - assert completion_cost > 0.0 - assert cost_per_web_search_request(usage=usage, model_info={}) > 0.0 - - def test_non_finite_reported_cost_is_rejected(self): - """NaN compares false against every budget threshold. - - Usage stores a provider supplied cost without validating it, so a caller who - controls the response body could report NaN and leave spend >= max_budget - false for the life of the key rather than mispricing one request. The - infinities are refused alongside it. Fall back to token pricing and keep - charging the web search surcharge, since no trustworthy total was reported. - """ - for reported_cost in (float("nan"), float("inf"), float("-inf")): - usage = Usage( - prompt_tokens=100, - completion_tokens=200, - total_tokens=300, - cost=reported_cost, - ) - setattr(usage, "server_side_tool_usage_details", {"web_search_calls": 3}) - - prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) - - assert math.isfinite(prompt_cost), reported_cost - assert math.isfinite(completion_cost), reported_cost - assert prompt_cost > 0.0, reported_cost - assert completion_cost > 0.0, reported_cost - assert cost_per_web_search_request(usage=usage, model_info={}) > 0.0, reported_cost def test_zero_reported_cost_is_honoured(self): """A reported zero is a real answer, not a missing value.""" diff --git a/tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py b/tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py deleted file mode 100644 index 83e8925f70b..00000000000 --- a/tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -xAI retired eight slugs on 2026-05-15 but kept them resolvable: chat slugs redirect to -grok-4.3 and bill at grok-4.3's rates, while the grok-code-fast slugs are aliases of -grok-build-0.1 and bill at its rates, so the registry must price them that way or spend -tracking is wrong. The grok-3-beta, grok-3-fast, grok-3-mini, and grok-4-1-fast slugs -are absent from /v1/language-models and resolve to grok-4.3 the same way (the chat -response names grok-4.3 as the served model), so they carry grok-4.3's rates too. -https://docs.x.ai/developers/migration/may-15-retirement -https://docs.x.ai/developers/models/grok-build-0.1 -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -REPO_ROOT = Path(__file__).parents[4] -PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" -BACKUP_PRICES_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" -MAP_PATHS = (PRICES_PATH, BACKUP_PRICES_PATH) - -REDIRECT_TARGET = "xai/grok-4.3" -GROK_3_MINI_SLUGS = ( - "xai/grok-3-mini", - "xai/grok-3-mini-beta", - "xai/grok-3-mini-fast", - "xai/grok-3-mini-fast-beta", - "xai/grok-3-mini-fast-latest", - "xai/grok-3-mini-latest", -) -REDIRECTED_SLUGS = ( - "xai/grok-3", - "xai/grok-3-beta", - "xai/grok-3-fast-beta", - "xai/grok-3-fast-latest", - "xai/grok-3-latest", - *GROK_3_MINI_SLUGS, - "xai/grok-4", - "xai/grok-4-0709", - "xai/grok-4-1-fast", - "xai/grok-4-1-fast-non-reasoning", - "xai/grok-4-1-fast-non-reasoning-latest", - "xai/grok-4-1-fast-reasoning", - "xai/grok-4-1-fast-reasoning-latest", - "xai/grok-4-fast-non-reasoning", - "xai/grok-4-fast-reasoning", - "xai/grok-4-latest", -) -CODE_REDIRECT_TARGET = "xai/grok-build-0.1" -CODE_SLUGS = ( - "xai/grok-code-fast", - "xai/grok-code-fast-1", - "xai/grok-code-fast-1-0825", -) -BASE_COST_FIELDS = ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost") -TIER_COST_FIELDS = ( - "input_cost_per_token_above_200k_tokens", - "output_cost_per_token_above_200k_tokens", - "cache_read_input_token_cost_above_200k_tokens", -) - - -@pytest.fixture(scope="module", params=[p.name for p in MAP_PATHS]) -def cost_map(request: pytest.FixtureRequest) -> dict: - path = next(p for p in MAP_PATHS if p.name == request.param) - return json.loads(path.read_text(encoding="utf-8")) - - -@pytest.mark.parametrize("slug", REDIRECTED_SLUGS) -def test_redirected_slug_bills_at_the_target_rate(cost_map: dict, slug: str): - target = cost_map[REDIRECT_TARGET] - entry = cost_map[slug] - for field in BASE_COST_FIELDS: - assert entry[field] == target[field], field - - -@pytest.mark.parametrize("slug", CODE_SLUGS) -def test_code_slug_bills_at_grok_build_rate(cost_map: dict, slug: str): - """grok-code-fast* are aliases of grok-build-0.1, not grok-4.3 redirects.""" - target = cost_map[CODE_REDIRECT_TARGET] - entry = cost_map[slug] - for field in (*BASE_COST_FIELDS, *TIER_COST_FIELDS): - assert entry[field] == target[field], field - - -@pytest.mark.parametrize("slug", REDIRECTED_SLUGS) -def test_redirected_slug_carries_the_target_tier_rates(cost_map: dict, slug: str): - """The request executes as grok-4.3, so it is tiered at grok-4.3's 200k boundary.""" - target = cost_map[REDIRECT_TARGET] - entry = cost_map[slug] - for field in TIER_COST_FIELDS: - assert entry[field] == target[field], field - assert {k for k in entry if "_above_" in k} == {k for k in target if "_above_" in k} - - -def test_both_cost_maps_agree_on_the_redirected_slugs(): - prices = json.loads(PRICES_PATH.read_text(encoding="utf-8")) - backup = json.loads(BACKUP_PRICES_PATH.read_text(encoding="utf-8")) - for slug in (*REDIRECTED_SLUGS, *CODE_SLUGS, REDIRECT_TARGET, CODE_REDIRECT_TARGET): - assert prices[slug] == backup[slug], slug diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 14b739e60ba..ffba53a5b7a 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -8057,7 +8057,6 @@ def test_model_has_no_cost_mapping_no_model_or_router_is_false(): [ "azure/speech/azure-tts", "mistral/mistral-ocr-latest", - "vertex_ai/imagen-3.0-generate-001", "dashscope/qwen-flash", ], ) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index ba8b5fa3ac4..094320225cd 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -588,43 +588,6 @@ class TestAzureAnthropicCostCalculation: == "claude-3-5-haiku-20241022" ) - def test_passthrough_logging_sets_response_cost_with_server_tool_use_dict(self): - from litellm.types.utils import Choices, Message, ModelResponse - - logging_obj = self._create_mock_logging_obj(model="claude-3-7-sonnet-20250219") - logging_obj.get_router_model_id.return_value = None - logging_obj.litellm_params = {} - - response = ModelResponse( - id="test-id", - choices=[ - Choices( - finish_reason="stop", - index=0, - message=Message(content="test", role="assistant"), - ) - ], - created=1234567890, - model="claude-3-7-sonnet-20250219", - usage={ - "prompt_tokens": 10, - "completion_tokens": 5, - "total_tokens": 15, - "server_tool_use": {"web_search_requests": 1}, - }, - ) - - kwargs = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( - litellm_model_response=response, - model="claude-3-7-sonnet-20250219", - kwargs={}, - start_time=datetime.now(), - end_time=datetime.now(), - logging_obj=logging_obj, - ) - - assert "response_cost" in kwargs - assert kwargs["response_cost"] > 0 class TestAnthropicBatchPassthroughCostTracking: @@ -2355,42 +2318,6 @@ class TestAnthropicResponseCostRecordedOnModelCallDetails: model_call_details["response_cost"], not from kwargs, so the streaming payload builder must record it there or streaming pass-through logs $0.""" - def test_create_payload_records_response_cost_on_model_call_details(self): - from litellm.types.utils import Choices, Message, ModelResponse - - logging_obj = MagicMock() - logging_obj.model_call_details = {} - logging_obj.get_router_model_id.return_value = None - logging_obj.litellm_params = {} - logging_obj.litellm_call_id = "test-call-id" - - response = ModelResponse( - id="test-id", - choices=[ - Choices( - finish_reason="stop", - index=0, - message=Message(content="hello", role="assistant"), - ) - ], - created=1234567890, - model="claude-3-7-sonnet-20250219", - usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, - ) - - kwargs = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( - litellm_model_response=response, - model="claude-3-7-sonnet-20250219", - kwargs={}, - start_time=datetime.now(), - end_time=datetime.now(), - logging_obj=logging_obj, - ) - - assert ( - logging_obj.model_call_details["response_cost"] == kwargs["response_cost"] - ) - assert logging_obj.model_call_details["response_cost"] > 0 class TestAnthropicPassthroughFastMode: diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py index 1d2d7d4d5c3..46f024366ec 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py @@ -462,30 +462,6 @@ class TestVertexAIBatchPassthroughHandler: assert mock_store.call_args[1]["unified_object_id"] assert mock_store.call_args[1]["is_batch_create"] is expected - def test_batch_cost_calculation_integration(self): - """Single Vertex AI response → non-zero cost with correct token counts.""" - from litellm.batches.batch_utils import calculate_vertex_ai_batch_cost_and_usage - - vertex_ai_batch_responses = [ - { - "response": { - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 5, - "totalTokenCount": 15, - } - } - } - ] - - result = calculate_vertex_ai_batch_cost_and_usage( - vertex_ai_batch_responses, model_name="gemini-2.0-flash-001" - ) - - assert result.usage.total_tokens == 15 - assert result.usage.prompt_tokens == 10 - assert result.usage.completion_tokens == 5 - assert result.cost > 0, "batch_cost_calculator should return a non-zero cost" def test_batch_response_transformation(self): """Test transformation of Vertex AI batch responses to OpenAI format""" @@ -639,76 +615,7 @@ class TestVertexAIBatchCostCalculation: batch_cost_calculator — no VertexGeminiConfig transformation involved. """ - def test_should_aggregate_cost_and_usage_across_responses(self): - """Two successful responses → costs and token counts are summed.""" - from litellm.batches.batch_utils import calculate_vertex_ai_batch_cost_and_usage - responses = [ - { - "response": { - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 5, - "totalTokenCount": 15, - } - } - }, - { - "response": { - "usageMetadata": { - "promptTokenCount": 8, - "candidatesTokenCount": 3, - "totalTokenCount": 11, - } - } - }, - ] - - result = calculate_vertex_ai_batch_cost_and_usage( - responses, model_name="gemini-2.0-flash-001" - ) - - assert result.usage.prompt_tokens == 18 - assert result.usage.completion_tokens == 8 - assert result.usage.total_tokens == 26 - assert result.cost > 0, "batch_cost_calculator should return a non-zero cost" - - def test_should_skip_responses_with_null_response_body(self): - """Failed lines (response: None) are skipped without error.""" - from litellm.batches.batch_utils import calculate_vertex_ai_batch_cost_and_usage - - responses = [ - { - "response": { - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 5, - "totalTokenCount": 15, - } - } - }, - {"status": "JOB_STATE_FAILED", "response": None}, - { - "response": { - "usageMetadata": { - "promptTokenCount": 8, - "candidatesTokenCount": 3, - "totalTokenCount": 11, - } - } - }, - ] - - result = calculate_vertex_ai_batch_cost_and_usage( - responses, model_name="gemini-2.0-flash-001" - ) - - assert result.usage.prompt_tokens == 18 - assert result.usage.completion_tokens == 8 - assert result.usage.total_tokens == 26 - assert result.cost > 0 - assert result.successful_requests == 2 - assert result.failed_requests == 1 def test_should_return_zeros_for_empty_response_list(self): """Empty input → zero cost and zero usage.""" @@ -739,143 +646,4 @@ class TestVertexAIBatchCostCalculation: assert result.usage.completion_tokens == 0 assert result.usage.total_tokens == 0 - @pytest.mark.asyncio - async def test_openai_shaped_output_records_nonzero_cost_and_usage(self): - """ - Regression test for the bug where Vertex batch cost/usage was always 0. - After PR #25627 (transform_file_content_response), the GCS predictions.jsonl - is rewritten into OpenAI batch shape before the cost-tracking path sees it. - With disable_vertex_batch_output_transformation=False (default), the cost - dispatch must fall through to the generic aggregation path rather than - calling calculate_vertex_ai_batch_cost_and_usage (which only reads raw - usageMetadata fields). - """ - import litellm - from litellm.batches.batch_utils import calculate_batch_cost_and_usage - - openai_shaped_responses = [ - { - "id": "batch_req_abc123", - "custom_id": "request-1", - "response": { - "status_code": 200, - "request_id": "chatcmpl-xyz", - "body": { - "id": "chatcmpl-xyz", - "object": "chat.completion", - "model": "gemini-2.0-flash-001", - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "Hello!"}, - "finish_reason": "stop", - } - ], - "usage": { - "prompt_tokens": 10, - "completion_tokens": 5, - "total_tokens": 15, - }, - }, - }, - "error": None, - }, - { - "id": "batch_req_def456", - "custom_id": "request-2", - "response": { - "status_code": 200, - "request_id": "chatcmpl-uvw", - "body": { - "id": "chatcmpl-uvw", - "object": "chat.completion", - "model": "gemini-2.0-flash-001", - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "World!"}, - "finish_reason": "stop", - } - ], - "usage": { - "prompt_tokens": 8, - "completion_tokens": 3, - "total_tokens": 11, - }, - }, - }, - "error": None, - }, - ] - - original_flag = getattr( - litellm, "disable_vertex_batch_output_transformation", False - ) - try: - litellm.disable_vertex_batch_output_transformation = False - - result = await calculate_batch_cost_and_usage( - file_content_dictionary=openai_shaped_responses, - custom_llm_provider="vertex_ai", - model_name="gemini-2.0-flash-001", - ) - finally: - litellm.disable_vertex_batch_output_transformation = original_flag - - assert ( - result.usage.prompt_tokens == 18 - ), f"expected 18 prompt tokens, got {result.usage.prompt_tokens}" - assert ( - result.usage.completion_tokens == 8 - ), f"expected 8 completion tokens, got {result.usage.completion_tokens}" - assert ( - result.usage.total_tokens == 26 - ), f"expected 26 total tokens, got {result.usage.total_tokens}" - assert ( - result.cost > 0 - ), f"expected non-zero cost for completed Vertex batch, got {result.cost}" - - @pytest.mark.asyncio - async def test_raw_vertex_output_still_works_when_transformation_disabled(self): - """ - When disable_vertex_batch_output_transformation=True the GCS file is returned - as raw Vertex predictions.jsonl; the specialized reader must be used. - """ - import litellm - from litellm.batches.batch_utils import calculate_batch_cost_and_usage - - raw_vertex_responses = [ - { - "request": {"contents": [{"role": "user", "parts": [{"text": "hi"}]}]}, - "status": "", - "response": { - "candidates": [{"content": {"parts": [{"text": "Hello!"}]}}], - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 5, - "totalTokenCount": 15, - }, - }, - "processed_time": "2026-01-01T00:00:00Z", - }, - ] - - original_flag = getattr( - litellm, "disable_vertex_batch_output_transformation", False - ) - try: - litellm.disable_vertex_batch_output_transformation = True - - result = await calculate_batch_cost_and_usage( - file_content_dictionary=raw_vertex_responses, - custom_llm_provider="vertex_ai", - model_name="gemini-2.0-flash-001", - ) - finally: - litellm.disable_vertex_batch_output_transformation = original_flag - - assert result.usage.prompt_tokens == 10 - assert result.usage.completion_tokens == 5 - assert result.usage.total_tokens == 15 - assert result.cost > 0, "raw Vertex shape should also produce non-zero cost" diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 004f07da431..de413a86521 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -634,25 +634,6 @@ def test_equal_modeled_usage_is_zero_under_equivalent_model_names() -> None: assert _savings("claude-opus-5", "anthropic/claude-opus-5", usage, usage) == 0.0 -def test_baseline_is_priced_under_its_own_provider(): - """Two providers can serve the same bare model name at different rates, so dropping - the provider prices the baseline against a vendor the operator never named. Here it - decides whether routing reads as a saving or a loss.""" - usage = Usage(prompt_tokens=100_000, completion_tokens=10_000, total_tokens=110_000) - azure = compute_autorouter_savings( - baseline_model="azure_ai/deepseek-r1", - selected_model="claude-haiku-4-5", - selected_provider="anthropic", - usage=usage, - ) - deepseek = compute_autorouter_savings( - baseline_model="deepseek/deepseek-r1", - selected_model="claude-haiku-4-5", - selected_provider="anthropic", - usage=usage, - ) - assert azure != pytest.approx(deepseek) - assert azure > 0 > deepseek def test_unresolvable_baseline_remains_unknown(): diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 16cbd109b24..e41c027d962 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -3887,179 +3887,7 @@ class TestSpendLogsPayload: } return mock_response - @pytest.mark.asyncio - async def test_spend_logs_payload_success_log_with_api_base(self, monkeypatch): - from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler - # Clear any env overrides that would change the recorded api_base - monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) - monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) - - litellm.callbacks = [_ProxyDBLogger(message_logging=False)] - # litellm._turn_on_debug() - - client = AsyncHTTPHandler() - - with ( - patch.object( - litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter, - "_insert_spend_log_to_db", - ) as mock_client, - patch.object(litellm.proxy.proxy_server, "prisma_client"), - patch.object(client, "post", side_effect=self.mock_anthropic_response), - ): - response = await litellm.acompletion( - model="claude-4-sonnet-20250514", - messages=[{"role": "user", "content": "Hello, world!"}], - metadata={"user_api_key_end_user_id": "test_user_1"}, - client=client, - ) - - assert response.choices[0].message.content == "Hi! My name is Claude." - - await _wait_for_mock_call(mock_client) - - kwargs = mock_client.call_args.kwargs - payload: SpendLogsPayload = kwargs["payload"] - expected_payload = SpendLogsPayload( - **{ - "request_id": "chatcmpl-34df56d5-4807-45c1-bb99-61e52586b802", - "call_type": "acompletion", - "api_key": "", - "cache_hit": "None", - "startTime": datetime.datetime( - 2025, 3, 24, 22, 2, 42, 975883, tzinfo=datetime.timezone.utc - ), - "endTime": datetime.datetime( - 2025, 3, 24, 22, 2, 42, 989132, tzinfo=datetime.timezone.utc - ), - "completionStartTime": datetime.datetime( - 2025, 3, 24, 22, 2, 42, 989132, tzinfo=datetime.timezone.utc - ), - "model": "claude-4-sonnet-20250514", - "user": "", - "team_id": "", - "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', - "cache_key": "Cache OFF", - "spend": 0.01383, - "total_tokens": 2598, - "prompt_tokens": 2095, - "completion_tokens": 503, - "request_tags": "[]", - "end_user": "test_user_1", - "api_base": "https://api.anthropic.com/v1/messages", - "model_group": "", - "model_id": "", - "requester_ip_address": None, - "custom_llm_provider": "anthropic", - "messages": "{}", - "response": "{}", - "proxy_server_request": "{}", - "status": "success", - "mcp_namespaced_tool_name": None, - "agent_id": None, - } - ) - - differences = _compare_nested_dicts( - payload, expected_payload, ignore_keys=ignored_keys - ) - if differences: - pytest.fail(f"Dictionary mismatch: {differences}") - - @pytest.mark.asyncio - async def test_spend_logs_payload_success_log_with_router(self, monkeypatch): - from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler - - # Clear any env overrides that would change the recorded api_base - monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) - monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) - - litellm.callbacks = [_ProxyDBLogger(message_logging=False)] - # litellm._turn_on_debug() - - client = AsyncHTTPHandler() - - router = Router( - model_list=[ - { - "model_name": "my-anthropic-model-group", - "litellm_params": { - "model": "claude-4-sonnet-20250514", - }, - "model_info": { - "id": "my-unique-model-id", - }, - } - ] - ) - - with ( - patch.object( - litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter, - "_insert_spend_log_to_db", - ) as mock_client, - patch.object(litellm.proxy.proxy_server, "prisma_client"), - patch.object(client, "post", side_effect=self.mock_anthropic_response), - ): - response = await router.acompletion( - model="my-anthropic-model-group", - messages=[{"role": "user", "content": "Hello, world!"}], - metadata={"user_api_key_end_user_id": "test_user_1"}, - client=client, - ) - - assert response.choices[0].message.content == "Hi! My name is Claude." - - await _wait_for_mock_call(mock_client) - - kwargs = mock_client.call_args.kwargs - payload: SpendLogsPayload = kwargs["payload"] - expected_payload = SpendLogsPayload( - **{ - "request_id": "chatcmpl-34df56d5-4807-45c1-bb99-61e52586b802", - "call_type": "acompletion", - "api_key": "", - "cache_hit": "None", - "startTime": datetime.datetime( - 2025, 3, 24, 22, 2, 42, 975883, tzinfo=datetime.timezone.utc - ), - "endTime": datetime.datetime( - 2025, 3, 24, 22, 2, 42, 989132, tzinfo=datetime.timezone.utc - ), - "completionStartTime": datetime.datetime( - 2025, 3, 24, 22, 2, 42, 989132, tzinfo=datetime.timezone.utc - ), - "model": "claude-4-sonnet-20250514", - "user": "", - "team_id": "", - "metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', - "cache_key": "Cache OFF", - "spend": 0.01383, - "total_tokens": 2598, - "prompt_tokens": 2095, - "completion_tokens": 503, - "request_tags": "[]", - "end_user": "test_user_1", - "api_base": "https://api.anthropic.com/v1/messages", - "model_group": "my-anthropic-model-group", - "model_id": "my-unique-model-id", - "requester_ip_address": None, - "custom_llm_provider": "anthropic", - "messages": "{}", - "response": "{}", - "proxy_server_request": "{}", - "status": "success", - "mcp_namespaced_tool_name": None, - "agent_id": None, - } - ) - - differences = _compare_nested_dicts( - payload, expected_payload, ignore_keys=ignored_keys - ) - if differences: - pytest.fail(f"Dictionary mismatch: {differences}") def _compare_nested_dicts( diff --git a/tests/test_litellm/responses/test_metadata_codex_callback.py b/tests/test_litellm/responses/test_metadata_codex_callback.py index f151f36be63..d5c411d63db 100644 --- a/tests/test_litellm/responses/test_metadata_codex_callback.py +++ b/tests/test_litellm/responses/test_metadata_codex_callback.py @@ -48,72 +48,6 @@ class MetadataCaptureCallback(CustomLogger): self.event.set() -@pytest.mark.asyncio -async def test_metadata_passed_to_custom_callback_codex_models(): - """ - Test that metadata passed to completion() is available in custom callback - when using codex models (responses API bridge path). - - Codex models have mode=responses and route through responses_api_bridge, - which passes litellm_metadata. The fix ensures this is preserved as - litellm_params.metadata for callback compatibility. - """ - from litellm.types.llms.openai import ResponsesAPIResponse - - mock_response = ResponsesAPIResponse.model_construct( - id="resp-test", - created_at=0, - output=[ - { - "type": "message", - "id": "msg-1", - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello!"}], - } - ], - object="response", - model="gpt-5.1-codex", - status="completed", - usage={ - "input_tokens": 5, - "output_tokens": 10, - "total_tokens": 15, - }, - ) - - test_metadata = {"foo": "bar", "trace_id": "test-123"} - callback = MetadataCaptureCallback() - original_callbacks = litellm.callbacks.copy() if litellm.callbacks else [] - litellm.callbacks = [callback] - - try: - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - new_callable=AsyncMock, - ) as mock_post: - mock_post.return_value = _make_mock_http_response( - mock_response.model_dump() - ) - # gpt-5.1-codex has mode=responses - routes through responses bridge - await litellm.acompletion( - model="gpt-5.1-codex", - messages=[{"role": "user", "content": "Hello"}], - metadata=test_metadata, - ) - - await asyncio.wait_for(callback.event.wait(), timeout=5.0) - - assert callback.captured_kwargs is not None, "Callback should have been invoked" - - litellm_params = callback.captured_kwargs.get("litellm_params", {}) - metadata = litellm_params.get("metadata") or {} - - assert "foo" in metadata, "metadata['foo'] should be accessible in callback" - assert metadata["foo"] == "bar" - assert metadata.get("trace_id") == "test-123" - finally: - litellm.callbacks = original_callbacks @pytest.mark.asyncio diff --git a/tests/test_litellm/rust_bridge/test_token_counter.py b/tests/test_litellm/rust_bridge/test_token_counter.py index 101fda629a7..3af7127a9b3 100644 --- a/tests/test_litellm/rust_bridge/test_token_counter.py +++ b/tests/test_litellm/rust_bridge/test_token_counter.py @@ -148,7 +148,6 @@ async def test_each_tokenizer_gets_its_own_cached_counter(fake_tokenizers: None) ("gpt-4o", "o200k_base"), ("gpt-4o-mini", "o200k_base"), ("gpt-4o-2024-08-06", "o200k_base"), - ("chatgpt-4o-latest", "o200k_base"), ("gpt-4.1", "o200k_base"), ("gpt-5", "o200k_base"), ("gpt-5-mini", "o200k_base"), diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index ecc01857f8c..e7ce9a74797 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -159,131 +159,8 @@ def test_cost_calculator_with_response_cost_in_additional_headers(): assert result == 1000 -def test_cost_calculator_with_usage(_local_model_cost_map, monkeypatch): - - usage = Usage( - prompt_tokens=120, - completion_tokens=100, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=10, - audio_tokens=90, - image_tokens=20, - ), - ) - mr = ModelResponse(usage=usage, model="gemini-2.0-flash-001") - - result = response_cost_calculator( - response_object=mr, - model="", - custom_llm_provider="vertex_ai", - call_type="acompletion", - optional_params={}, - cache_hit=None, - base_model=None, - ) - - model_info = litellm.model_cost["gemini-2.0-flash-001"] - - # Step 1: Test a model where input_cost_per_image_token is not set. - # In this case the calculation should use input_cost_per_token as fallback. - assert model_info.get("input_cost_per_image_token") is None, ( - "Test case expects that input_cost_per_image_token is not set" - ) - - expected_cost = ( - usage.prompt_tokens_details.audio_tokens * model_info["input_cost_per_audio_token"] - + usage.prompt_tokens_details.text_tokens * model_info["input_cost_per_token"] - + usage.prompt_tokens_details.image_tokens * model_info["input_cost_per_token"] - + usage.completion_tokens * model_info["output_cost_per_token"] - ) - - assert result == expected_cost, f"Got {result}, Expected {expected_cost}" - - # Step 2: Set input_cost_per_image_token. - # In this case the explicit cost information should be used. - temp_model_info_object = dict(model_info) - temp_model_info_object["input_cost_per_image_token"] = 0.5 - - monkeypatch.setattr( - litellm, - "model_cost", - {"gemini-2.0-flash-001": temp_model_info_object}, - ) - - # Invalidate caches after modifying litellm.model_cost - from litellm.utils import _invalidate_model_cost_lowercase_map - - _invalidate_model_cost_lowercase_map() - - result = response_cost_calculator( - response_object=mr, - model="", - custom_llm_provider="vertex_ai", - call_type="acompletion", - optional_params={}, - cache_hit=None, - base_model=None, - ) - - expected_cost = ( - usage.prompt_tokens_details.audio_tokens * temp_model_info_object["input_cost_per_audio_token"] - + usage.prompt_tokens_details.text_tokens * temp_model_info_object["input_cost_per_token"] - + usage.prompt_tokens_details.image_tokens * temp_model_info_object["input_cost_per_image_token"] - + usage.completion_tokens * temp_model_info_object["output_cost_per_token"] - ) - - assert result == expected_cost, f"Got {result}, Expected {expected_cost}" -def test_handle_realtime_stream_cost_calculation_stores_cost_breakdown(): - """Regression: realtime cost must populate logging_obj.cost_breakdown so the - spend logs / UI show input vs output cost (issue: cost_breakdown was None for - /v1/realtime even though a total spend was computed).""" - from datetime import datetime - - from litellm.litellm_core_utils.litellm_logging import Logging - - results: OpenAIRealtimeStreamList = [ - {"type": "session.created", "session": {"model": "gpt-4o-realtime-preview"}}, - { - "type": "response.done", - "response": { - "usage": { - "input_tokens": 100, - "output_tokens": 50, - "total_tokens": 150, - } - }, - }, - ] - combined_usage_object = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results( - results=results, - ) - - logging_obj = Logging( - model="gpt-4o-realtime-preview", - messages=[], - stream=False, - call_type="_arealtime", - start_time=datetime.now(), - litellm_call_id="realtime-cost-breakdown-test", - function_id="realtime-cost-breakdown-test", - ) - - total_cost = handle_realtime_stream_cost_calculation( - results=results, - combined_usage_object=combined_usage_object, - custom_llm_provider="openai", - litellm_model_name="gpt-4o-realtime-preview", - litellm_logging_obj=logging_obj, - ) - - assert total_cost > 0 - assert logging_obj.cost_breakdown is not None - assert logging_obj.cost_breakdown["input_cost"] > 0 - assert logging_obj.cost_breakdown["output_cost"] > 0 - assert abs(logging_obj.cost_breakdown["input_cost"] + logging_obj.cost_breakdown["output_cost"] - total_cost) < 1e-9 - assert abs(logging_obj.cost_breakdown["total_cost"] - total_cost) < 1e-9 def test_realtime_stream_combines_text_and_audio_token_details(): @@ -1124,126 +1001,6 @@ def test_bedrock_cost_calculator_comparison_with_without_cache(): print(f"Cost with cache: {cost_with_cache}") -def test_log_context_cost_calculation(): - """ - Test that log context cost calculation works correctly with tiered pricing. - - This test verifies that when using extended context (above 200k tokens), - the log context costs are calculated using the appropriate tiered rates. - """ - from litellm import completion_cost - from litellm.types.utils import ( - Choices, - Message, - ModelResponse, - PromptTokensDetailsWrapper, - Usage, - ) - - # Create a mock response with extended context usage - extended_context_response = ModelResponse( - id="test-extended-context-response", - created=1750733889, - model="claude-4-sonnet-20250514", - object="chat.completion", - system_fingerprint=None, - choices=[ - Choices( - finish_reason="stop", - index=0, - message=Message( - content="This is a test response for extended context cost calculation.", - role="assistant", - tool_calls=None, - function_call=None, - ), - ) - ], - usage=Usage( - total_tokens=350000, # Above 200k threshold - prompt_tokens=301000, # Above 200k threshold - completion_tokens=50000, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=300000, - cached_tokens=0, # No cache hits - audio_tokens=None, - image_tokens=None, - character_count=None, - video_length_seconds=None, - cache_creation_tokens=1000, - ), - completion_tokens_details=None, - _cache_creation_input_tokens=1000, # Some tokens added to cache - ), - ) - - # Calculate the cost using the extended context model - result = completion_cost( - completion_response=extended_context_response, - model="claude-4-sonnet-20250514", - custom_llm_provider="anthropic", - ) - - # Debug: Print the actual result - print(f"DEBUG: Actual cost result: ${result:.6f}") - - # Get model info to understand the pricing - from litellm import get_model_info - - model_info = get_model_info(model="claude-4-sonnet-20250514", custom_llm_provider="anthropic") - - # Calculate expected cost based on actual model pricing - input_cost_per_token = model_info.get("input_cost_per_token", 0) - output_cost_per_token = model_info.get("output_cost_per_token", 0) - cache_creation_cost_per_token = model_info.get("cache_creation_input_token_cost", 0) - - # Check if tiered pricing is applied - input_cost_above_200k = model_info.get("input_cost_per_token_above_200k_tokens", input_cost_per_token) - output_cost_above_200k = model_info.get("output_cost_per_token_above_200k_tokens", output_cost_per_token) - cache_creation_above_200k = model_info.get( - "cache_creation_input_token_cost_above_200k_tokens", - cache_creation_cost_per_token, - ) - - print(f"DEBUG: Base input cost per token: ${input_cost_per_token:.2e}") - print(f"DEBUG: Base output cost per token: ${output_cost_per_token:.2e}") - print(f"DEBUG: Base cache creation cost per token: ${cache_creation_cost_per_token:.2e}") - - # Handle tiered pricing - if not available, use base pricing - if input_cost_above_200k is not None: - print(f"DEBUG: Tiered input cost per token (>200k): ${input_cost_above_200k:.2e}") - else: - print("DEBUG: No tiered input pricing available, using base pricing") - input_cost_above_200k = input_cost_per_token - - if output_cost_above_200k is not None: - print(f"DEBUG: Tiered output cost per token (>200k): ${output_cost_above_200k:.2e}") - else: - print("DEBUG: No tiered output pricing available, using base pricing") - output_cost_above_200k = output_cost_per_token - - if cache_creation_above_200k is not None: - print(f"DEBUG: Tiered cache creation cost per token (>200k): ${cache_creation_above_200k:.2e}") - else: - print("DEBUG: No tiered cache creation pricing available, using base pricing") - cache_creation_above_200k = cache_creation_cost_per_token - - # Since we're above 200k tokens, we should use tiered pricing if available - expected_input_cost = 300000 * input_cost_above_200k - expected_output_cost = 50000 * output_cost_above_200k - expected_cache_cost = 1000 * cache_creation_above_200k - expected_total = expected_input_cost + expected_output_cost + expected_cache_cost - - print(f"DEBUG: Expected total: ${expected_total:.6f}") - - # Allow for small floating point differences - assert abs(result - expected_total) < 1e-6, f"Expected cost ${expected_total:.6f}, but got ${result:.6f}" - - print(f"✓ Log context cost calculation with tiered pricing is correct: ${result:.6f}") - print(f" - Input tokens (300k): ${expected_input_cost:.6f}") - print(f" - Output tokens (50k): ${expected_output_cost:.6f}") - print(f" - Cache creation (1k): ${expected_cache_cost:.6f}") - print(f" - Total: ${result:.6f}") def test_gemini_25_explicit_caching_cost_direct_usage(): @@ -1814,56 +1571,6 @@ def test_cost_margin_with_discount(monkeypatch): print(f" - Expected: ${expected_cost:.6f}") -def test_azure_image_generation_cost_calculator(): - from unittest.mock import MagicMock - - from litellm.types.utils import ( - ImageObject, - ImageResponse, - ImageUsage, - ImageUsageInputTokensDetails, - ) - - response_cost_calculator_kwargs = { - "response_object": ImageResponse( - created=1761785270, - background=None, - data=[ - ImageObject( - b64_json=None, - revised_prompt="A futuristic, techno-inspired green duck wearing cool modern sunglasses. The duck has a sleek, metallic appearance with glowing neon green accents, standing on a high-tech urban background with holographic billboards and illuminated city lights in the distance. The duck's feathers have a glossy, high-tech sheen, resembling a robotic design but still maintaining its avian features. The scene has a vibrant, cyberpunk aesthetic with a neon color palette.", - url="test-azure-blob-url-with-sas-token", - ) - ], - output_format=None, - quality="hd", - size=None, - usage=ImageUsage( - input_tokens=0, - input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=0), - output_tokens=0, - total_tokens=0, - ), - ), - "model": "azure/dall-e-3", - "cache_hit": False, - "custom_llm_provider": "azure", - "base_model": "azure/dall-e-3", - "call_type": "aimage_generation", - "optional_params": {}, - "custom_pricing": False, - "prompt": "", - "standard_built_in_tools_params": { - "web_search_options": None, - "file_search": None, - }, - "router_model_id": "6738c432ffc9b733597c6b86613ca20dc5f49bde591fd3d03e7cd6aa25bb241e", - "litellm_logging_obj": MagicMock(), - "service_tier": None, - } - - cost = response_cost_calculator(**response_cost_calculator_kwargs) - assert cost > 0.079 def test_completion_cost_extracts_service_tier_from_response(_local_model_cost_map): @@ -2616,87 +2323,6 @@ def test_gemini_without_cache_tokens_details(): print("✅ Gemini without cacheTokensDetails works correctly") -def test_gemini_implicit_caching_cost_calculation(): - """ - Test for Issue #16341: Gemini implicit cached tokens not counted in spend log - - When Gemini uses implicit caching, it returns cachedContentTokenCount but NOT - cacheTokensDetails. In this case, we should subtract cachedContentTokenCount - from text_tokens to correctly calculate costs. - - See: https://github.com/BerriAI/litellm/issues/16341 - """ - from litellm import completion_cost - from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig, - ) - from litellm.types.utils import Choices, Message, ModelResponse - - # Simulate Gemini response with implicit caching (cachedContentTokenCount only) - completion_response = { - "usageMetadata": { - "promptTokenCount": 10000, - "candidatesTokenCount": 5, - "totalTokenCount": 10005, - "cachedContentTokenCount": 8000, # Implicit caching - no cacheTokensDetails - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 10000}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 5}], - } - } - - usage = VertexGeminiConfig._calculate_usage(completion_response) - - # Verify parsing - assert usage.cache_read_input_tokens == 8000, ( - f"cache_read_input_tokens should be 8000, got {usage.cache_read_input_tokens}" - ) - assert usage.prompt_tokens_details.cached_tokens == 8000, ( - f"cached_tokens should be 8000, got {usage.prompt_tokens_details.cached_tokens}" - ) - - # CRITICAL: text_tokens should be (10000 - 8000) = 2000, NOT 10000 - # This is the fix for issue #16341 - assert usage.prompt_tokens_details.text_tokens == 2000, ( - f"text_tokens should be 2000 (10000 - 8000), got {usage.prompt_tokens_details.text_tokens}" - ) - - # Verify cost calculation uses cached token pricing - response = ModelResponse( - id="mock-id", - model="gemini-2.0-flash", - choices=[ - Choices( - index=0, - message=Message(role="assistant", content="Hello!"), - finish_reason="stop", - ) - ], - usage=usage, - ) - - cost = completion_cost( - completion_response=response, - model="gemini-2.0-flash", - custom_llm_provider="gemini", - ) - - # Get model pricing for verification - import litellm - - model_info = litellm.get_model_info("gemini/gemini-2.0-flash") - input_cost = model_info.get("input_cost_per_token", 0) - cache_read_cost = model_info.get("cache_read_input_token_cost", input_cost) - output_cost = model_info.get("output_cost_per_token", 0) - - # Expected cost: (2000 * input) + (8000 * cache_read) + (5 * output) - expected_cost = (2000 * input_cost) + (8000 * cache_read_cost) + (5 * output_cost) - - assert abs(cost - expected_cost) < 1e-9, ( - f"Cost calculation is wrong. Got ${cost:.6f}, expected ${expected_cost:.6f}. " - f"Cached tokens may not be using reduced pricing." - ) - - print("✅ Issue #16341 fix verified: Gemini implicit caching cost calculated correctly") def test_additional_costs_only_for_azure_ai(_local_model_cost_map): diff --git a/tests/test_litellm/test_count_tokens_public_api.py b/tests/test_litellm/test_count_tokens_public_api.py index 2918d0aa522..84121dd63e4 100644 --- a/tests/test_litellm/test_count_tokens_public_api.py +++ b/tests/test_litellm/test_count_tokens_public_api.py @@ -157,21 +157,3 @@ def test_acount_tokens_no_api_key_falls_back(monkeypatch): assert result.tokenizer_type == "local_tokenizer" -async def test_acount_tokens_local_fallback_counts_off_the_event_loop(): - from tests.large_text import text - from tests.test_litellm.litellm_core_utils.event_loop_lag import ( - assert_loop_stayed_free, - timed_with_loop_lags, - warm_tokenizer, - ) - - model = "together_ai/meta-llama/Llama-3-8b-chat-hf" - warm_tokenizer(model) - - result, took, lags = await timed_with_loop_lags( - lambda: litellm.acount_tokens(model=model, messages=[{"role": "user", "content": text * 100}]) - ) - - assert result.tokenizer_type == "local_tokenizer" - assert result.total_tokens > 100_000 - assert_loop_stayed_free(took, lags) diff --git a/tests/test_litellm/test_gpt_image_cost_calculator.py b/tests/test_litellm/test_gpt_image_cost_calculator.py index 0f471430daf..d026285f9ae 100644 --- a/tests/test_litellm/test_gpt_image_cost_calculator.py +++ b/tests/test_litellm/test_gpt_image_cost_calculator.py @@ -90,27 +90,6 @@ class TestGPTImageCostCalculator: class TestGPTImageCostRouting: """Test that gpt-image models are properly routed to the token-based calculator""" - def test_openai_dalle_routes_to_pixel_calculator(self): - """Test that OpenAI DALL-E still routes to pixel-based calculator""" - from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils - - image_response = ImageResponse( - created=1234567890, - data=[ImageObject(url="http://example.com/image.jpg")], - ) - image_response.size = "1024x1024" - image_response.quality = "standard" - - cost = CostCalculatorUtils.route_image_generation_cost_calculator( - model="dall-e-3", - completion_response=image_response, - custom_llm_provider="openai", - size="1024x1024", - quality="standard", - n=1, - ) - - assert cost >= 0 class TestGPTImage15OutputImageTokens: diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index f5344c01f3a..411377f29cf 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -850,36 +850,6 @@ async def test_arouter_async_get_healthy_deployments(): assert result[0]["litellm_params"]["model"] == "gpt-3.5-turbo" -@pytest.mark.asyncio -@patch("litellm.amoderation") -async def test_arouter_amoderation_with_credential_name(mock_amoderation): - """ - Test that router.amoderation passes litellm_credential_name to the underlying litellm.amoderation call - """ - mock_amoderation.return_value = AsyncMock() - - router = litellm.Router( - model_list=[ - { - "model_name": "text-moderation-stable", - "litellm_params": { - "model": "text-moderation-stable", - "litellm_credential_name": "my-custom-auth", - }, - }, - ], - ) - - await router.amoderation(input="I love everyone!", model="text-moderation-stable") - - mock_amoderation.assert_called_once() - call_kwargs = mock_amoderation.call_args[1] # Get the kwargs of the call - print( - "call kwargs for router.amoderation=", - json.dumps(call_kwargs, indent=4, default=str), - ) - assert call_kwargs["litellm_credential_name"] == "my-custom-auth" - assert call_kwargs["model"] == "text-moderation-stable" def test_arouter_test_team_model(): diff --git a/tests/test_litellm/test_together_ai_model_metadata.py b/tests/test_litellm/test_together_ai_model_metadata.py index 7176ba4f219..dccba3d66f9 100644 --- a/tests/test_litellm/test_together_ai_model_metadata.py +++ b/tests/test_litellm/test_together_ai_model_metadata.py @@ -95,15 +95,6 @@ def _successor(info: dict[str, object]) -> str | None: return successor if isinstance(successor, str) else None -def test_together_successor_metadata_points_at_known_models(cost_map: CostMap): - successors = { - model: successor - for model, info in cost_map.items() - if model.startswith("together_ai/") and (successor := _successor(info)) is not None - } - assert len(successors) >= 10 - for model, successor in successors.items(): - assert successor in cost_map, f"{model} names successor {successor} that is not in the map" def test_together_backup_cost_map_in_sync(cost_map: CostMap): diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index ead805eed42..07982f51153 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1465,12 +1465,6 @@ class TestProxyFunctionCalling: ("gemini/gemini-2.5-pro", "litellm_proxy/gemini/gemini-2.5-pro", True), ("gemini/gemini-2.5-flash", "litellm_proxy/gemini/gemini-2.5-flash", True), # Groq models (mixed support) - ("groq/gemma-7b-it", "litellm_proxy/groq/gemma-7b-it", True), - ( - "groq/llama-3.3-70b-versatile", - "litellm_proxy/groq/llama-3.3-70b-versatile", - True, - ), # Cohere models (generally don't support function calling) ("command-nightly", "litellm_proxy/command-nightly", False), ], diff --git a/tests/test_litellm/test_xai_responses_auto_routing.py b/tests/test_litellm/test_xai_responses_auto_routing.py index d405ea1e6c6..c31f599dd79 100644 --- a/tests/test_litellm/test_xai_responses_auto_routing.py +++ b/tests/test_litellm/test_xai_responses_auto_routing.py @@ -46,34 +46,6 @@ class TestXAIResponsesAutoRouting: assert model_info.get("mode") != "responses" assert updated_model == model - def test_responses_api_bridge_check_with_tools(self): - """Test that with tools, xAI automatically routes to Responses API""" - model = "grok-3" - custom_llm_provider = "xai" - tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the weather", - "parameters": { - "type": "object", - "properties": {"location": {"type": "string"}}, - }, - }, - } - ] - web_search_options = None - - model_info, updated_model = responses_api_bridge_check( - model=model, - custom_llm_provider=custom_llm_provider, - web_search_options=web_search_options, - ) - - # Should auto-route to responses mode when tools are present - assert model_info.get("mode") == "chat" - assert updated_model == model def test_responses_api_bridge_check_with_empty_tools(self): """Test that with empty tools list, xAI does not route to Responses API""" @@ -134,57 +106,8 @@ class TestXAIResponsesAutoRouting: assert model_info.get("mode") == "responses" assert updated_model == "grok-3" # prefix removed - def test_responses_api_bridge_check_with_code_interpreter_tool(self): - """Test auto-routing with code_interpreter tool""" - model = "grok-3" - custom_llm_provider = "xai" - tools = [{"type": "code_interpreter"}] - web_search_options = None - model_info, updated_model = responses_api_bridge_check( - model=model, - custom_llm_provider=custom_llm_provider, - web_search_options=web_search_options, - ) - # Should auto-route with code_interpreter tool - assert model_info.get("mode") == "chat" - assert updated_model == model - def test_responses_api_bridge_check_with_web_search_tool(self): - """Test auto-routing with web_search tool""" - model = "grok-4" - custom_llm_provider = "xai" - tools = [ - {"type": "web_search", "filters": {"allowed_domains": ["wikipedia.org"]}} - ] - web_search_options = None - - model_info, updated_model = responses_api_bridge_check( - model=model, - custom_llm_provider=custom_llm_provider, - web_search_options=web_search_options, - ) - - # Should auto-route with web_search tool - assert model_info.get("mode") == "chat" - assert updated_model == model - - def test_responses_api_bridge_check_with_x_search_tool(self): - """Test auto-routing with x_search tool""" - model = "grok-4" - custom_llm_provider = "xai" - tools = [{"type": "x_search", "allowed_x_handles": ["@elonmusk"]}] - web_search_options = None - - model_info, updated_model = responses_api_bridge_check( - model=model, - custom_llm_provider=custom_llm_provider, - web_search_options=web_search_options, - ) - - # Should auto-route with x_search tool - assert model_info.get("mode") == "chat" - assert updated_model == model def test_responses_api_bridge_check_with_web_search_options(self): """Test auto-routing with web_search_options""" diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index e4a33d5772c..47609261a25 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -157,27 +157,6 @@ def test_foundry_gpt_6_astra_keeps_sampling_params_when_reasoning_effort_is_none assert optional_params == {"reasoning_effort": "none", "temperature": 0.2, "top_p": 0.9} -def test_a_gpt_5_name_without_a_foundry_row_keeps_reading_its_own_entry( - monkeypatch: pytest.MonkeyPatch, _local_model_cost_map -): - """Most gpt-5-family names have no azure_ai/ row. Reading an azure_ai/ key for those finds - nothing, and an openai.azure.com base sends the name down the azure provider, which has no key - for it either, so every effort answer would silently fall back to false and take temperature, - top_p and logprobs down with it.""" - monkeypatch.setenv("AZURE_AI_API_BASE", "https://example-resource.openai.azure.com") - monkeypatch.setenv("AZURE_AI_API_KEY", "placeholder") - - optional_params = litellm.utils.get_optional_params( - model="gpt-5.1-chat-latest", - custom_llm_provider="azure_ai", - temperature=0.2, - top_p=0.9, - logprobs=True, - ) - - assert optional_params["temperature"] == 0.2 - assert optional_params["top_p"] == 0.9 - assert optional_params["logprobs"] is True def test_azure_ai_grok_stop_parameter_handling(): diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 6815f00267c..3f740edf834 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1695,21 +1695,6 @@ def test_nim_vllm_extras_translated_end_to_end_in_request_body(): assert request_body["top_k"] == 40 -def test_in_schema_unsupported_params_still_raise(): - with pytest.raises(litellm.UnsupportedParamsError): - litellm.get_optional_params( - model="accounts/fireworks/models/llama-v3-70b-instruct", - custom_llm_provider="fireworks_ai", - drop_params=False, - store=True, - ) - optional_params = litellm.get_optional_params( - model="accounts/fireworks/models/llama-v3-70b-instruct", - custom_llm_provider="fireworks_ai", - drop_params=True, - store=True, - ) - assert "store" not in optional_params def test_streaming_preserves_selected_model_for_private_accounting(): diff --git a/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py b/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py index c39affc18a8..4e7a2931cda 100644 --- a/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py +++ b/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py @@ -715,7 +715,7 @@ class TestMoonshotReasoningEffort: def force_local_model_cost(self, monkeypatch): monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map()) - @pytest.mark.parametrize("model", ["kimi-k3", "kimi-k2.5", "kimi-k2.6", "kimi-k2-thinking"]) + @pytest.mark.parametrize("model", ["kimi-k3", "kimi-k2.5", "kimi-k2.6"]) def test_reasoning_model_supports_reasoning_effort(self, model): assert "reasoning_effort" in MoonshotChatConfig().get_supported_openai_params(model) From c37fe46534519c9be7f0a80a0f6786d154e6bbd7 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:20:18 -0700 Subject: [PATCH 48/48] test(vcr): guard leaked cassette patches and make injected-transport embedding tests immune (#42542) * test(vcr): guard leaked cassette patches and make injected-transport embedding tests immune * test(vcr): derive the leak guard's patch points from vcrpy's own reset list * test(vcr): share CapturingTransport and switch the encoding_format embedding test to it --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/_vcr_conftest_common.py | 245 ++++++++++-------- tests/capturing_transport.py | 25 ++ tests/llm_translation/conftest.py | 30 ++- .../test_litellm_proxy_provider.py | 98 +++---- tests/llm_translation/test_nvidia_nim.py | 32 +-- tests/llm_translation/test_vcr_leak_guard.py | 72 +++++ tests/local_testing/conftest.py | 1 - tests/local_testing/test_embedding.py | 27 +- 8 files changed, 331 insertions(+), 199 deletions(-) create mode 100644 tests/capturing_transport.py create mode 100644 tests/llm_translation/test_vcr_leak_guard.py diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 4d5a73779ea..ab046674eb6 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -8,6 +8,7 @@ from __future__ import annotations import ast import atexit import hashlib +import inspect import json import os import re @@ -15,10 +16,18 @@ import socket import sys import threading from collections import defaultdict -from typing import Iterable +from collections.abc import Iterable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final +from unittest import mock +import aiohttp import pytest +import vcr import vcr.matchers as _vcr_matchers +import vcr.patch as _vcr_patch from tests._vcr_redis_persister import ( MAX_EPISODES_PER_CASSETTE, @@ -127,9 +136,7 @@ def emit_vcr_diagnostic_log(terminalreporter) -> None: with open(path, "r", encoding="utf-8") as fh: content = fh.read() except OSError as exc: - read_errors.append( - f" [failed to read {name}: {type(exc).__name__}: {exc}]" - ) + read_errors.append(f" [failed to read {name}: {type(exc).__name__}: {exc}]") continue for line in content.splitlines(): if not line.strip(): @@ -142,9 +149,7 @@ def emit_vcr_diagnostic_log(terminalreporter) -> None: return terminalreporter.write_sep("=", "VCR DIAGNOSTIC LOG", bold=True) - terminalreporter.write_line( - f" source dir: {directory} (deduplicated; full log archived as a CI artifact)" - ) + terminalreporter.write_line(f" source dir: {directory} (deduplicated; full log archived as a CI artifact)") for line in read_errors: terminalreporter.write_line(line) @@ -235,9 +240,7 @@ def pin_httpx_multipart_boundary(monkeypatch) -> None: boundary = VCR_FIXED_MULTIPART_BOUNDARY.encode("ascii") return _original_init(self, data=data, files=files, boundary=boundary, **kwargs) - monkeypatch.setattr( - _httpx_multipart.MultipartStream, "__init__", _init_with_fixed_boundary - ) + monkeypatch.setattr(_httpx_multipart.MultipartStream, "__init__", _init_with_fixed_boundary) @pytest.fixture(scope="session", autouse=True) @@ -270,11 +273,7 @@ def _replace_b64_json_in_place(obj) -> bool: changed = False if isinstance(obj, dict): for key, value in obj.items(): - if ( - key == "b64_json" - and isinstance(value, str) - and len(value) > len(VCR_IMAGE_B64_PLACEHOLDER) - ): + if key == "b64_json" and isinstance(value, str) and len(value) > len(VCR_IMAGE_B64_PLACEHOLDER): obj[key] = VCR_IMAGE_B64_PLACEHOLDER changed = True elif _replace_b64_json_in_place(value): @@ -296,16 +295,12 @@ def _strip_image_b64_payloads(response): preserves all those checks while shrinking cassettes by ~99%. """ if not isinstance(response, dict): - vcr_diag_write_line( - f"[vcr-strip-b64] response is {type(response).__name__!r}, not " - "dict; skipping b64 scrub" - ) + vcr_diag_write_line(f"[vcr-strip-b64] response is {type(response).__name__!r}, not dict; skipping b64 scrub") return response body = response.get("body") if not isinstance(body, dict): vcr_diag_write_line( - f"[vcr-strip-b64] response['body'] is {type(body).__name__!r}, " - "not dict; skipping b64 scrub" + f"[vcr-strip-b64] response['body'] is {type(body).__name__!r}, not dict; skipping b64 scrub" ) return response raw = body.get("string") @@ -316,10 +311,7 @@ def _strip_image_b64_payloads(response): try: text = bytes(raw).decode("utf-8") except UnicodeDecodeError: - vcr_diag_write_line( - "[vcr-strip-b64] response body bytes are not valid UTF-8; " - "skipping b64 scrub" - ) + vcr_diag_write_line("[vcr-strip-b64] response body bytes are not valid UTF-8; skipping b64 scrub") return response was_bytes = True elif isinstance(raw, str): @@ -327,8 +319,7 @@ def _strip_image_b64_payloads(response): was_bytes = False else: vcr_diag_write_line( - f"[vcr-strip-b64] response['body']['string'] is " - f"{type(raw).__name__!r}, not bytes/str; skipping b64 scrub" + f"[vcr-strip-b64] response['body']['string'] is {type(raw).__name__!r}, not bytes/str; skipping b64 scrub" ) return response @@ -349,9 +340,7 @@ def _strip_image_b64_payloads(response): for key in list(headers): if str(key).lower() == "content-length": value = headers[key] - headers[key] = ( - [new_len_value] if isinstance(value, list) else new_len_value - ) + headers[key] = [new_len_value] if isinstance(value, list) else new_len_value return response @@ -409,15 +398,11 @@ def _canonical_body(request) -> tuple[bytes, str]: # selected. This mirrors the existing SigV4 / multipart-boundary / b64-image # normalizations already in this module, and means the already-bloated # cassettes start replaying immediately without a flush + re-record. -_VCR_UUID_RE = re.compile( - rb"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" -) +_VCR_UUID_RE = re.compile(rb"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") _VCR_LITELLM_BATCH_JOB_RE = re.compile(rb"litellm-batch-[0-9a-fA-F]{8}") # ISO-8601 timestamps, e.g. ``2026-05-25T03:40:37.262045Z`` / # ``2026-05-25T03:40:37+00:00``. -_VCR_ISO_TS_RE = re.compile( - rb"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?" -) +_VCR_ISO_TS_RE = re.compile(rb"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?") # Unix epoch as 13-digit milliseconds, then 10-digit ``time.time()`` float, # then 10-digit integer seconds. Anchored to ``1`` + 9/12 digits, which keeps # them inside the 2001-2033 / 2001-2033 epoch windows and avoids matching @@ -639,10 +624,7 @@ def _should_drop_telemetry_record(request) -> bool: return False if not _is_telemetry_request(request): return False - if ( - _is_telemetry_export_request(request) - and not _current_test_replays_telemetry_export() - ): + if _is_telemetry_export_request(request) and not _current_test_replays_telemetry_export(): return True return not _current_test_records_telemetry() @@ -767,9 +749,7 @@ def _iter_header_values(headers, name: str): yield value -_AWS_SIGV4_CREDENTIAL_RE = re.compile( - r"AWS4-HMAC-SHA256\s+Credential=([^/\s,]+)/", re.IGNORECASE -) +_AWS_SIGV4_CREDENTIAL_RE = re.compile(r"AWS4-HMAC-SHA256\s+Credential=([^/\s,]+)/", re.IGNORECASE) # Google OAuth2 access tokens always start with ``ya29.`` regardless of how # they were minted (service account, metadata server, impersonation). @@ -891,9 +871,7 @@ def _normalize_multipart_boundary(request) -> None: return try: - headers[content_type_key] = content_type_value.replace( - match.group(0), fixed_param - ) + headers[content_type_key] = content_type_value.replace(match.group(0), fixed_param) except (TypeError, AttributeError): return @@ -985,8 +963,7 @@ def _materialize_iterable_body(request) -> None: uri = getattr(request, "uri", getattr(request, "url", "?")) first_type = type(chunks[0]).__name__ if chunks else "empty" vcr_diag_write_line( - f"[vcr-materialize] FALLBACK: {method} {uri} chunk type " - f"{first_type!r} not coerced to bytes; storing b''" + f"[vcr-materialize] FALLBACK: {method} {uri} chunk type {first_type!r} not coerced to bytes; storing b''" ) out = b"" @@ -1026,9 +1003,7 @@ def _key_fingerprint_matcher(r1, r2) -> None: return def _fp(req): - for value in _iter_header_values( - getattr(req, "headers", None), KEY_FINGERPRINT_HEADER - ): + for value in _iter_header_values(getattr(req, "headers", None), KEY_FINGERPRINT_HEADER): if value is None: continue return value if isinstance(value, str) else str(value) @@ -1159,13 +1134,11 @@ def _print_atexit_banner() -> None: _emit("VCR CASSETTE CACHE DEGRADED") if save_failures: _emit( - f" {save_failures} cassette save failure(s); last error: " - f"{health.get('save_failure_last_error', '')}" + f" {save_failures} cassette save failure(s); last error: {health.get('save_failure_last_error', '')}" ) if load_failures: _emit( - f" {load_failures} cassette load failure(s); last error: " - f"{health.get('load_failure_last_error', '')}" + f" {load_failures} cassette load failure(s); last error: {health.get('load_failure_last_error', '')}" ) if snapshot: _emit(_format_capacity_line(snapshot)) @@ -1276,11 +1249,7 @@ class _RespxUsageVisitor(ast.NodeVisitor): if isinstance(dec, ast.Call): dec = dec.func if isinstance(dec, ast.Attribute): - return ( - isinstance(dec.value, ast.Name) - and dec.value.id == "respx" - and dec.attr == "mock" - ) + return isinstance(dec.value, ast.Name) and dec.value.id == "respx" and dec.attr == "mock" return False def _is_pytest_mark_respx(self, dec: ast.expr) -> bool: @@ -1307,9 +1276,7 @@ class _RespxUsageVisitor(ast.NodeVisitor): # ``def test_foo(respx_mock): ...`` — pytest supplies the fixture # whenever the parameter name appears, regardless of marker. all_args = ( - list(args.args) - + list(args.kwonlyargs) - + (list(args.posonlyargs) if hasattr(args, "posonlyargs") else []) + list(args.args) + list(args.kwonlyargs) + (list(args.posonlyargs) if hasattr(args, "posonlyargs") else []) ) for a in all_args: if a.arg == "respx_mock": @@ -1566,9 +1533,7 @@ def _emit_outcome_payload( }, ) ) - node.user_properties.append( - (_USER_PROP_RECORDED_BY, os.environ.get("PYTEST_XDIST_WORKER", "")) - ) + node.user_properties.append((_USER_PROP_RECORDED_BY, os.environ.get("PYTEST_XDIST_WORKER", ""))) def aggregate_report_outcome(report) -> None: @@ -1616,9 +1581,7 @@ def aggregate_report_outcome(report) -> None: if verdict == VERDICT_MISS_OVERFLOW: _session_stats["overflow_tests"].append(nodeid) elif verdict == VERDICT_UNMARKED_LIVE_CALL: - _session_stats["unmarked_live_call_tests"].append( - (nodeid, list(outcome.get("live_call_hosts") or [])) - ) + _session_stats["unmarked_live_call_tests"].append((nodeid, list(outcome.get("live_call_hosts") or []))) skip_reason = outcome.get("skip_reason") if skip_reason: @@ -1635,9 +1598,7 @@ def session_stats_snapshot() -> dict: "overflow_tests": list(_session_stats["overflow_tests"]), "unmarked_live_call_tests": list(_session_stats["unmarked_live_call_tests"]), "skip_reason_counts": dict(_session_stats["skip_reason_counts"]), - "skip_reason_examples": { - k: list(v) for k, v in _session_stats["skip_reason_examples"].items() - }, + "skip_reason_examples": {k: list(v) for k, v in _session_stats["skip_reason_examples"].items()}, } @@ -1810,9 +1771,7 @@ def record_vcr_outcome(request, vcr) -> None: # Cassette is None ⇒ test wasn't VCR-marked. Honor the skip reason # we tagged at collection time, and pull live-call hosts captured by # the socket probe (if any). - skip_reason = getattr( - request.node, VCR_SKIP_REASON_USER_ATTR, SKIP_REASON_FILE_OPT_OUT - ) + skip_reason = getattr(request.node, VCR_SKIP_REASON_USER_ATTR, SKIP_REASON_FILE_OPT_OUT) _session_stats["skip_reason_counts"][skip_reason] += 1 hosts = getattr(request.node, _LIVE_CALL_BUFFER_KEY, []) or [] @@ -1837,9 +1796,7 @@ def record_vcr_outcome(request, vcr) -> None: live_call_hosts=hosts, ) if vcr_outcome_logging_enabled(): - request.node.user_properties.append( - (_USER_PROP_VERDICT_LINE, _format_verdict_line(verdict, None, extra)) - ) + request.node.user_properties.append((_USER_PROP_VERDICT_LINE, _format_verdict_line(verdict, None, extra))) def install_live_call_probe(request, vcr) -> None: @@ -1858,9 +1815,7 @@ def install_live_call_probe(request, vcr) -> None: # Track the current test for telemetry-leak suppression (applies to every # test, VCR-marked or not). See ``_should_drop_telemetry_record``. global _current_test_nodeid - _current_test_nodeid = str( - getattr(getattr(request, "node", None), "nodeid", "") or "" - ) + _current_test_nodeid = str(getattr(getattr(request, "node", None), "nodeid", "") or "") if vcr is not None or vcr_disabled(): return None probe = _LiveCallProbe() @@ -1876,10 +1831,7 @@ def _format_capacity_line(snapshot: dict) -> str: pct = float(snapshot.get("used_pct", 0.0) or 0.0) used_mb = used / (1024 * 1024) cap_mb = cap / (1024 * 1024) - return ( - f" Cassette Redis usage: {used_mb:.1f} MiB / {cap_mb:.1f} MiB " - f"({pct:.1f}% of maxmemory)" - ) + return f" Cassette Redis usage: {used_mb:.1f} MiB / {cap_mb:.1f} MiB ({pct:.1f}% of maxmemory)" def emit_vcr_classification_summary(terminalreporter) -> None: @@ -1940,14 +1892,10 @@ def emit_vcr_classification_summary(terminalreporter) -> None: total_leaks = sum(leak_counts.values()) terminalreporter.write_sep("-", "VCR COST LEAK CHECK", bold=True) if total_leaks: - rendered = ", ".join( - f"{verdict}={count}" for verdict, count in leak_counts.items() if count - ) + rendered = ", ".join(f"{verdict}={count}" for verdict, count in leak_counts.items() if count) terminalreporter.write_line(f" FAIL: {rendered}") else: - terminalreporter.write_line( - " PASS: no overflow, partial, not-persisted, or unmarked live-call verdicts" - ) + terminalreporter.write_line(" PASS: no overflow, partial, not-persisted, or unmarked live-call verdicts") overflow = snapshot["overflow_tests"] if overflow: @@ -2007,18 +1955,14 @@ def emit_cassette_cache_session_banner(terminalreporter) -> None: snapshot = cassette_cache_capacity_snapshot() if save_failures or load_failures: - terminalreporter.write_sep( - "=", "VCR CASSETTE CACHE DEGRADED", red=True, bold=True - ) + terminalreporter.write_sep("=", "VCR CASSETTE CACHE DEGRADED", red=True, bold=True) if save_failures: terminalreporter.write_line( - f" {save_failures} cassette save failure(s); last error: " - f"{health.get('save_failure_last_error', '')}" + f" {save_failures} cassette save failure(s); last error: {health.get('save_failure_last_error', '')}" ) if load_failures: terminalreporter.write_line( - f" {load_failures} cassette load failure(s); last error: " - f"{health.get('load_failure_last_error', '')}" + f" {load_failures} cassette load failure(s); last error: {health.get('load_failure_last_error', '')}" ) terminalreporter.write_line( " Tests still passed because cassette persistence is best-effort, " @@ -2031,9 +1975,7 @@ def emit_cassette_cache_session_banner(terminalreporter) -> None: return if snapshot and snapshot["used_pct"] >= CASSETTE_CACHE_HIGH_WATER_FRACTION * 100: - terminalreporter.write_sep( - "=", "VCR CASSETTE CACHE NEAR CAPACITY", yellow=True, bold=True - ) + terminalreporter.write_sep("=", "VCR CASSETTE CACHE NEAR CAPACITY", yellow=True, bold=True) terminalreporter.write_line(_format_capacity_line(snapshot)) terminalreporter.write_line( " No save failures yet, but Redis is approaching maxmemory. " @@ -2082,13 +2024,104 @@ class VerboseReporterState: if reporter is None: return verdict = next( - ( - v - for k, v in (report.user_properties or []) - if k == _USER_PROP_VERDICT_LINE - ), + (v for k, v in (report.user_properties or []) if k == _USER_PROP_VERDICT_LINE), None, ) if not verdict: return reporter.write_line(f"{verdict} :: {report.nodeid}") + + +@dataclass(frozen=True, slots=True) +class VcrPatchPoint: + owner: object + attribute: str + original: object + + @property + def name(self) -> str: + return f"{_patch_owner_name(self.owner)}.{self.attribute}" + + def current(self) -> object: + current: Final[object] = getattr(self.owner, self.attribute) + return current + + def is_patched(self) -> bool: + return self.current() is not self.original + + def restore(self) -> None: + setattr(self.owner, self.attribute, self.original) + + +def _patch_owner_name(owner: object) -> str: + if inspect.isclass(owner): + return f"{owner.__module__}.{owner.__qualname__}" + if inspect.ismodule(owner): + return owner.__name__ + return repr(owner) + + +def _vcr_patch_point(patcher: mock._patch[object]) -> VcrPatchPoint: + owner: Final[object] = patcher.getter() + return VcrPatchPoint(owner=owner, attribute=patcher.attribute, original=patcher.new) + + +_VCR_PATCH_POINTS: Final = ( + *(_vcr_patch_point(patcher) for patcher in _vcr_patch.reset_patchers()), + VcrPatchPoint(aiohttp.ClientSession, "_request", _vcr_patch._AiohttpClientSessionRequest), +) + + +@dataclass(frozen=True, slots=True) +class VcrPatchLeak: + patch_points: tuple[str, ...] + cassette_paths: tuple[str, ...] + + +def _cassette_paths_wrapped_into(fn: object) -> tuple[str, ...]: + if not inspect.isfunction(fn): + return () + cassette: Final = inspect.getclosurevars(fn).nonlocals.get("cassette") + own: Final = (str(cassette._path),) if isinstance(cassette, vcr.cassette.Cassette) else () + return own + _cassette_paths_wrapped_into(getattr(fn, "__wrapped__", None)) + + +def detect_vcr_patch_leak() -> VcrPatchLeak | None: + leaked: Final = tuple(point for point in _VCR_PATCH_POINTS if point.is_patched()) + if not leaked: + return None + return VcrPatchLeak( + patch_points=tuple(point.name for point in leaked), + cassette_paths=tuple( + dict.fromkeys(path for point in leaked for path in _cassette_paths_wrapped_into(point.current())) + ), + ) + + +def restore_vcr_patch_points() -> None: + for point in _VCR_PATCH_POINTS: + point.restore() + + +def guard_vcr_patch_points(item: pytest.Item, teardown_failed: bool) -> None: + leak: Final = detect_vcr_patch_leak() + if leak is None: + return + restore_vcr_patch_points() + if teardown_failed: + return + pytest.fail( + f"{item.nodeid} finished with a vcrpy cassette still patched into " + f"{', '.join(leak.patch_points)} (cassettes: {', '.join(leak.cassette_paths) or 'unknown'}); " + "the originals were restored so later tests are unaffected", + pytrace=False, + ) + + +@contextmanager +def rewound_new_episodes_cassette(cassette_dir: Path) -> Iterator[vcr.cassette.Cassette]: + cassette_path: Final = cassette_dir / "rewound_owner.yaml" + cassette_path.write_text("interactions: []\nversion: 1\n") + recorder: Final = vcr.VCR(cassette_library_dir=str(cassette_dir)) + with recorder.use_cassette(cassette_path.name, record_mode="new_episodes") as cassette: + yield cassette diff --git a/tests/capturing_transport.py b/tests/capturing_transport.py new file mode 100644 index 00000000000..496c286685c --- /dev/null +++ b/tests/capturing_transport.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Final + +import httpx +from pydantic import BaseModel, TypeAdapter + +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) + + +class CapturingTransport(httpx.AsyncBaseTransport, httpx.BaseTransport): + def __init__(self, response: BaseModel) -> None: + self._response: Final = response + self.request_bodies: tuple[Mapping[str, object], ...] = () + + def handle_request(self, request: httpx.Request) -> httpx.Response: + return self._respond(request.read()) + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + return self._respond(await request.aread()) + + def _respond(self, body: bytes) -> httpx.Response: + self.request_bodies = (*self.request_bodies, _JSON_OBJECT.validate_json(body)) + return httpx.Response(200, json=self._response.model_dump(mode="json")) diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 567040c1d19..a88dcf4ae3e 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -7,6 +7,8 @@ import asyncio import importlib +from collections.abc import Generator +from typing import Final import pytest @@ -20,6 +22,7 @@ from tests._vcr_conftest_common import ( # noqa: E402,F401 emit_cassette_cache_session_banner, emit_vcr_classification_summary, emit_vcr_diagnostic_log, + guard_vcr_patch_points, install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, @@ -37,17 +40,12 @@ def fake_openai_endpoint(): # Per-item respx detection (``apply_vcr_auto_marker_to_items``) handles # the vast majority of respx-vs-vcrpy conflicts automatically. The entries -# below are the persister's and the WebSocket VCR's own unit-test files, which -# exercise ``save_cassette`` / ``load_cassette`` against fakeredis and must not -# themselves run under a live cassette context. +# below are the persister's, the WebSocket VCR's, and the cassette patch-leak +# guard's own unit-test files, which exercise ``save_cassette`` / +# ``load_cassette`` against fakeredis or enter cassettes themselves and must +# not run under a live cassette context. _VCR_AUTO_MARKER_SKIP_FILES = frozenset( - {"test_vcr_redis_persister.py", "test_ws_vcr.py"} -) - -_VCR_INCOMPATIBLE_NODEID_SUFFIXES: tuple[str, ...] = ( - "test_nvidia_nim.py::test_embedding_nvidia_nim", - "test_litellm_proxy_provider.py::test_litellm_gateway_from_sdk_embedding[False]", - "test_litellm_proxy_provider.py::test_litellm_gateway_from_sdk_embedding[True]", + {"test_vcr_redis_persister.py", "test_ws_vcr.py", "test_vcr_leak_guard.py"} ) @@ -77,6 +75,17 @@ def _vcr_outcome_gate(request, vcr): record_vcr_outcome(request, vcr) +@pytest.hookimpl(wrapper=True, trylast=True) +def pytest_runtest_teardown(item: pytest.Item) -> Generator[None, object, object]: + try: + result: Final = yield + except BaseException: + guard_vcr_patch_points(item, teardown_failed=True) + raise + guard_vcr_patch_points(item, teardown_failed=False) + return result + + def pytest_configure(config): _verbose_state.remember_pluginmanager(config) reset_vcr_diag_dir() @@ -172,7 +181,6 @@ def pytest_collection_modifyitems(config, items): apply_vcr_auto_marker_to_items( items, skip_files=_VCR_AUTO_MARKER_SKIP_FILES, - skip_nodeid_suffixes=_VCR_INCOMPATIBLE_NODEID_SUFFIXES, ) custom_logger_tests = [ diff --git a/tests/llm_translation/test_litellm_proxy_provider.py b/tests/llm_translation/test_litellm_proxy_provider.py index 8630259877d..a10fc55ecc5 100644 --- a/tests/llm_translation/test_litellm_proxy_provider.py +++ b/tests/llm_translation/test_litellm_proxy_provider.py @@ -2,6 +2,8 @@ import json import re from datetime import datetime from io import BytesIO +from pathlib import Path +from typing import Final from unittest.mock import AsyncMock @@ -12,7 +14,12 @@ import pytest from unittest.mock import MagicMock, patch from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler import pytest_asyncio -from openai import AsyncOpenAI +from openai import AsyncOpenAI, OpenAI +from openai.types import CreateEmbeddingResponse, Embedding +from openai.types.create_embedding_response import Usage + +from tests.capturing_transport import CapturingTransport +from tests._vcr_conftest_common import rewound_new_episodes_cassette @pytest.mark.asyncio @@ -87,62 +94,61 @@ async def test_litellm_gateway_from_sdk_structured_output(): assert "json_schema" in json_schema -@pytest.mark.parametrize("is_async", [False, True]) +_GATEWAY_EMBEDDING_RESPONSE: Final = CreateEmbeddingResponse( + object="list", + data=(Embedding(object="embedding", index=0, embedding=(0.1, 0.2, 0.3)),), + model="my-vllm-model", + usage=Usage(prompt_tokens=2, total_tokens=2), +) + + +async def _gateway_embedding_via_injected_client( + is_async: bool, +) -> tuple[CapturingTransport, litellm.EmbeddingResponse]: + transport: Final = CapturingTransport(_GATEWAY_EMBEDDING_RESPONSE) + response: Final = ( + await litellm.aembedding( + model="litellm_proxy/my-vllm-model", + input="Hello world", + client=AsyncOpenAI(api_key="fake-key", http_client=httpx.AsyncClient(transport=transport)), + api_base="my-custom-api-base", + ) + if is_async + else litellm.embedding( + model="litellm_proxy/my-vllm-model", + input="Hello world", + client=OpenAI(api_key="fake-key", http_client=httpx.Client(transport=transport)), + api_base="my-custom-api-base", + ) + ) + return transport, response + + +@pytest.mark.parametrize("is_async", (False, True)) @pytest.mark.asyncio -async def test_litellm_gateway_from_sdk_embedding(is_async): +async def test_litellm_gateway_from_sdk_embedding(is_async: bool): litellm.set_verbose = True litellm._turn_on_debug() - captured_bodies = [] - - def handler(request: httpx.Request) -> httpx.Response: - captured_bodies.append(json.loads(request.content)) - return httpx.Response( - 200, - json={ - "object": "list", - "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], - "model": "my-vllm-model", - "usage": {"prompt_tokens": 2, "total_tokens": 2}, - }, - ) - - if is_async: - from openai import AsyncOpenAI - - openai_client = AsyncOpenAI( - api_key="fake-key", - http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), - ) - response = await litellm.aembedding( - model="litellm_proxy/my-vllm-model", - input="Hello world", - client=openai_client, - api_base="my-custom-api-base", - ) - else: - from openai import OpenAI - - openai_client = OpenAI( - api_key="fake-key", - http_client=httpx.Client(transport=httpx.MockTransport(handler)), - ) - response = litellm.embedding( - model="litellm_proxy/my-vllm-model", - input="Hello world", - client=openai_client, - api_base="my-custom-api-base", - ) - - request_body = captured_bodies[0] - print("Request body - {}".format(request_body)) + transport, response = await _gateway_embedding_via_injected_client(is_async) + request_body: Final = transport.request_bodies[0] assert "Hello world" == request_body["input"] assert "my-vllm-model" == request_body["model"] assert "encoding_format" not in request_body assert response.data[0]["embedding"] == [0.1, 0.2, 0.3] +@pytest.mark.asyncio +async def test_litellm_gateway_from_sdk_embedding_under_foreign_cassette(tmp_path: Path): + with rewound_new_episodes_cassette(tmp_path): + sync_transport, _ = await _gateway_embedding_via_injected_client(is_async=False) + async_transport, _ = await _gateway_embedding_via_injected_client(is_async=True) + + assert tuple(body["input"] for body in sync_transport.request_bodies) == ("Hello world",) + assert tuple(body["input"] for body in async_transport.request_bodies) == ("Hello world",) + + @pytest.mark.parametrize("is_async", [False, True]) @pytest.mark.asyncio async def test_litellm_gateway_from_sdk_image_generation(is_async): diff --git a/tests/llm_translation/test_nvidia_nim.py b/tests/llm_translation/test_nvidia_nim.py index d5942e674d0..0f16c01fd2f 100644 --- a/tests/llm_translation/test_nvidia_nim.py +++ b/tests/llm_translation/test_nvidia_nim.py @@ -1,17 +1,21 @@ import json from datetime import datetime +from typing import Final from unittest.mock import AsyncMock import httpx import pytest +from openai.types import CreateEmbeddingResponse, Embedding +from openai.types.create_embedding_response import Usage as EmbeddingUsage from unittest.mock import patch, MagicMock import litellm from litellm import Choices, Message, ModelResponse, EmbeddingResponse, Usage from litellm import completion from base_rerank_unit_tests import BaseLLMRerankTest +from tests.capturing_transport import CapturingTransport def test_completion_nvidia_nim(): @@ -63,33 +67,23 @@ def test_embedding_nvidia_nim(): litellm.set_verbose = True from openai import OpenAI - captured_bodies = [] - - def handler(request: httpx.Request) -> httpx.Response: - captured_bodies.append(json.loads(request.content)) - return httpx.Response( - 200, - json={ - "object": "list", - "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], - "model": "nvidia/nv-embedqa-e5-v5", - "usage": {"prompt_tokens": 6, "total_tokens": 6}, - }, + transport: Final = CapturingTransport( + CreateEmbeddingResponse( + object="list", + data=(Embedding(object="embedding", index=0, embedding=(0.1, 0.2, 0.3)),), + model="nvidia/nv-embedqa-e5-v5", + usage=EmbeddingUsage(prompt_tokens=6, total_tokens=6), ) - - client = OpenAI( - api_key="fake-api-key", - http_client=httpx.Client(transport=httpx.MockTransport(handler)), ) - response = litellm.embedding( + client: Final = OpenAI(api_key="fake-api-key", http_client=httpx.Client(transport=transport)) + response: Final = litellm.embedding( model="nvidia_nim/nvidia/nv-embedqa-e5-v5", input="What is the meaning of life?", input_type="passage", dimensions=1024, client=client, ) - request_body = captured_bodies[0] - print("request_body: ", request_body) + request_body: Final = transport.request_bodies[0] assert request_body["input"] == "What is the meaning of life?" assert request_body["model"] == "nvidia/nv-embedqa-e5-v5" assert request_body["input_type"] == "passage" diff --git a/tests/llm_translation/test_vcr_leak_guard.py b/tests/llm_translation/test_vcr_leak_guard.py new file mode 100644 index 00000000000..5372342790b --- /dev/null +++ b/tests/llm_translation/test_vcr_leak_guard.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import re +from pathlib import Path +from typing import Final + +import httpx +import httpx2 +import pytest + +from tests._vcr_conftest_common import ( + detect_vcr_patch_leak, + guard_vcr_patch_points, + restore_vcr_patch_points, + rewound_new_episodes_cassette, +) + +_ORIGINAL_MOCK_HANDLE_ASYNC_REQUEST: Final = httpx.MockTransport.handle_async_request +_ORIGINAL_HTTPX2_MOCK_HANDLE_ASYNC_REQUEST: Final = httpx2.MockTransport.handle_async_request + + +@pytest.fixture +def leaked_cassette_dir(tmp_path: Path): + context: Final = rewound_new_episodes_cassette(tmp_path) + context.__enter__() + yield tmp_path + context.__exit__(None, None, None) + + +def test_no_leak_when_no_cassette_is_active(): + assert detect_vcr_patch_leak() is None + + +def test_leaked_cassette_is_detected_named_and_restorable(leaked_cassette_dir: Path): + leak: Final = detect_vcr_patch_leak() + + assert leak is not None + assert {"httpx.MockTransport.handle_async_request", "aiohttp.client.ClientSession._request"} <= set( + leak.patch_points + ) + assert leak.cassette_paths == (str(leaked_cassette_dir / "rewound_owner.yaml"),) + + restore_vcr_patch_points() + + assert detect_vcr_patch_leak() is None + assert httpx.MockTransport.handle_async_request is _ORIGINAL_MOCK_HANDLE_ASYNC_REQUEST + + +def test_leak_is_detected_on_every_transport_family_vcrpy_patches(leaked_cassette_dir: Path): + leak: Final = detect_vcr_patch_leak() + + assert leak is not None + assert "httpx2.MockTransport.handle_async_request" in leak.patch_points + assert httpx2.MockTransport.handle_async_request is not _ORIGINAL_HTTPX2_MOCK_HANDLE_ASYNC_REQUEST + + restore_vcr_patch_points() + + assert httpx2.MockTransport.handle_async_request is _ORIGINAL_HTTPX2_MOCK_HANDLE_ASYNC_REQUEST + + +def test_guard_fails_the_leaking_test_and_restores_the_originals(request, leaked_cassette_dir: Path): + with pytest.raises(pytest.fail.Exception, match=re.escape(request.node.nodeid)) as failure: + guard_vcr_patch_points(request.node, teardown_failed=False) + + assert str(leaked_cassette_dir / "rewound_owner.yaml") in str(failure.value) + assert detect_vcr_patch_leak() is None + + +def test_guard_restores_silently_when_the_teardown_already_failed(request, leaked_cassette_dir: Path): + guard_vcr_patch_points(request.node, teardown_failed=True) + + assert detect_vcr_patch_leak() is None diff --git a/tests/local_testing/conftest.py b/tests/local_testing/conftest.py index 228457f4d55..d03f074f557 100644 --- a/tests/local_testing/conftest.py +++ b/tests/local_testing/conftest.py @@ -93,7 +93,6 @@ _VCR_INCOMPATIBLE_FILES = frozenset( # carry no real provider cost. _VCR_INCOMPATIBLE_NODEID_SUFFIXES: tuple[str, ...] = ( "test_router.py::test_router_text_completion_client", - "test_embedding.py::test_encoding_format_omitted_by_default_for_openai_sdk", ) diff --git a/tests/local_testing/test_embedding.py b/tests/local_testing/test_embedding.py index c119334da6f..acbc4f20405 100644 --- a/tests/local_testing/test_embedding.py +++ b/tests/local_testing/test_embedding.py @@ -15,6 +15,9 @@ from unittest.mock import AsyncMock, MagicMock, patch import litellm from litellm import completion, completion_cost, embedding +from openai.types import CreateEmbeddingResponse +from openai.types.create_embedding_response import Usage as EmbeddingUsage +from tests.capturing_transport import CapturingTransport from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE litellm.set_verbose = False @@ -1268,23 +1271,15 @@ def test_encoding_format_omitted_by_default_for_openai_sdk(monkeypatch): Optional global override: `LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT`. """ monkeypatch.delenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", raising=False) - captured_bodies = [] - - def handler(request: httpx.Request) -> httpx.Response: - captured_bodies.append(json.loads(request.content)) - return httpx.Response( - 200, - json={ - "object": "list", - "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], - "model": "text-embedding-ada-002", - "usage": {"prompt_tokens": 1, "total_tokens": 1}, - }, + transport = CapturingTransport( + CreateEmbeddingResponse( + object="list", + data=(Embedding(object="embedding", index=0, embedding=(0.1, 0.2, 0.3)),), + model="text-embedding-ada-002", + usage=EmbeddingUsage(prompt_tokens=1, total_tokens=1), ) - - client = openai.OpenAI( - api_key="sk-test", http_client=httpx.Client(transport=httpx.MockTransport(handler)) ) + client = openai.OpenAI(api_key="sk-test", http_client=httpx.Client(transport=transport)) response = embedding( model="text-embedding-ada-002", @@ -1294,7 +1289,7 @@ def test_encoding_format_omitted_by_default_for_openai_sdk(monkeypatch): ) assert response.data[0]["embedding"] == [0.1, 0.2, 0.3] - assert "encoding_format" not in captured_bodies[0], ( + assert "encoding_format" not in transport.request_bodies[0], ( "encoding_format should be omitted from the upstream request when not provided by user" )