From 491491480195685e3987d3e6d47a987d3d51f9ae Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 1 Sep 2026 14:45:12 -0700 Subject: [PATCH 01/88] feat(guardrails): roll up Bedrock guardrail cost per usage counter The daily guardrail usage rollup stored billable units per counter but no cost, so the usage endpoints could only report units. The Bedrock hook now stamps guardrail_cost_by_unit next to guardrail_usage, the spend-log aggregator sums it into a new nullable cost column on LiteLLM_DailyGuardrailUsageUnits, and /guardrails/usage/overview and /guardrails/usage/detail/{id} return cost, totalCost and cost_by_unit / cost_by_team / cost_by_key alongside the existing unit breakdowns. Cost is nullable on purpose. Rows written before this migration, and rows whose hook had no pricing entry, read as null rather than $0, and a single unpriced increment keeps that row's cost unknown instead of partial. guardrail_cost and the spend/budget path are untouched. Claude-Session: https://claude.ai/code/session_01EX13mWex6RaBo9PYnkAtFW --- basedpyright-code-budget.json | 2 +- .../migration.sql | 2 + .../litellm_proxy_extras/schema.prisma | 1 + .../llm_cost_calc/guardrail_cost.py | 47 ++++++- litellm/proxy/_lazy_openapi_snapshot.json | 129 ++++++++++++++++- .../guardrail_hooks/bedrock_guardrails.py | 48 ++++--- litellm/proxy/guardrails/usage_endpoints.py | 54 ++++++-- litellm/proxy/guardrails/usage_tracking.py | 63 +++++++-- litellm/proxy/schema.prisma | 1 + litellm/types/utils.py | 6 + schema.prisma | 1 + .../llm_cost_calc/test_guardrail_cost.py | 54 ++++++++ .../test_bedrock_guardrails.py | 27 +++- .../proxy/guardrails/test_usage_endpoints.py | 74 +++++++++- .../proxy/guardrails/test_usage_tracking.py | 130 +++++++++++++++++- type-discipline-budget.json | 2 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 20 +++ 17 files changed, 605 insertions(+), 56 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000001_add_guardrail_usage_units_cost/migration.sql diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index df52069e71f..d64978180fb 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -93,7 +93,7 @@ "limit": 181 }, "reportTypedDictNotRequiredAccess": { - "limit": 24 + "limit": 22 }, "reportUndefinedVariable": { "limit": 0 diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000001_add_guardrail_usage_units_cost/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000001_add_guardrail_usage_units_cost/migration.sql new file mode 100644 index 00000000000..27a86a0b09a --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000001_add_guardrail_usage_units_cost/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "LiteLLM_DailyGuardrailUsageUnits" ADD COLUMN IF NOT EXISTS "cost" DOUBLE PRECISION; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 7604ceadf7a..3134d7dde0e 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1123,6 +1123,7 @@ model LiteLLM_DailyGuardrailUsageUnits { api_key String // hashed virtual key; empty string when unknown usage_unit String // provider counter name, e.g. Bedrock's contentPolicyUnits units BigInt @default(0) + cost Float? // USD billed for these units; null when any contributing increment was unpriced created_at DateTime @default(now()) updated_at DateTime @updatedAt diff --git a/litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py b/litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py index ad1880d4cc2..64e82053c94 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py +++ b/litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py @@ -1,8 +1,8 @@ import math from collections.abc import Mapping -from typing import Final +from typing import Annotated, Final -from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_logger @@ -30,6 +30,30 @@ class GuardrailCostEntry(BaseModel): _GUARDRAIL_COST_ENTRY_ADAPTER: Final[TypeAdapter[GuardrailCostEntry]] = TypeAdapter(GuardrailCostEntry) +class GuardrailCostByUnitEntry(BaseModel): + """The rollup-side view of a ``guardrail_information`` entry, validated apart from + ``GuardrailCostEntry`` so a forged per-counter map can never zero the spend path.""" + + model_config = ConfigDict(extra="ignore", frozen=True) + + guardrail_cost_by_unit: Mapping[str, Annotated[float, Field(ge=0, allow_inf_nan=False)]] | None = None + guardrail_cost_in_spend: bool | None = True + + +_GUARDRAIL_COST_BY_UNIT_ADAPTER: Final[TypeAdapter[GuardrailCostByUnitEntry]] = TypeAdapter(GuardrailCostByUnitEntry) + + +def billed_guardrail_cost_by_unit(raw: object) -> Mapping[str, float] | None: + """Per-counter USD the daily rollup may record for one raw ``guardrail_information`` + entry; None when the entry is unpriced, report-only, or malformed.""" + try: + entry: Final = _GUARDRAIL_COST_BY_UNIT_ADAPTER.validate_python(raw) + except ValidationError as e: + verbose_logger.warning("Ignoring malformed guardrail_information entry for guardrail cost rollup: %s", e) + return None + return None if entry.guardrail_cost_in_spend is False else entry.guardrail_cost_by_unit + + def _bedrock_guardrail_pricing(aws_region_name: str | None) -> GuardrailPricing | None: regional_key: Final = f"bedrock/{aws_region_name}/guardrails" if aws_region_name else None for key in (regional_key, BEDROCK_GUARDRAIL_PRICING_KEY): @@ -42,11 +66,24 @@ def _bedrock_guardrail_pricing(aws_region_name: str | None) -> GuardrailPricing return None -def bedrock_guardrail_cost(usage_units: Mapping[str, int], aws_region_name: str | None) -> float: +def bedrock_guardrail_cost_by_unit( + usage_units: Mapping[str, int], aws_region_name: str | None +) -> Mapping[str, float] | None: + """USD per counter, keyed like ``usage_units``; None when no pricing entry exists.""" pricing: Final = _bedrock_guardrail_pricing(aws_region_name) if pricing is None: - return 0.0 - return sum(units * pricing.guardrail_cost_per_unit.get(counter, 0.0) for counter, units in usage_units.items()) + return None + return { # mutable-ok: stamped into guardrail_information, which safe_dumps only serializes as a plain dict + counter: units * pricing.guardrail_cost_per_unit.get(counter, 0.0) for counter, units in usage_units.items() + } + + +def guardrail_cost_total(cost_by_unit: Mapping[str, float] | None) -> float: + return sum(cost_by_unit.values()) if cost_by_unit is not None else 0.0 + + +def bedrock_guardrail_cost(usage_units: Mapping[str, int], aws_region_name: str | None) -> float: + return guardrail_cost_total(bedrock_guardrail_cost_by_unit(usage_units, aws_region_name)) AZURE_PROMPT_SHIELD_TEXT_RECORD_UNIT: Final = "text_records" diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 5af45b29226..5385b4d6f7e 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -13039,6 +13039,59 @@ ], "title": "Avgscore" }, + "cost": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cost" + }, + "cost_by_key": { + "additionalProperties": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "title": "Cost By Key", + "type": "object" + }, + "cost_by_team": { + "additionalProperties": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "title": "Cost By Team", + "type": "object" + }, + "cost_by_unit": { + "additionalProperties": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "title": "Cost By Unit", + "type": "object" + }, "description": { "anyOf": [ { @@ -13140,7 +13193,11 @@ "usage_units", "usage_units_daily", "usage_units_by_team", - "usage_units_by_key" + "usage_units_by_key", + "cost", + "cost_by_unit", + "cost_by_team", + "cost_by_key" ], "title": "UsageDetailResponse", "type": "object" @@ -13295,6 +13352,17 @@ "title": "Totalblocked", "type": "integer" }, + "totalCost": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Totalcost" + }, "totalRequests": { "title": "Totalrequests", "type": "integer" @@ -13313,7 +13381,8 @@ "totalRequests", "totalBlocked", "passRate", - "totalUsageUnits" + "totalUsageUnits", + "totalCost" ], "title": "UsageOverviewResponse", "type": "object" @@ -13342,6 +13411,17 @@ ], "title": "Avgscore" }, + "cost": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cost" + }, "failRate": { "title": "Failrate", "type": "number" @@ -13393,13 +13473,25 @@ "avgLatency", "status", "trend", - "usageUnits" + "usageUnits", + "cost" ], "title": "UsageOverviewRow", "type": "object" }, "UsageUnitsDailyPoint": { "properties": { + "cost": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cost" + }, "date": { "title": "Date", "type": "string" @@ -13414,7 +13506,8 @@ }, "required": [ "date", - "units" + "units", + "cost" ], "title": "UsageUnitsDailyPoint", "type": "object" @@ -28773,6 +28866,17 @@ "title": "Totalblocked", "type": "integer" }, + "totalCost": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Totalcost" + }, "totalRequests": { "title": "Totalrequests", "type": "integer" @@ -28791,7 +28895,8 @@ "totalRequests", "totalBlocked", "passRate", - "totalUsageUnits" + "totalUsageUnits", + "totalCost" ], "title": "UsageOverviewResponse", "type": "object" @@ -28820,6 +28925,17 @@ ], "title": "Avgscore" }, + "cost": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cost" + }, "failRate": { "title": "Failrate", "type": "number" @@ -28871,7 +28987,8 @@ "avgLatency", "status", "trend", - "usageUnits" + "usageUnits", + "cost" ], "title": "UsageOverviewRow", "type": "object" diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 0237d82a0d9..c6a85c3bfbb 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -35,7 +35,10 @@ from litellm.litellm_core_utils.core_helpers import redact_nested_match_and_rege from litellm.litellm_core_utils.litellm_logging import ( _get_masked_values, # pyright: ignore[reportPrivateUsage] # the shared header-masking helper has no public name ) -from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import bedrock_guardrail_cost +from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import ( + bedrock_guardrail_cost_by_unit, + guardrail_cost_total, +) from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler from litellm.llms.base_llm.guardrail_translation.utils import ( effective_scan_only_tool_results_for_guardrail, @@ -109,6 +112,7 @@ _BEDROCK_TOO_LARGE_ERROR_SUBSTRINGS: Final = ( _BEDROCK_APPLY_GUARDRAIL_MAX_THROTTLE_RETRIES: Final = 3 _BEDROCK_APPLY_GUARDRAIL_BASE_BACKOFF_SECONDS: Final = 0.5 _BEDROCK_WHITESPACE: Final = re.compile(r"\s") +_NO_TRACING_DETAIL: Final[GuardrailTracingDetail] = {} # Resource-less, detect-only InvokeGuardrailChecks API (no guardrail resource required). _BEDROCK_INVOKE_GUARDRAIL_CHECKS_PATH: Final = "/guardrail-checks/invoke" # InvokeGuardrailChecks accepts at most 10 content blocks per message. A message with @@ -2147,25 +2151,37 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): OTEL integration can expose it as a queryable span attribute without re-parsing the redacted guardrail_response blob. """ - tracing_detail: Final[GuardrailTracingDetail] = {} violation_categories: Final = self._extract_violation_category_names(response) - if violation_categories: - tracing_detail["violation_categories"] = violation_categories bedrock_action: Final = response.get("action") - if isinstance(bedrock_action, str): - tracing_detail["guardrail_action"] = bedrock_action - usage: Final = response.get("usage") - if isinstance(usage, dict): - usage_units: Final = { # mutable-ok: json.dumps'd into spend log metadata downstream - key: value for key, value in usage.items() if isinstance(value, int) - } - if usage_units: - tracing_detail["guardrail_usage"] = usage_units - tracing_detail["guardrail_cost"] = bedrock_guardrail_cost( - usage_units=usage_units, aws_region_name=aws_region_name - ) + categories_detail: Final[GuardrailTracingDetail] = {"violation_categories": violation_categories} + action_detail: Final[GuardrailTracingDetail] = {"guardrail_action": bedrock_action} + tracing_detail: Final[GuardrailTracingDetail] = { + **(categories_detail if violation_categories else _NO_TRACING_DETAIL), + **(action_detail if isinstance(bedrock_action, str) else _NO_TRACING_DETAIL), + **self._usage_tracing_detail(response.get("usage"), aws_region_name), + } return tracing_detail + @staticmethod + def _usage_tracing_detail( + usage: BedrockGuardrailUsage | None, aws_region_name: str | None + ) -> GuardrailTracingDetail: + if not isinstance(usage, dict): + return _NO_TRACING_DETAIL + usage_units: Final = { # mutable-ok: json.dumps'd into spend log metadata downstream + key: value for key, value in usage.items() if isinstance(value, int) + } + if not usage_units: + return _NO_TRACING_DETAIL + cost_by_unit: Final = bedrock_guardrail_cost_by_unit(usage_units=usage_units, aws_region_name=aws_region_name) + priced_detail: Final[GuardrailTracingDetail] = {"guardrail_cost_by_unit": cost_by_unit} + usage_detail: Final[GuardrailTracingDetail] = { + "guardrail_usage": usage_units, + "guardrail_cost": guardrail_cost_total(cost_by_unit), + **(priced_detail if cost_by_unit is not None else _NO_TRACING_DETAIL), + } + return usage_detail + def _extract_violation_category_names(self, response: BedrockGuardrailResponse) -> list[str]: """ Flatten the BLOCKED assessments into a list of human-readable category diff --git a/litellm/proxy/guardrails/usage_endpoints.py b/litellm/proxy/guardrails/usage_endpoints.py index 7a0edbddca8..69516487d7c 100644 --- a/litellm/proxy/guardrails/usage_endpoints.py +++ b/litellm/proxy/guardrails/usage_endpoints.py @@ -169,6 +169,20 @@ def _units_by( return MappingProxyType({key: _sum_counter_units(group) for key, group in groupby(ordered, key=key_of)}) +def _sum_tracked_cost(rows: "Iterable[prisma_models.LiteLLM_DailyGuardrailUsageUnits]") -> float | None: + """Sum over rows with a tracked cost; None when no row has one (pre-migration or unpriced).""" + tracked: Final = tuple(r.cost for r in rows if r.cost is not None) + return sum(tracked) if tracked else None + + +def _cost_by( + rows: "Sequence[prisma_models.LiteLLM_DailyGuardrailUsageUnits]", + key_of: "Callable[[prisma_models.LiteLLM_DailyGuardrailUsageUnits], str]", +) -> Mapping[str, float | None]: + ordered: Final = sorted(rows, key=key_of) + return MappingProxyType({key: _sum_tracked_cost(group) for key, group in groupby(ordered, key=key_of)}) + + # --- Response models --- @@ -218,6 +232,8 @@ class UsageOverviewRow(BaseModel): status: str # healthy | warning | critical trend: str # up | down | stable usageUnits: Mapping[str, int] + cost: float | None + """USD billed for usageUnits over the window, summed over days with tracked cost; null when none have it.""" class UsageOverviewResponse(BaseModel): @@ -227,11 +243,18 @@ class UsageOverviewResponse(BaseModel): totalBlocked: int passRate: float totalUsageUnits: Mapping[str, int] + totalCost: float | None + + +_EMPTY_OVERVIEW: Final = UsageOverviewResponse( + rows=[], chart=[], totalRequests=0, totalBlocked=0, passRate=100.0, totalUsageUnits=_EMPTY_UNITS, totalCost=None +) class UsageUnitsDailyPoint(BaseModel): date: str units: Mapping[str, int] + cost: float | None class UsageDetailResponse(BaseModel): @@ -251,6 +274,10 @@ class UsageDetailResponse(BaseModel): usage_units_daily: Sequence[UsageUnitsDailyPoint] usage_units_by_team: Mapping[str, Mapping[str, int]] usage_units_by_key: Mapping[str, Mapping[str, int]] + cost: float | None + cost_by_unit: Mapping[str, float | None] + cost_by_team: Mapping[str, float | None] + cost_by_key: Mapping[str, float | None] class UsageLogEntry(BaseModel): @@ -367,6 +394,7 @@ def _guardrail_overview_rows( agg: Mapping[str, _MetricTotals], prev_agg: Mapping[str, float], units_agg: Mapping[str, Mapping[str, int]], + cost_agg: Mapping[str, float | None], ) -> list[UsageOverviewRow]: rows: Final[list[UsageOverviewRow]] = [] covered_keys: Final[set[str]] = set() @@ -393,6 +421,7 @@ def _guardrail_overview_rows( break trend = _trend_from_comparison(fail_rate, prev_fail) row_units: Mapping[str, int] = next((units_agg[k] for k in lookup_keys if k in units_agg), _EMPTY_UNITS) + row_cost: float | None = next((cost_agg[k] for k in lookup_keys if k in cost_agg), None) rows.append( UsageOverviewRow( id=gid, @@ -406,6 +435,7 @@ def _guardrail_overview_rows( status=_status_from_fail_rate(fail_rate), trend=trend, usageUnits=row_units, + cost=row_cost, ) ) # Add rows for guardrails with metrics but not in guardrails table (e.g. MCP, config) @@ -429,6 +459,7 @@ def _guardrail_overview_rows( status=_status_from_fail_rate(fail_rate), trend=trend, usageUnits=units_agg.get(agg_key, _EMPTY_UNITS), + cost=cost_agg.get(agg_key), ) ) return rows @@ -459,6 +490,7 @@ def _policy_overview_rows( status=_status_from_fail_rate(fail_rate), trend=trend, usageUnits=_EMPTY_UNITS, + cost=None, ) ) return rows @@ -479,9 +511,7 @@ async def guardrails_usage_overview( from litellm.proxy.proxy_server import prisma_client if prisma_client is None: - return UsageOverviewResponse( - rows=[], chart=[], totalRequests=0, totalBlocked=0, passRate=100.0, totalUsageUnits=_EMPTY_UNITS - ) + return _EMPTY_OVERVIEW start, end = _resolve_usage_window(start_date, end_date) @@ -516,11 +546,12 @@ async def guardrails_usage_overview( agg: Final = _aggregate_daily_metrics(metrics, "guardrail_id") prev_agg: Final = _prev_fail_rates(metrics_prev, "guardrail_id") units_agg: Final = _units_by(units_rows, lambda r: r.guardrail_id) + cost_agg: Final = _cost_by(units_rows, lambda r: r.guardrail_id) chart: Final = _chart_from_metrics(metrics) total_requests: Final = sum(a["requests"] for a in agg.values()) total_blocked: Final = sum(a["blocked"] for a in agg.values()) pass_rate: Final = (100.0 * (total_requests - total_blocked) / total_requests) if total_requests else 100.0 - rows: Final = _guardrail_overview_rows(guardrails, agg, prev_agg, units_agg) + rows: Final = _guardrail_overview_rows(guardrails, agg, prev_agg, units_agg, cost_agg) return UsageOverviewResponse( rows=rows, chart=chart, @@ -528,6 +559,7 @@ async def guardrails_usage_overview( totalBlocked=total_blocked, passRate=round(pass_rate, 1), totalUsageUnits=_sum_counter_units(units_rows), + totalCost=_sum_tracked_cost(units_rows), ) except Exception as e: from litellm.proxy.utils import handle_exception_on_proxy @@ -619,7 +651,10 @@ async def guardrails_usage_detail( guardrail_info: Final = _to_dict(_get_guardrail_field(guardrail, "guardrail_info")) _guardrail_name: Final = _get_guardrail_field(guardrail, "guardrail_name") daily_unit_sums: Final = sorted(_units_by(units_rows, lambda r: r.date).items()) - units_daily: Final = tuple(UsageUnitsDailyPoint(date=d, units=units) for d, units in daily_unit_sums) + daily_cost: Final = _cost_by(units_rows, lambda r: r.date) + units_daily: Final = tuple( + UsageUnitsDailyPoint(date=d, units=units, cost=daily_cost.get(d)) for d, units in daily_unit_sums + ) return UsageDetailResponse( guardrail_id=guardrail_id, @@ -638,6 +673,10 @@ async def guardrails_usage_detail( usage_units_daily=units_daily, usage_units_by_team=_units_by(units_rows, lambda r: r.team_id), usage_units_by_key=_units_by(units_rows, lambda r: r.api_key), + cost=_sum_tracked_cost(units_rows), + cost_by_unit=_cost_by(units_rows, _counter_name), + cost_by_team=_cost_by(units_rows, lambda r: r.team_id), + cost_by_key=_cost_by(units_rows, lambda r: r.api_key), ) @@ -857,9 +896,7 @@ async def policies_usage_overview( from litellm.proxy.proxy_server import prisma_client if prisma_client is None: - return UsageOverviewResponse( - rows=[], chart=[], totalRequests=0, totalBlocked=0, passRate=100.0, totalUsageUnits=_EMPTY_UNITS - ) + return _EMPTY_OVERVIEW start, end = _resolve_usage_window(start_date, end_date) @@ -891,6 +928,7 @@ async def policies_usage_overview( totalBlocked=total_blocked, passRate=round(pass_rate, 1), totalUsageUnits=_EMPTY_UNITS, + totalCost=None, ) except Exception as e: from litellm.proxy.utils import handle_exception_on_proxy diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index b8ae09afc00..41cad232efe 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -6,7 +6,7 @@ insert into SpendLogGuardrailIndex when spend logs are written. import asyncio import json from collections import defaultdict -from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence +from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence from datetime import datetime, timezone from functools import partial from itertools import groupby @@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Final, NamedTuple, TypeVar from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import billed_guardrail_cost_by_unit from litellm.proxy._types import DB_RETRY_SAFE_ERROR_TYPES from litellm.proxy.utils import PrismaClient from litellm.repositories.table_repositories import ( @@ -44,6 +45,11 @@ class _UsageUnitKey(NamedTuple): usage_unit: str +class _UsageUnitIncrement(NamedTuple): + units: int + cost: float | None + + class _MetricsKey(NamedTuple): guardrail_id: str date: str @@ -67,22 +73,38 @@ class PendingRollups: def __init__(self) -> None: self.lock: Final = asyncio.Lock() self.metrics: Mapping[_MetricsKey, Mapping[str, int]] = MappingProxyType({}) - self.units: Mapping[_UsageUnitKey, int] = MappingProxyType({}) + self.units: Mapping[_UsageUnitKey, _UsageUnitIncrement] = MappingProxyType({}) _PENDING_ROLLUPS: Final = PendingRollups() _NO_COUNTERS: Final[Mapping[str, int]] = MappingProxyType({}) +_NO_INCREMENT: Final = _UsageUnitIncrement(units=0, cost=0.0) def _merged_keys(base: Mapping[_RowKey, object], extra: Mapping[_RowKey, object]) -> tuple[_RowKey, ...]: return (*base, *(key for key in extra if key not in base)) +def _summed_increments(increments: Iterable[_UsageUnitIncrement]) -> _UsageUnitIncrement: + """Units add; cost adds too unless any increment was unpriced, which makes the sum unknown.""" + materialized: Final = tuple(increments) + costs: Final = tuple(i.cost for i in materialized) + return _UsageUnitIncrement( + units=sum(i.units for i in materialized), + cost=None if any(c is None for c in costs) else sum(c for c in costs if c is not None), + ) + + def _merged_unit_rows( - base: Mapping[_UsageUnitKey, int], extra: Mapping[_UsageUnitKey, int] -) -> Mapping[_UsageUnitKey, int]: - return MappingProxyType({key: base.get(key, 0) + extra.get(key, 0) for key in _merged_keys(base, extra)}) + base: Mapping[_UsageUnitKey, _UsageUnitIncrement], extra: Mapping[_UsageUnitKey, _UsageUnitIncrement] +) -> Mapping[_UsageUnitKey, _UsageUnitIncrement]: + return MappingProxyType( + { + key: _summed_increments((base.get(key, _NO_INCREMENT), extra.get(key, _NO_INCREMENT))) + for key in _merged_keys(base, extra) + } + ) def _merged_metric_rows( @@ -209,7 +231,9 @@ def _parse_payload_start_time(payload: Mapping[str, Any]) -> datetime | None: return None -def _iter_usage_unit_increments(logs_to_process: Sequence[Mapping[str, Any]]) -> Iterator[tuple[_UsageUnitKey, int]]: +def _iter_usage_unit_increments( + logs_to_process: Sequence[Mapping[str, Any]], +) -> Iterator[tuple[_UsageUnitKey, _UsageUnitIncrement]]: for payload in logs_to_process: start_time = _parse_payload_start_time(payload) if not payload.get("request_id") or start_time is None: @@ -222,26 +246,37 @@ def _iter_usage_unit_increments(logs_to_process: Sequence[Mapping[str, Any]]) -> usage = entry.get("guardrail_usage") if not guardrail_id or not isinstance(usage, dict): continue + cost_by_unit = billed_guardrail_cost_by_unit(entry) for unit_name, units in usage.items(): if isinstance(units, int) and not isinstance(units, bool) and units > 0: - yield _UsageUnitKey(guardrail_id, date_key, team_id, api_key, str(unit_name)), units + key = _UsageUnitKey(guardrail_id, date_key, team_id, api_key, str(unit_name)) + cost = cost_by_unit.get(str(unit_name)) if cost_by_unit is not None else None + yield key, _UsageUnitIncrement(units=units, cost=cost) -def _sum_usage_unit_increments(logs_to_process: Sequence[Mapping[str, Any]]) -> Mapping[_UsageUnitKey, int]: +def _sum_usage_unit_increments( + logs_to_process: Sequence[Mapping[str, Any]], +) -> Mapping[_UsageUnitKey, _UsageUnitIncrement]: ordered: Final = sorted(_iter_usage_unit_increments(logs_to_process), key=itemgetter(0)) return MappingProxyType( - {key: sum(units for _, units in group) for key, group in groupby(ordered, key=itemgetter(0))} + { + key: _summed_increments(increment for _, increment in group) + for key, group in groupby(ordered, key=itemgetter(0)) + } ) -async def _upsert_usage_unit_row(prisma_client: PrismaClient, key: _UsageUnitKey, units: int) -> None: +async def _upsert_usage_unit_row( + prisma_client: PrismaClient, key: _UsageUnitKey, increment: _UsageUnitIncrement +) -> None: row: Final[prisma_types.LiteLLM_DailyGuardrailUsageUnitsCreateInput] = { "guardrail_id": key.guardrail_id, "date": key.date, "team_id": key.team_id, "api_key": key.api_key, "usage_unit": key.usage_unit, - "units": units, + "units": increment.units, + "cost": increment.cost, } where: Final[_UsageUnitWhereUnique] = { "guardrail_id_date_team_id_api_key_usage_unit": { @@ -252,9 +287,13 @@ async def _upsert_usage_unit_row(prisma_client: PrismaClient, key: _UsageUnitKey "usage_unit": key.usage_unit, } } + # NULL + x stays NULL in SQL, so an unknown cost stays unknown; writing NULL outright makes it so data: Final[prisma_types.LiteLLM_DailyGuardrailUsageUnitsUpsertInput] = { "create": row, - "update": {"units": {"increment": units}}, + "update": { + "units": {"increment": increment.units}, + "cost": {"increment": increment.cost} if increment.cost is not None else None, + }, } await DailyGuardrailUsageUnitsRepository(prisma_client).table.upsert(where=where, data=data) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 7604ceadf7a..3134d7dde0e 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1123,6 +1123,7 @@ model LiteLLM_DailyGuardrailUsageUnits { api_key String // hashed virtual key; empty string when unknown usage_unit String // provider counter name, e.g. Bedrock's contentPolicyUnits units BigInt @default(0) + cost Float? // USD billed for these units; null when any contributing increment was unpriced created_at DateTime @default(now()) updated_at DateTime @updatedAt diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3a0883b6607..8a6b1c13b2d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3142,6 +3142,11 @@ class StandardLoggingGuardrailInformation(TypedDict, total=False): provider hook. Summed into the request's ``response_cost`` so it counts against spend and budgets like token cost, unless ``guardrail_cost_in_spend`` is False.""" + guardrail_cost_by_unit: ReadOnly[Mapping[str, float] | None] + """``guardrail_cost`` split per ``guardrail_usage`` counter, so the daily + per-counter usage rollup can carry cost at its own grain. Absent when the + hook had no pricing for the invocation.""" + guardrail_cost_in_spend: ReadOnly[bool | None] """Whether ``guardrail_cost`` participates in the request's ``response_cost`` and the spend/budget aggregates built from it. Absent, None, or True keeps the default @@ -3193,6 +3198,7 @@ class GuardrailTracingDetail(TypedDict, total=False): guardrail_action: str | None guardrail_usage: ReadOnly[Mapping[str, int] | None] guardrail_cost: ReadOnly[float | None] + guardrail_cost_by_unit: ReadOnly[Mapping[str, float] | None] guardrail_cost_in_spend: ReadOnly[bool | None] diff --git a/schema.prisma b/schema.prisma index 7604ceadf7a..3134d7dde0e 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1123,6 +1123,7 @@ model LiteLLM_DailyGuardrailUsageUnits { api_key String // hashed virtual key; empty string when unknown usage_unit String // provider counter name, e.g. Bedrock's contentPolicyUnits units BigInt @default(0) + cost Float? // USD billed for these units; null when any contributing increment was unpriced created_at DateTime @default(now()) updated_at DateTime @updatedAt diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py index baaef31036c..6e9920d6f1d 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py @@ -5,6 +5,8 @@ import pytest import litellm from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import ( bedrock_guardrail_cost, + bedrock_guardrail_cost_by_unit, + billed_guardrail_cost_by_unit, cost_breakdown_with_guardrail, guardrail_information_cost, ) @@ -56,6 +58,58 @@ def test_bedrock_guardrail_cost_no_pricing_entry(monkeypatch): assert bedrock_guardrail_cost(usage_units={"contentPolicyUnits": 1}, aws_region_name="us-east-1") == 0.0 +def test_bedrock_guardrail_cost_by_unit_prices_every_counter_it_was_given(synthetic_cost_map): + """LIT-5652: the daily rollup stores one row per counter, so pricing must come + back at that grain, keyed exactly like the usage (free and unknown counters + included at 0.0) and summing to the scalar the spend path bills.""" + usage = {"contentPolicyUnits": 2, "topicPolicyUnits": 1, "wordPolicyUnits": 5, "someFutureCounter": 3} + by_unit = bedrock_guardrail_cost_by_unit(usage_units=usage, aws_region_name="us-east-1") + assert by_unit is not None + assert by_unit.keys() == usage.keys() + assert by_unit["contentPolicyUnits"] == pytest.approx(0.0003) + assert by_unit["topicPolicyUnits"] == pytest.approx(0.00015) + assert (by_unit["wordPolicyUnits"], by_unit["someFutureCounter"]) == (0.0, 0.0) + assert sum(by_unit.values()) == pytest.approx( + bedrock_guardrail_cost(usage_units=usage, aws_region_name="us-east-1") + ) + + +def test_bedrock_guardrail_cost_by_unit_is_none_without_pricing_so_unpriced_is_not_free(monkeypatch): + """The scalar keeps returning 0.0 for the spend path; the per-unit view must + say "unknown" instead so the rollup stores NULL rather than a $0 that would + hide the exact silent-spend problem this feature exists to surface.""" + monkeypatch.setattr(litellm, "model_cost", {}) + assert bedrock_guardrail_cost_by_unit(usage_units={"contentPolicyUnits": 1}, aws_region_name="us-east-1") is None + + +def test_billed_guardrail_cost_by_unit_reads_the_hook_stamp(): + entry = {"guardrail_name": "bedrock", "guardrail_cost_by_unit": {"contentPolicyUnits": 0.15, "wordPolicyUnits": 0}} + assert billed_guardrail_cost_by_unit(entry) == {"contentPolicyUnits": 0.15, "wordPolicyUnits": 0.0} + + +@pytest.mark.parametrize( + "entry", + [ + {"guardrail_name": "no-pricing", "guardrail_usage": {"contentPolicyUnits": 1}}, + {"guardrail_cost_by_unit": {"text_records": 0.5}, "guardrail_cost_in_spend": False}, + {"guardrail_cost_by_unit": {"contentPolicyUnits": -0.5}}, + {"guardrail_cost_by_unit": {"contentPolicyUnits": float("nan")}}, + {"guardrail_cost_by_unit": {"contentPolicyUnits": float("inf")}}, + {"guardrail_cost_by_unit": {"contentPolicyUnits": "bad"}}, + {"guardrail_cost_by_unit": "not-a-map"}, + {"guardrail_cost_by_unit": {"contentPolicyUnits": 0.1}, "guardrail_cost_in_spend": "maybe"}, + "not-an-entry", + ], +) +def test_billed_guardrail_cost_by_unit_is_none_when_unpriced_report_only_or_forged(entry): + assert billed_guardrail_cost_by_unit(entry) is None + + +def test_billed_guardrail_cost_by_unit_treats_none_in_spend_as_billed(): + entry = {"guardrail_cost_by_unit": {"contentPolicyUnits": 0.15}, "guardrail_cost_in_spend": None} + assert billed_guardrail_cost_by_unit(entry) == {"contentPolicyUnits": 0.15} + + def test_shipped_bedrock_guardrail_prices_match_aws_pricing_page(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index bcda1b8b61d..ec8996a8489 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -2961,9 +2961,7 @@ async def test_streaming_hook_reraises_guardrail_service_failures(): guardrail = _sse_guardrail() with patch.object(guardrail, "make_bedrock_api_request", new_callable=AsyncMock) as mock_api: - mock_api.side_effect = HTTPException( - status_code=500, detail="Bedrock guardrail throttle retries exhausted" - ) + mock_api.side_effect = HTTPException(status_code=500, detail="Bedrock guardrail throttle retries exhausted") with pytest.raises(HTTPException) as exc: await _drain_streaming_hook(guardrail) @@ -5104,6 +5102,26 @@ def test_build_tracing_detail_surfaces_usage_counters_and_cost(monkeypatch): assert detail["guardrail_usage"] == {"topicPolicyUnits": 1, "contentPolicyUnits": 2, "wordPolicyUnits": 0} assert detail["guardrail_cost"] == pytest.approx(0.00045) + by_unit = detail["guardrail_cost_by_unit"] + assert by_unit is not None and by_unit.keys() == detail["guardrail_usage"].keys() + assert by_unit["topicPolicyUnits"] == pytest.approx(0.00015) + assert by_unit["contentPolicyUnits"] == pytest.approx(0.0003) + assert by_unit["wordPolicyUnits"] == 0.0 + + +def test_build_tracing_detail_omits_cost_by_unit_when_unpriced_but_keeps_scalar_zero(monkeypatch): + """LIT-5652: without a cost-map entry the spend path still bills 0.0, but the + per-counter stamp must be absent so the rollup records NULL, not $0.""" + monkeypatch.setattr(litellm, "model_cost", {}) + guardrail = BedrockGuardrail(guardrailIdentifier="test-guardrail", guardrailVersion="DRAFT") + + detail = guardrail._build_tracing_detail( + {"action": "NONE", "usage": {"contentPolicyUnits": 5}}, aws_region_name="us-east-1" + ) + + assert detail["guardrail_usage"] == {"contentPolicyUnits": 5} + assert detail["guardrail_cost"] == 0.0 + assert "guardrail_cost_by_unit" not in detail def test_build_tracing_detail_omits_guardrail_usage_when_bedrock_reports_none(): @@ -5115,6 +5133,7 @@ def test_build_tracing_detail_omits_guardrail_usage_when_bedrock_reports_none(): ): assert "guardrail_usage" not in detail assert "guardrail_cost" not in detail + assert "guardrail_cost_by_unit" not in detail @pytest.mark.asyncio @@ -5478,7 +5497,7 @@ async def test_unbuffered_end_of_stream_hook_yields_chunks_before_scan(): scan_index = events.index("scan") chunk_events = [e for e in events if e != "scan"] assert events.count("scan") == 1 - assert [e for e in events[:scan_index] if e != "scan"] == chunk_events[: scan_index] + assert [e for e in events[:scan_index] if e != "scan"] == chunk_events[:scan_index] assert ("chunk", "Hello") in events[:scan_index] assert ("chunk", " world") in events[:scan_index] assert len(chunk_events) == 3 diff --git a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py index 1665fa03639..b8455b01e35 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py @@ -85,6 +85,7 @@ def _units_row( api_key: str = "", usage_unit: str = "contentPolicyUnits", units: int = 1, + cost: float | None = None, ) -> Any: r = MagicMock() r.guardrail_id = guardrail_id @@ -93,6 +94,7 @@ def _units_row( r.api_key = api_key r.usage_unit = usage_unit r.units = units + r.cost = cost return r @@ -279,8 +281,8 @@ async def test_detail_breaks_units_down_by_day_team_and_key(): ) assert resp.usage_units == {"contentPolicyUnits": 3, "topicPolicyUnits": 1} assert [p.model_dump() for p in resp.usage_units_daily] == [ - {"date": "2026-04-24", "units": {"topicPolicyUnits": 1}}, - {"date": "2026-04-25", "units": {"contentPolicyUnits": 3}}, + {"date": "2026-04-24", "units": {"topicPolicyUnits": 1}, "cost": None}, + {"date": "2026-04-25", "units": {"contentPolicyUnits": 3}, "cost": None}, ] assert resp.usage_units_by_team == { "team-a": {"contentPolicyUnits": 2, "topicPolicyUnits": 1}, @@ -311,6 +313,73 @@ async def test_overview_degrades_units_to_empty_when_units_table_is_missing(): row = next(r for r in resp.rows if r.id == "yaml-uuid") assert (row.requestsEvaluated, row.usageUnits) == (4, {}) assert (resp.totalRequests, resp.totalBlocked, resp.totalUsageUnits) == (4, 1, {}) + assert (row.cost, resp.totalCost) == (None, None) + + +@pytest.mark.asyncio +async def test_overview_reports_cost_per_row_and_total_summing_only_tracked_days(): + """LIT-5652: cost rides the units rollup. Rows written before the cost column + (or by an unpriced hook) carry NULL and must drop out of the sum rather than + read as $0, and a guardrail with only NULL rows reports None, not 0.0.""" + prisma = _prisma( + find_many=[], + metrics=[_metric("yaml-pii", requests=4, passed=3, blocked=1)], + units=[ + _units_row("yaml-pii", usage_unit="contentPolicyUnits", units=1000, cost=0.15), + _units_row("yaml-pii", team_id="team-a", usage_unit="contentPolicyUnits", units=2000, cost=0.3), + _units_row("yaml-pii", date="2026-04-24", usage_unit="contentPolicyUnits", units=5000, cost=None), + _units_row("legacy-guard", usage_unit="topicPolicyUnits", units=7, cost=None), + ], + ) + handler = _config_handler( + _yaml_guardrail(guardrail_id="yaml-uuid", name="yaml-pii"), + _yaml_guardrail(guardrail_id="legacy-uuid", name="legacy-guard"), + ) + p1, p2 = _patches(prisma, handler) + with p1, p2: + resp = await guardrails_usage_overview(start_date=START, end_date=END, user_api_key_dict=ADMIN) + by_id = {r.id: r for r in resp.rows} + assert by_id["yaml-uuid"].cost == pytest.approx(0.45) + assert by_id["legacy-uuid"].cost is None + assert resp.totalCost == pytest.approx(0.45) + + +@pytest.mark.asyncio +async def test_detail_breaks_cost_down_by_unit_day_team_and_key(): + """Every cost breakdown keeps the same keys as its units twin so the UI can + render them side by side, with None where that group has no tracked cost.""" + prisma = _prisma( + find_unique=None, + units=[ + _units_row("yaml-pii", date="2026-04-25", team_id="team-a", api_key="hash-1", units=1000, cost=0.15), + _units_row("yaml-pii", date="2026-04-25", team_id="", api_key="hash-2", units=200, cost=0.03), + _units_row( + "yaml-pii", + date="2026-04-24", + team_id="team-a", + api_key="hash-1", + usage_unit="topicPolicyUnits", + units=10, + cost=None, + ), + ], + ) + handler = _config_handler(_yaml_guardrail()) + p1, p2 = _patches(prisma, handler) + with p1, p2: + resp = await guardrails_usage_detail( + guardrail_id="yaml-1", start_date=START, end_date=END, user_api_key_dict=ADMIN + ) + assert resp.cost == pytest.approx(0.18) + assert resp.cost_by_unit == {"contentPolicyUnits": pytest.approx(0.18), "topicPolicyUnits": None} + assert [p.model_dump() for p in resp.usage_units_daily] == [ + {"date": "2026-04-24", "units": {"topicPolicyUnits": 10}, "cost": None}, + {"date": "2026-04-25", "units": {"contentPolicyUnits": 1200}, "cost": pytest.approx(0.18)}, + ] + assert resp.cost_by_team == {"team-a": pytest.approx(0.15), "": pytest.approx(0.03)} + assert resp.cost_by_key == {"hash-1": pytest.approx(0.15), "hash-2": pytest.approx(0.03)} + assert resp.cost_by_team.keys() == resp.usage_units_by_team.keys() + assert resp.cost_by_key.keys() == resp.usage_units_by_key.keys() @pytest.mark.asyncio @@ -330,6 +399,7 @@ async def test_detail_degrades_units_to_empty_when_units_table_is_missing(): {}, {}, ) + assert (resp.cost, resp.cost_by_unit, resp.cost_by_team, resp.cost_by_key) == (None, {}, {}, {}) # ---- logs ------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py index 6da121703d7..50845385443 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py @@ -30,6 +30,8 @@ def _payload( api_key: str = "hashed-key-1", usage: dict[str, Any] | None = None, guardrail_status: str = "success", + cost_by_unit: dict[str, Any] | None = None, + cost_in_spend: bool | None = None, ) -> dict[str, Any]: entry: dict[str, Any] = { "guardrail_id": "bedrock-guard", @@ -37,6 +39,10 @@ def _payload( } if usage is not None: entry["guardrail_usage"] = usage + if cost_by_unit is not None: + entry["guardrail_cost_by_unit"] = cost_by_unit + if cost_in_spend is not None: + entry["guardrail_cost_in_spend"] = cost_in_spend return { "request_id": request_id, "startTime": datetime(2026, 8, 17, 12, 0, tzinfo=timezone.utc), @@ -58,6 +64,18 @@ def _units_upserts(prisma: MagicMock) -> dict[tuple, int]: return out +def _cost_upserts(prisma: MagicMock) -> dict[str, tuple[float | None, object]]: + """usage_unit -> (cost written on create, cost clause sent on update).""" + calls = prisma.db.litellm_dailyguardrailusageunits.upsert.call_args_list + return { + c.kwargs["data"]["create"]["usage_unit"]: ( + c.kwargs["data"]["create"]["cost"], + c.kwargs["data"]["update"]["cost"], + ) + for c in calls + } + + @pytest.mark.asyncio async def test_usage_units_rolled_up_by_guardrail_team_key_and_date(): """ @@ -181,7 +199,9 @@ async def test_retry_exhausted_rows_are_requeued_and_land_on_the_next_flush(): down, [_payload("r1", usage={"topicPolicyUnits": 2})], sleep=sleep, pending=pending ) - assert dict(pending.units) == {("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "topicPolicyUnits"): 2} + assert dict(pending.units) == { + ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "topicPolicyUnits"): (2, None) + } recovered = _prisma() await process_spend_logs_guardrail_usage( @@ -320,3 +340,111 @@ async def test_payload_without_request_id_is_skipped_like_the_metrics_path(): ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "topicPolicyUnits"): 1, } assert prisma.db.litellm_dailyguardrailmetrics.upsert.call_args.kwargs["data"]["create"]["requests_evaluated"] == 1 + + +@pytest.mark.asyncio +async def test_cost_rolled_up_per_counter_alongside_units(): + """LIT-5652: the hook's per-counter cost lands on the same daily row as the + units it priced, summed across payloads exactly like the units are, and the + update path increments it so a second flush on the same day keeps adding.""" + prisma = _prisma() + logs = [ + _payload( + "r1", + usage={"contentPolicyUnits": 1000, "wordPolicyUnits": 50}, + cost_by_unit={"contentPolicyUnits": 0.15, "wordPolicyUnits": 0.0}, + ), + _payload( + "r2", + usage={"contentPolicyUnits": 2000, "wordPolicyUnits": 10}, + cost_by_unit={"contentPolicyUnits": 0.3, "wordPolicyUnits": 0.0}, + ), + ] + + await process_spend_logs_guardrail_usage(prisma, logs) + + assert _units_upserts(prisma) == { + ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "contentPolicyUnits"): 3000, + ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "wordPolicyUnits"): 60, + } + costs = _cost_upserts(prisma) + assert costs["contentPolicyUnits"][0] == pytest.approx(0.45) + assert costs["contentPolicyUnits"][1] == {"increment": pytest.approx(0.45)} + assert costs["wordPolicyUnits"] == (0.0, {"increment": 0.0}) + + +@pytest.mark.asyncio +async def test_unpriced_increment_makes_the_rows_cost_unknown_not_partial(): + """A payload with usage but no per-counter cost (a hook without pricing, a + pre-upgrade proxy in a mixed fleet) must poison that row's cost to NULL on + both create and update. Keeping the priced part would understate the day + while looking exact.""" + prisma = _prisma() + logs = [ + _payload("r1", usage={"contentPolicyUnits": 1000}, cost_by_unit={"contentPolicyUnits": 0.15}), + _payload("r2", usage={"contentPolicyUnits": 1000}), + ] + + await process_spend_logs_guardrail_usage(prisma, logs) + + assert _units_upserts(prisma) == { + ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "contentPolicyUnits"): 2000, + } + assert _cost_upserts(prisma) == {"contentPolicyUnits": (None, None)} + + +@pytest.mark.asyncio +async def test_report_only_and_forged_costs_are_not_rolled_up_but_units_are(): + """guardrail_cost_in_spend=False (Azure Prompt Shield) keeps its cost out of + spend, so the rollup must not record it either or the dashboard would show + a number the budget never charged. A negative or non-finite per-counter cost + is treated the same way rather than subtracting from the day.""" + prisma = _prisma() + logs = [ + _payload("r1", usage={"text_records": 3}, cost_by_unit={"text_records": 0.5}, cost_in_spend=False), + _payload("r2", usage={"contentPolicyUnits": 10}, cost_by_unit={"contentPolicyUnits": -0.5}), + _payload("r3", usage={"topicPolicyUnits": 10}, cost_by_unit={"topicPolicyUnits": float("inf")}), + ] + + await process_spend_logs_guardrail_usage(prisma, logs) + + assert _units_upserts(prisma) == { + ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "text_records"): 3, + ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "contentPolicyUnits"): 10, + ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "topicPolicyUnits"): 10, + } + assert _cost_upserts(prisma) == { + "text_records": (None, None), + "contentPolicyUnits": (None, None), + "topicPolicyUnits": (None, None), + } + + +@pytest.mark.asyncio +async def test_requeued_cost_is_added_to_the_next_flush(): + """Cost must survive the connection-error requeue the same way units do, or + a DB blip would silently drop dollars while keeping the units they bought.""" + pending = PendingRollups() + down = _prisma() + down.db.litellm_dailyguardrailmetrics.upsert.side_effect = httpx.ConnectError("db down") + down.db.litellm_dailyguardrailusageunits.upsert.side_effect = httpx.ConnectError("db down") + sleep, _ = _fake_sleep() + + await process_spend_logs_guardrail_usage( + down, + [_payload("r1", usage={"contentPolicyUnits": 1000}, cost_by_unit={"contentPolicyUnits": 0.15})], + sleep=sleep, + pending=pending, + ) + recovered = _prisma() + await process_spend_logs_guardrail_usage( + recovered, + [_payload("r2", usage={"contentPolicyUnits": 2000}, cost_by_unit={"contentPolicyUnits": 0.3})], + sleep=sleep, + pending=pending, + ) + + assert _units_upserts(recovered) == { + ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "contentPolicyUnits"): 3000, + } + assert _cost_upserts(recovered)["contentPolicyUnits"][0] == pytest.approx(0.45) diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 3d2e97d55a5..458db0c9810 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -3,7 +3,7 @@ "limit": 22367 }, "LIT002": { - "limit": 26777 + "limit": 26775 }, "LIT003": { "limit": 269 diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index e944062e15e..a3d8b22a672 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -37791,6 +37791,20 @@ export interface components { avgLatency: number | null; /** Avgscore */ avgScore: number | null; + /** Cost */ + cost: number | null; + /** Cost By Key */ + cost_by_key: { + [key: string]: number | null; + }; + /** Cost By Team */ + cost_by_team: { + [key: string]: number | null; + }; + /** Cost By Unit */ + cost_by_unit: { + [key: string]: number | null; + }; /** Description */ description: string | null; /** Failrate */ @@ -37872,6 +37886,8 @@ export interface components { rows: components["schemas"]["UsageOverviewRow"][]; /** Totalblocked */ totalBlocked: number; + /** Totalcost */ + totalCost: number | null; /** Totalrequests */ totalRequests: number; /** Totalusageunits */ @@ -37885,6 +37901,8 @@ export interface components { avgLatency: number | null; /** Avgscore */ avgScore: number | null; + /** Cost */ + cost: number | null; /** Failrate */ failRate: number; /** Id */ @@ -37908,6 +37926,8 @@ export interface components { }; /** UsageUnitsDailyPoint */ UsageUnitsDailyPoint: { + /** Cost */ + cost: number | null; /** Date */ date: string; /** Units */ From 6f904d4414823783531464192a5b2f09b5071b49 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:55:54 -0700 Subject: [PATCH 02/88] fix(bedrock_mantle): anchor MANTLE_HOST_RE so custom hosts are not rewritten to the public host --- litellm/llms/bedrock_mantle/common_utils.py | 2 +- ...drock_mantle_passthrough_transformation.py | 21 ++++++++++ ...bedrock_mantle_responses_transformation.py | 23 +++++++++++ .../test_bedrock_mantle_transformation.py | 40 +++++++++++++++++++ 4 files changed, 85 insertions(+), 1 deletion(-) diff --git a/litellm/llms/bedrock_mantle/common_utils.py b/litellm/llms/bedrock_mantle/common_utils.py index d877fbb4e09..850738bc320 100644 --- a/litellm/llms/bedrock_mantle/common_utils.py +++ b/litellm/llms/bedrock_mantle/common_utils.py @@ -29,7 +29,7 @@ from litellm.secret_managers.main import get_secret_str BEDROCK_MANTLE_DEFAULT_REGION: Final = "us-east-1" # Standard Mantle host: https://bedrock-mantle..api.aws (group 1 = region). -MANTLE_HOST_RE: Final = re.compile(r"^https?://bedrock-mantle\.([^/.]+)\.api\.aws", re.IGNORECASE) +MANTLE_HOST_RE: Final = re.compile(r"^https?://bedrock-mantle\.([^/.]+)\.api\.aws(?=/|$)", re.IGNORECASE) def resolve_mantle_bearer_token(api_key: str | None) -> str | None: diff --git a/tests/test_litellm/llms/bedrock_mantle/passthrough/test_bedrock_mantle_passthrough_transformation.py b/tests/test_litellm/llms/bedrock_mantle/passthrough/test_bedrock_mantle_passthrough_transformation.py index 8c6eda605ca..090de0a9d3e 100644 --- a/tests/test_litellm/llms/bedrock_mantle/passthrough/test_bedrock_mantle_passthrough_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/passthrough/test_bedrock_mantle_passthrough_transformation.py @@ -68,6 +68,27 @@ def test_explicit_region_and_non_mantle_api_base_are_kept(no_ambient_aws): assert base_url == vpc_endpoint +@pytest.mark.parametrize( + "lookalike_host", + [ + "https://bedrock-mantle.us-east-1.api.aws.internal.example.com", + "https://bedrock-mantle.us-gov-west-1.api.aws-int.example.com", + "https://bedrock-mantle.us-east-1.api.aws:8443", + ], +) +def test_lookalike_mantle_host_api_base_is_kept(no_ambient_aws, lookalike_host): + url, base_url = BedrockMantlePassthroughConfig().get_complete_url( + api_base=lookalike_host, + api_key=None, + model="us.openai.gpt-5.6-sol", + endpoint=INVOKE_ENDPOINT, + request_query_params=None, + litellm_params={"api_base": lookalike_host}, + ) + assert str(url) == f"{lookalike_host}/{INVOKE_ENDPOINT}" + assert base_url == lookalike_host + + def test_region_falls_back_to_the_mantle_default_without_any_hint(no_ambient_aws): url, _ = BedrockMantlePassthroughConfig().get_complete_url( api_base=None, diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index 0033f4467bb..21f72c0b86a 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -26,6 +26,12 @@ from litellm.llms.bedrock_mantle.responses.transformation import ( from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +LOOKALIKE_MANTLE_HOSTS = ( + "https://bedrock-mantle.us-east-1.api.aws.internal.example.com", + "https://bedrock-mantle.us-gov-west-1.api.aws-int.example.com", + "https://bedrock-mantle.us-east-1.api.aws:8443", +) + class TestBedrockMantleResponsesURL: def test_url_uses_region_from_env(self, monkeypatch): @@ -1555,6 +1561,23 @@ class TestBedrockMantleResponsesSigV4: ) assert url == "https://mantle-proxy.internal.example/openai/v1/responses" + @pytest.mark.parametrize("lookalike_host", LOOKALIKE_MANTLE_HOSTS) + def test_lookalike_mantle_host_from_api_base_is_preserved(self, monkeypatch, lookalike_host): + monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) + cfg = BedrockMantleResponsesAPIConfig() + url = cfg.get_complete_url( + api_base=f"{lookalike_host}/openai/v1", + litellm_params={"aws_region_name": "us-east-2"}, + ) + assert url == f"{lookalike_host}/openai/v1/responses" + + @pytest.mark.parametrize("lookalike_host", LOOKALIKE_MANTLE_HOSTS) + def test_lookalike_mantle_host_from_env_is_preserved(self, monkeypatch, lookalike_host): + monkeypatch.setenv("BEDROCK_MANTLE_API_BASE", lookalike_host) + cfg = BedrockMantleResponsesAPIConfig() + url = cfg.get_complete_url(api_base=None, litellm_params={}) + assert url == f"{lookalike_host}/openai/v1/responses" + def test_caller_authorization_does_not_override_sigv4(self, monkeypatch): """Adversarial-review regression: a caller-supplied Authorization header (e.g. from extra_headers, surviving the relaxed validate_environment) must not clobber diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index cd775abf136..a0a707fd7a3 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -399,6 +399,46 @@ class TestBedrockMantleChatAuth: assert "/eu-west-1/bedrock/aws4_request" in headers["Authorization"] assert "/us-west-2/bedrock/aws4_request" not in headers["Authorization"] + @pytest.mark.parametrize( + ("region_params", "env", "expected_region"), + [ + ({"aws_region_name": "us-west-2"}, {}, "us-west-2"), + ({}, {"BEDROCK_MANTLE_REGION": "ap-southeast-2"}, "ap-southeast-2"), + ], + ) + def test_sigv4_scope_ignores_the_region_segment_of_a_lookalike_host( + self, monkeypatch, region_params, env, expected_region + ): + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + for var in ( + "BEDROCK_MANTLE_API_KEY", + "AWS_BEARER_TOKEN_BEDROCK", + "BEDROCK_MANTLE_REGION", + "BEDROCK_MANTLE_API_BASE", + "AWS_REGION", + "AWS_REGION_NAME", + ): + monkeypatch.delenv(var, raising=False) + for var, value in env.items(): + monkeypatch.setenv(var, value) + + cfg = BedrockMantleChatConfig(aws_signer=BaseAWSLLM()) + headers, _ = cfg.sign_request( + headers={}, + optional_params={ + "aws_access_key_id": "AKIAEXAMPLE", + "aws_secret_access_key": "c2VjcmV0LXRlc3Qtc2VjcmV0LXRlc3Qtc2VjcmV0", + **region_params, + }, + request_data={"input": "hi"}, + api_base="https://bedrock-mantle.eu-west-1.api.aws.internal.example.com/openai/v1/chat/completions", + api_key=None, + ) + + assert f"/{expected_region}/bedrock/aws4_request" in headers["Authorization"] + assert "/eu-west-1/bedrock/aws4_request" not in headers["Authorization"] + def test_no_bearer_and_no_credentials_raises_value_error(self, monkeypatch): from unittest.mock import MagicMock From 7a8226e75275ea87b82e501d6a89f6a8981d4d78 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 19:40:53 +0000 Subject: [PATCH 03/88] fix(model_prices): registry audit 2026-09-02, add claude-mythos-5-1 and gpt-daybreak aliases, fix gpt-5.5 Fast and W&B pricing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 153 +++++++++++++++--- model_prices_and_context_window.json | 153 +++++++++++++++--- tests/test_litellm/test_cost_calculator.py | 5 + .../test_daybreak_model_metadata.py | 27 +++- 4 files changed, 301 insertions(+), 37 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2846d12db6e..7cf956edfdd 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -29620,7 +29620,45 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "source": "https://developers.openai.com/api/docs/models/daybreak-red-latest", + "source": "https://developers.openai.com/api/docs/models/gpt-daybreak-red-latest", + "supports_computer_use": true, + "supports_parallel_function_calling": true + }, + "gpt-daybreak-red-latest": { + "cache_creation_input_token_cost": 1.5625e-05, + "cache_creation_input_token_cost_above_272k_tokens": 3.125e-05, + "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_above_272k_tokens": 2.5e-06, + "input_cost_per_token": 1.25e-05, + "input_cost_per_token_above_272k_tokens": 2.5e-05, + "litellm_provider": "openai", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.5e-05, + "output_cost_per_token_above_272k_tokens": 0.0001125, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "source": "https://developers.openai.com/api/docs/models/gpt-daybreak-red-latest", "supports_computer_use": true, "supports_parallel_function_calling": true }, @@ -29660,7 +29698,45 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "source": "https://developers.openai.com/api/docs/models/daybreak-blue-latest", + "source": "https://developers.openai.com/api/docs/models/gpt-daybreak-blue-latest", + "supports_parallel_function_calling": true + }, + "gpt-daybreak-blue-latest": { + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1e-05, + "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8e-07, + "input_cost_per_token": 4e-06, + "input_cost_per_token_above_272k_tokens": 8e-06, + "litellm_provider": "openai", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "output_cost_per_token_above_272k_tokens": 3e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "source": "https://developers.openai.com/api/docs/models/gpt-daybreak-blue-latest", "supports_parallel_function_calling": true }, "chat-latest": { @@ -29699,12 +29775,12 @@ "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_flex": 2.5e-07, - "cache_read_input_token_cost_priority": 1e-06, + "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, - "input_cost_per_token_priority": 1e-05, + "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -29714,7 +29790,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, - "output_cost_per_token_priority": 6e-05, + "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -29756,12 +29832,12 @@ "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_flex": 2.5e-07, - "cache_read_input_token_cost_priority": 1e-06, + "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, - "input_cost_per_token_priority": 1e-05, + "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -29771,7 +29847,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, - "output_cost_per_token_priority": 6e-05, + "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -46311,8 +46387,8 @@ "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, - "input_cost_per_token": 0.01, - "output_cost_per_token": 0.01, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1e-07, "litellm_provider": "wandb", "mode": "chat" }, @@ -46330,8 +46406,8 @@ "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, - "input_cost_per_token": 0.01, - "output_cost_per_token": 0.01, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1e-07, "litellm_provider": "wandb", "mode": "chat" }, @@ -46396,8 +46472,8 @@ "max_tokens": 161000, "max_input_tokens": 161000, "max_output_tokens": 161000, - "input_cost_per_token": 0.135, - "output_cost_per_token": 0.54, + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 5.4e-06, "litellm_provider": "wandb", "mode": "chat" }, @@ -46405,8 +46481,8 @@ "max_tokens": 161000, "max_input_tokens": 161000, "max_output_tokens": 161000, - "input_cost_per_token": 0.114, - "output_cost_per_token": 0.275, + "input_cost_per_token": 1.14e-06, + "output_cost_per_token": 2.75e-06, "litellm_provider": "wandb", "mode": "chat" }, @@ -46424,8 +46500,8 @@ "max_tokens": 64000, "max_input_tokens": 64000, "max_output_tokens": 64000, - "input_cost_per_token": 0.017, - "output_cost_per_token": 0.066, + "input_cost_per_token": 1.7e-07, + "output_cost_per_token": 6.6e-07, "litellm_provider": "wandb", "mode": "chat" }, @@ -54419,6 +54495,47 @@ "us": 1.1 } }, + "claude-mythos-5-1": { + "deprecation_date": "2027-09-01", + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 512, + "supports_native_structured_output": true, + "source": "https://platform.claude.com/docs/en/models/mythos-5-1/overview" + }, "claude-mythos-preview": { "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2846d12db6e..7cf956edfdd 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -29620,7 +29620,45 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "source": "https://developers.openai.com/api/docs/models/daybreak-red-latest", + "source": "https://developers.openai.com/api/docs/models/gpt-daybreak-red-latest", + "supports_computer_use": true, + "supports_parallel_function_calling": true + }, + "gpt-daybreak-red-latest": { + "cache_creation_input_token_cost": 1.5625e-05, + "cache_creation_input_token_cost_above_272k_tokens": 3.125e-05, + "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_above_272k_tokens": 2.5e-06, + "input_cost_per_token": 1.25e-05, + "input_cost_per_token_above_272k_tokens": 2.5e-05, + "litellm_provider": "openai", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.5e-05, + "output_cost_per_token_above_272k_tokens": 0.0001125, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "source": "https://developers.openai.com/api/docs/models/gpt-daybreak-red-latest", "supports_computer_use": true, "supports_parallel_function_calling": true }, @@ -29660,7 +29698,45 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "source": "https://developers.openai.com/api/docs/models/daybreak-blue-latest", + "source": "https://developers.openai.com/api/docs/models/gpt-daybreak-blue-latest", + "supports_parallel_function_calling": true + }, + "gpt-daybreak-blue-latest": { + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1e-05, + "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8e-07, + "input_cost_per_token": 4e-06, + "input_cost_per_token_above_272k_tokens": 8e-06, + "litellm_provider": "openai", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "output_cost_per_token_above_272k_tokens": 3e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "source": "https://developers.openai.com/api/docs/models/gpt-daybreak-blue-latest", "supports_parallel_function_calling": true }, "chat-latest": { @@ -29699,12 +29775,12 @@ "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_flex": 2.5e-07, - "cache_read_input_token_cost_priority": 1e-06, + "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, - "input_cost_per_token_priority": 1e-05, + "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -29714,7 +29790,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, - "output_cost_per_token_priority": 6e-05, + "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -29756,12 +29832,12 @@ "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_flex": 2.5e-07, - "cache_read_input_token_cost_priority": 1e-06, + "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, - "input_cost_per_token_priority": 1e-05, + "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -29771,7 +29847,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, - "output_cost_per_token_priority": 6e-05, + "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -46311,8 +46387,8 @@ "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, - "input_cost_per_token": 0.01, - "output_cost_per_token": 0.01, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1e-07, "litellm_provider": "wandb", "mode": "chat" }, @@ -46330,8 +46406,8 @@ "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, - "input_cost_per_token": 0.01, - "output_cost_per_token": 0.01, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1e-07, "litellm_provider": "wandb", "mode": "chat" }, @@ -46396,8 +46472,8 @@ "max_tokens": 161000, "max_input_tokens": 161000, "max_output_tokens": 161000, - "input_cost_per_token": 0.135, - "output_cost_per_token": 0.54, + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 5.4e-06, "litellm_provider": "wandb", "mode": "chat" }, @@ -46405,8 +46481,8 @@ "max_tokens": 161000, "max_input_tokens": 161000, "max_output_tokens": 161000, - "input_cost_per_token": 0.114, - "output_cost_per_token": 0.275, + "input_cost_per_token": 1.14e-06, + "output_cost_per_token": 2.75e-06, "litellm_provider": "wandb", "mode": "chat" }, @@ -46424,8 +46500,8 @@ "max_tokens": 64000, "max_input_tokens": 64000, "max_output_tokens": 64000, - "input_cost_per_token": 0.017, - "output_cost_per_token": 0.066, + "input_cost_per_token": 1.7e-07, + "output_cost_per_token": 6.6e-07, "litellm_provider": "wandb", "mode": "chat" }, @@ -54419,6 +54495,47 @@ "us": 1.1 } }, + "claude-mythos-5-1": { + "deprecation_date": "2027-09-01", + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 512, + "supports_native_structured_output": true, + "source": "https://platform.claude.com/docs/en/models/mythos-5-1/overview" + }, "claude-mythos-preview": { "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 7c2174018e8..862d77f01b2 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -175,6 +175,11 @@ def test_wandb_model_api_pricing_entries(_local_model_cost_map): expected_pricing = { "wandb/moonshotai/Kimi-K2.5": (6e-07, 3e-06), "wandb/MiniMaxAI/MiniMax-M2.5": (3e-07, 1.2e-06), + "wandb/Qwen/Qwen3-235B-A22B-Instruct-2507": (1e-07, 1e-07), + "wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": (1e-07, 1e-07), + "wandb/deepseek-ai/DeepSeek-R1-0528": (1.35e-06, 5.4e-06), + "wandb/deepseek-ai/DeepSeek-V3-0324": (1.14e-06, 2.75e-06), + "wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": (1.7e-07, 6.6e-07), } for model_name, (input_cost, output_cost) in expected_pricing.items(): diff --git a/tests/test_litellm/test_daybreak_model_metadata.py b/tests/test_litellm/test_daybreak_model_metadata.py index d04cca3c077..068bc01e103 100644 --- a/tests/test_litellm/test_daybreak_model_metadata.py +++ b/tests/test_litellm/test_daybreak_model_metadata.py @@ -14,6 +14,17 @@ DAYBREAK_MODELS = ( ) BLUE_ALIAS = "daybreak-blue-latest" BLUE_SNAPSHOT = "gpt-5.6-sol" +OFFICIAL_ALIAS_SNAPSHOTS = ( + ("gpt-daybreak-blue-latest", "gpt-5.6-sol"), + ("gpt-daybreak-red-latest", "gpt-5.6-cyber"), +) +PRICE_FIELDS = ( + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + "input_cost_per_token_above_272k_tokens", + "output_cost_per_token_above_272k_tokens", +) def _load(path): @@ -44,7 +55,21 @@ def test_blue_alias_matches_its_snapshot_computer_use(): assert cost_map[BLUE_SNAPSHOT]["supports_computer_use"] is True -@pytest.mark.parametrize("model", (*DAYBREAK_MODELS, BLUE_SNAPSHOT)) +@pytest.mark.parametrize(("alias", "snapshot"), OFFICIAL_ALIAS_SNAPSHOTS) +def test_official_alias_tracks_snapshot(alias, snapshot): + cost_map = _load(MAIN_PATH) + alias_info = cost_map[alias] + snapshot_info = cost_map[snapshot] + + assert alias_info["supported_endpoints"] == ["/v1/responses"] + assert alias_info["source"] == f"https://developers.openai.com/api/docs/models/{alias}" + assert {field: alias_info.get(field) for field in PRICE_FIELDS} == { + field: snapshot_info.get(field) for field in PRICE_FIELDS + } + assert alias_info["max_output_tokens"] == snapshot_info["max_output_tokens"] + + +@pytest.mark.parametrize("model", (*DAYBREAK_MODELS, BLUE_SNAPSHOT, *(alias for alias, _ in OFFICIAL_ALIAS_SNAPSHOTS))) def test_backup_matches_main(model): main_cost = _load(MAIN_PATH) backup_cost = _load(BACKUP_PATH) From 9c5b20abdd7978c3b6d52b9dff1f8703049ff4dc Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 20:07:21 +0000 Subject: [PATCH 04/88] fix(model_prices): add Nebius, watsonx and Volcengine models and correct watsonx list prices Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 475 ++++++++++++++++-- model_prices_and_context_window.json | 475 ++++++++++++++++-- 2 files changed, 856 insertions(+), 94 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7cf956edfdd..a6967c12c94 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -35226,16 +35226,16 @@ "source": "https://nebius.com/prices" }, "nebius/google/gemma-3-27b-it": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 6e-08, - "output_cost_per_token": 2e-07, + "max_tokens": 110000, + "max_input_tokens": 110000, + "max_output_tokens": 110000, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://nebius.com/prices" + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/google%2Fgemma-3-27b-it" }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { "max_tokens": 128000, @@ -35347,15 +35347,15 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-32B": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, + "max_tokens": 40960, + "max_input_tokens": 40960, + "max_output_tokens": 40960, "input_cost_per_token": 1e-07, "output_cost_per_token": 3e-07, "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices" + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-32B" }, "nebius/Qwen/Qwen3-30B-A3B": { "max_tokens": 32768, @@ -35436,16 +35436,16 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-VL-72B-Instruct": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 1.3e-07, - "output_cost_per_token": 4e-07, + "max_tokens": 32000, + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 7.5e-07, "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://nebius.com/prices" + "source": "https://tokenfactory.nebius.com/models/catalog/image2text/Qwen%2FQwen2.5-VL-72B-Instruct" }, "nebius/Qwen/Qwen2-VL-72B-Instruct": { "max_tokens": 131072, @@ -35470,6 +35470,320 @@ "supports_vision": true, "source": "https://nebius.com/prices" }, + "nebius/deepseek-ai/DeepSeek-V4-Flash": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash" + }, + "nebius/deepseek-ai/DeepSeek-V4-Flash-0731": { + "max_tokens": 1024000, + "max_input_tokens": 1024000, + "max_output_tokens": 1024000, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash-0731" + }, + "nebius/deepseek-ai/DeepSeek-V4-Pro": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 1.75e-06, + "output_cost_per_token": 3.5e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro" + }, + "nebius/MiniMaxAI/MiniMax-M2.5": { + "max_tokens": 196608, + "max_input_tokens": 196608, + "max_output_tokens": 196608, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M2.5" + }, + "nebius/MiniMaxAI/MiniMax-M3": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M3" + }, + "nebius/moonshotai/Kimi-K2.6": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 9.5e-07, + "output_cost_per_token": 4e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K2.6" + }, + "nebius/moonshotai/Kimi-K2.7-Code": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 9.5e-07, + "output_cost_per_token": 4e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/moonshotai%2FKimi-K2.7-Code" + }, + "nebius/moonshotai/Kimi-K3": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K3" + }, + "nebius/NousResearch/Hermes-4-405B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-405B" + }, + "nebius/NousResearch/Hermes-4-70B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-70B" + }, + "nebius/nvidia/Cosmos3-Super-Reasoner": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://tokenfactory.nebius.com/models/catalog/image2text/nvidia%2FCosmos3-Super-Reasoner" + }, + "nebius/nvidia/Llama-3_1-Nemotron-Ultra-253B-v1": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FLlama-3_1-Nemotron-Ultra-253B-v1" + }, + "nebius/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 2.4e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNVIDIA-Nemotron-3-Nano-30B-A3B" + }, + "nebius/nvidia/Nemotron-3-Nano-Omni": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 2.4e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Nano-Omni" + }, + "nebius/nvidia/nemotron-3-super-120b-a12b": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 9e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2Fnemotron-3-super-120b-a12b" + }, + "nebius/nvidia/Nemotron-3-Ultra-550b-a55b": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Ultra-550b-a55b" + }, + "nebius/nvidia/Nemotron-3_5-Lightning": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 2.4e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3_5-Lightning" + }, + "nebius/openai/gpt-oss-120b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/openai%2Fgpt-oss-120b" + }, + "nebius/openbmb/MiniCPM-V-4_5": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "input_cost_per_token": 6.58e-07, + "output_cost_per_token": 1.11e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "source": "https://tokenfactory.nebius.com/models/catalog/image2text/openbmb%2FMiniCPM-V-4_5" + }, + "nebius/Qwen/Qwen3-235B-A22B-Instruct-2507": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-235B-A22B-Instruct-2507" + }, + "nebius/Qwen/Qwen3-30B-A3B-Instruct-2507": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-30B-A3B-Instruct-2507" + }, + "nebius/Qwen/Qwen3-Next-80B-A3B-Thinking": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-Next-80B-A3B-Thinking" + }, + "nebius/Qwen/Qwen3.5-397B-A17B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 3.6e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.5-397B-A17B" + }, + "nebius/zai-org/GLM-5.1": { + "max_tokens": 202752, + "max_input_tokens": 202752, + "max_output_tokens": 202752, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.1" + }, + "nebius/zai-org/GLM-5.2": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.2" + }, + "nebius/zai-org/GLM-5.3-Flash": { + "max_tokens": 1024000, + "max_input_tokens": 1024000, + "max_output_tokens": 1024000, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3-Flash" + }, "nebius/BAAI/bge-en-icl": { "max_tokens": 32768, "max_input_tokens": 32768, @@ -35497,6 +35811,15 @@ "mode": "embedding", "source": "https://nebius.com/prices" }, + "nebius/Qwen/Qwen3-Embedding-8B": { + "max_tokens": 40960, + "max_input_tokens": 40960, + "input_cost_per_token": 1e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "nebius", + "mode": "embedding", + "source": "https://tokenfactory.nebius.com/models/catalog/embedding/Qwen%2FQwen3-Embedding-8B" + }, "nvidia.nemotron-nano-12b-v2": { "input_cost_per_token": 2e-07, "litellm_provider": "bedrock_converse", @@ -46551,16 +46874,30 @@ "supports_vision": false }, "watsonx/bigscience/mt0-xxl-13b": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 0.0005, - "output_cost_per_token": 0.002, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 1.908e-06, + "output_cost_per_token": 1.908e-06, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": false, "supports_parallel_function_calling": false, - "supports_vision": false + "supports_vision": false, + "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" + }, + "watsonx/bigscience/mt0-xxl": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 1.908e-06, + "output_cost_per_token": 1.908e-06, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false, + "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/core42/jais-13b-chat": { "max_tokens": 8192, @@ -46623,16 +46960,17 @@ "supports_vision": false }, "watsonx/ibm/granite-4-h-small": { - "max_tokens": 20480, - "max_input_tokens": 20480, + "max_tokens": 131072, + "max_input_tokens": 131072, "max_output_tokens": 20480, - "input_cost_per_token": 6e-08, - "output_cost_per_token": 2.5e-07, + "input_cost_per_token": 6.36e-08, + "output_cost_per_token": 2.65e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": false + "supports_vision": false, + "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/ibm/granite-guardian-3-2-2b": { "max_tokens": 8192, @@ -46755,28 +47093,43 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-3-70b-instruct": { - "max_tokens": 128000, - "max_input_tokens": 128000, + "max_tokens": 131072, + "max_input_tokens": 131072, "max_output_tokens": 128000, - "input_cost_per_token": 7.1e-07, - "output_cost_per_token": 7.1e-07, + "input_cost_per_token": 7.526e-07, + "output_cost_per_token": 7.526e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": false + "supports_vision": false, + "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b": { - "max_tokens": 128000, - "max_input_tokens": 128000, + "max_tokens": 131072, + "max_input_tokens": 131072, "max_output_tokens": 128000, - "input_cost_per_token": 3.5e-07, - "output_cost_per_token": 1.4e-06, + "input_cost_per_token": 3.71e-07, + "output_cost_per_token": 1.484e-06, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": false + "supports_vision": false, + "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" + }, + "watsonx/meta-llama/llama-4-maverick-17b-128e-instruct-fp8": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 128000, + "input_cost_per_token": 3.71e-07, + "output_cost_per_token": 1.484e-06, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-guard-3-11b-vision": { "max_tokens": 128000, @@ -46815,16 +47168,17 @@ "supports_vision": false }, "watsonx/mistralai/mistral-small-3-1-24b-instruct-2503": { - "max_tokens": 32000, - "max_input_tokens": 32000, + "max_tokens": 131072, + "max_input_tokens": 131072, "max_output_tokens": 32000, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 3e-07, + "input_cost_per_token": 1.06e-07, + "output_cost_per_token": 3.18e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": false + "supports_vision": false, + "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/mistralai/pixtral-12b-2409": { "max_tokens": 128000, @@ -46839,16 +47193,17 @@ "supports_vision": true }, "watsonx/openai/gpt-oss-120b": { - "max_tokens": 8192, - "max_input_tokens": 8192, + "max_tokens": 131072, + "max_input_tokens": 131072, "max_output_tokens": 8192, - "input_cost_per_token": 1.5e-07, - "output_cost_per_token": 6e-07, + "input_cost_per_token": 1.59e-07, + "output_cost_per_token": 6.36e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": false, "supports_parallel_function_calling": false, - "supports_vision": false + "supports_vision": false, + "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/sdaia/allam-1-13b-instruct": { "max_tokens": 8192, @@ -53520,6 +53875,32 @@ } ] }, + "volcengine/doubao-seed-2-1-pro-260628": { + "litellm_provider": "volcengine", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "source": "https://www.volcengine.com/docs/82379/1330310", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": false, + "supports_vision": true + }, + "volcengine/doubao-seed-2-1-turbo-260628": { + "litellm_provider": "volcengine", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "source": "https://www.volcengine.com/docs/82379/1330310", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": false, + "supports_vision": true + }, "bedrock/us-east-1/zai.glm-5": { "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7cf956edfdd..a6967c12c94 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -35226,16 +35226,16 @@ "source": "https://nebius.com/prices" }, "nebius/google/gemma-3-27b-it": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 6e-08, - "output_cost_per_token": 2e-07, + "max_tokens": 110000, + "max_input_tokens": 110000, + "max_output_tokens": 110000, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://nebius.com/prices" + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/google%2Fgemma-3-27b-it" }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { "max_tokens": 128000, @@ -35347,15 +35347,15 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-32B": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, + "max_tokens": 40960, + "max_input_tokens": 40960, + "max_output_tokens": 40960, "input_cost_per_token": 1e-07, "output_cost_per_token": 3e-07, "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices" + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-32B" }, "nebius/Qwen/Qwen3-30B-A3B": { "max_tokens": 32768, @@ -35436,16 +35436,16 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-VL-72B-Instruct": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 1.3e-07, - "output_cost_per_token": 4e-07, + "max_tokens": 32000, + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 7.5e-07, "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://nebius.com/prices" + "source": "https://tokenfactory.nebius.com/models/catalog/image2text/Qwen%2FQwen2.5-VL-72B-Instruct" }, "nebius/Qwen/Qwen2-VL-72B-Instruct": { "max_tokens": 131072, @@ -35470,6 +35470,320 @@ "supports_vision": true, "source": "https://nebius.com/prices" }, + "nebius/deepseek-ai/DeepSeek-V4-Flash": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash" + }, + "nebius/deepseek-ai/DeepSeek-V4-Flash-0731": { + "max_tokens": 1024000, + "max_input_tokens": 1024000, + "max_output_tokens": 1024000, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash-0731" + }, + "nebius/deepseek-ai/DeepSeek-V4-Pro": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 1.75e-06, + "output_cost_per_token": 3.5e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro" + }, + "nebius/MiniMaxAI/MiniMax-M2.5": { + "max_tokens": 196608, + "max_input_tokens": 196608, + "max_output_tokens": 196608, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M2.5" + }, + "nebius/MiniMaxAI/MiniMax-M3": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M3" + }, + "nebius/moonshotai/Kimi-K2.6": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 9.5e-07, + "output_cost_per_token": 4e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K2.6" + }, + "nebius/moonshotai/Kimi-K2.7-Code": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 9.5e-07, + "output_cost_per_token": 4e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/moonshotai%2FKimi-K2.7-Code" + }, + "nebius/moonshotai/Kimi-K3": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K3" + }, + "nebius/NousResearch/Hermes-4-405B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-405B" + }, + "nebius/NousResearch/Hermes-4-70B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-70B" + }, + "nebius/nvidia/Cosmos3-Super-Reasoner": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://tokenfactory.nebius.com/models/catalog/image2text/nvidia%2FCosmos3-Super-Reasoner" + }, + "nebius/nvidia/Llama-3_1-Nemotron-Ultra-253B-v1": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FLlama-3_1-Nemotron-Ultra-253B-v1" + }, + "nebius/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 2.4e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNVIDIA-Nemotron-3-Nano-30B-A3B" + }, + "nebius/nvidia/Nemotron-3-Nano-Omni": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 2.4e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Nano-Omni" + }, + "nebius/nvidia/nemotron-3-super-120b-a12b": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 9e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2Fnemotron-3-super-120b-a12b" + }, + "nebius/nvidia/Nemotron-3-Ultra-550b-a55b": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Ultra-550b-a55b" + }, + "nebius/nvidia/Nemotron-3_5-Lightning": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 2.4e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3_5-Lightning" + }, + "nebius/openai/gpt-oss-120b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/openai%2Fgpt-oss-120b" + }, + "nebius/openbmb/MiniCPM-V-4_5": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "input_cost_per_token": 6.58e-07, + "output_cost_per_token": 1.11e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "source": "https://tokenfactory.nebius.com/models/catalog/image2text/openbmb%2FMiniCPM-V-4_5" + }, + "nebius/Qwen/Qwen3-235B-A22B-Instruct-2507": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-235B-A22B-Instruct-2507" + }, + "nebius/Qwen/Qwen3-30B-A3B-Instruct-2507": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-30B-A3B-Instruct-2507" + }, + "nebius/Qwen/Qwen3-Next-80B-A3B-Thinking": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-Next-80B-A3B-Thinking" + }, + "nebius/Qwen/Qwen3.5-397B-A17B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 3.6e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.5-397B-A17B" + }, + "nebius/zai-org/GLM-5.1": { + "max_tokens": 202752, + "max_input_tokens": 202752, + "max_output_tokens": 202752, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.1" + }, + "nebius/zai-org/GLM-5.2": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.2" + }, + "nebius/zai-org/GLM-5.3-Flash": { + "max_tokens": 1024000, + "max_input_tokens": 1024000, + "max_output_tokens": 1024000, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3-Flash" + }, "nebius/BAAI/bge-en-icl": { "max_tokens": 32768, "max_input_tokens": 32768, @@ -35497,6 +35811,15 @@ "mode": "embedding", "source": "https://nebius.com/prices" }, + "nebius/Qwen/Qwen3-Embedding-8B": { + "max_tokens": 40960, + "max_input_tokens": 40960, + "input_cost_per_token": 1e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "nebius", + "mode": "embedding", + "source": "https://tokenfactory.nebius.com/models/catalog/embedding/Qwen%2FQwen3-Embedding-8B" + }, "nvidia.nemotron-nano-12b-v2": { "input_cost_per_token": 2e-07, "litellm_provider": "bedrock_converse", @@ -46551,16 +46874,30 @@ "supports_vision": false }, "watsonx/bigscience/mt0-xxl-13b": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 0.0005, - "output_cost_per_token": 0.002, + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 1.908e-06, + "output_cost_per_token": 1.908e-06, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": false, "supports_parallel_function_calling": false, - "supports_vision": false + "supports_vision": false, + "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" + }, + "watsonx/bigscience/mt0-xxl": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 1.908e-06, + "output_cost_per_token": 1.908e-06, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false, + "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/core42/jais-13b-chat": { "max_tokens": 8192, @@ -46623,16 +46960,17 @@ "supports_vision": false }, "watsonx/ibm/granite-4-h-small": { - "max_tokens": 20480, - "max_input_tokens": 20480, + "max_tokens": 131072, + "max_input_tokens": 131072, "max_output_tokens": 20480, - "input_cost_per_token": 6e-08, - "output_cost_per_token": 2.5e-07, + "input_cost_per_token": 6.36e-08, + "output_cost_per_token": 2.65e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": false + "supports_vision": false, + "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/ibm/granite-guardian-3-2-2b": { "max_tokens": 8192, @@ -46755,28 +47093,43 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-3-70b-instruct": { - "max_tokens": 128000, - "max_input_tokens": 128000, + "max_tokens": 131072, + "max_input_tokens": 131072, "max_output_tokens": 128000, - "input_cost_per_token": 7.1e-07, - "output_cost_per_token": 7.1e-07, + "input_cost_per_token": 7.526e-07, + "output_cost_per_token": 7.526e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": false + "supports_vision": false, + "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b": { - "max_tokens": 128000, - "max_input_tokens": 128000, + "max_tokens": 131072, + "max_input_tokens": 131072, "max_output_tokens": 128000, - "input_cost_per_token": 3.5e-07, - "output_cost_per_token": 1.4e-06, + "input_cost_per_token": 3.71e-07, + "output_cost_per_token": 1.484e-06, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": false + "supports_vision": false, + "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" + }, + "watsonx/meta-llama/llama-4-maverick-17b-128e-instruct-fp8": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 128000, + "input_cost_per_token": 3.71e-07, + "output_cost_per_token": 1.484e-06, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-guard-3-11b-vision": { "max_tokens": 128000, @@ -46815,16 +47168,17 @@ "supports_vision": false }, "watsonx/mistralai/mistral-small-3-1-24b-instruct-2503": { - "max_tokens": 32000, - "max_input_tokens": 32000, + "max_tokens": 131072, + "max_input_tokens": 131072, "max_output_tokens": 32000, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 3e-07, + "input_cost_per_token": 1.06e-07, + "output_cost_per_token": 3.18e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": false + "supports_vision": false, + "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/mistralai/pixtral-12b-2409": { "max_tokens": 128000, @@ -46839,16 +47193,17 @@ "supports_vision": true }, "watsonx/openai/gpt-oss-120b": { - "max_tokens": 8192, - "max_input_tokens": 8192, + "max_tokens": 131072, + "max_input_tokens": 131072, "max_output_tokens": 8192, - "input_cost_per_token": 1.5e-07, - "output_cost_per_token": 6e-07, + "input_cost_per_token": 1.59e-07, + "output_cost_per_token": 6.36e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": false, "supports_parallel_function_calling": false, - "supports_vision": false + "supports_vision": false, + "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/sdaia/allam-1-13b-instruct": { "max_tokens": 8192, @@ -53520,6 +53875,32 @@ } ] }, + "volcengine/doubao-seed-2-1-pro-260628": { + "litellm_provider": "volcengine", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "source": "https://www.volcengine.com/docs/82379/1330310", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": false, + "supports_vision": true + }, + "volcengine/doubao-seed-2-1-turbo-260628": { + "litellm_provider": "volcengine", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "source": "https://www.volcengine.com/docs/82379/1330310", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": false, + "supports_vision": true + }, "bedrock/us-east-1/zai.glm-5": { "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, From 60ffde65e0a1929c372023333af302a7793c119b Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 20:08:43 +0000 Subject: [PATCH 05/88] fix(model_prices): drop unpriced Volcengine Seed 2.1 entries, they would record zero spend Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 26 ------------------- model_prices_and_context_window.json | 26 ------------------- 2 files changed, 52 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a6967c12c94..f6316fba884 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -53875,32 +53875,6 @@ } ] }, - "volcengine/doubao-seed-2-1-pro-260628": { - "litellm_provider": "volcengine", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": false, - "supports_vision": true - }, - "volcengine/doubao-seed-2-1-turbo-260628": { - "litellm_provider": "volcengine", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": false, - "supports_vision": true - }, "bedrock/us-east-1/zai.glm-5": { "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a6967c12c94..f6316fba884 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -53875,32 +53875,6 @@ } ] }, - "volcengine/doubao-seed-2-1-pro-260628": { - "litellm_provider": "volcengine", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": false, - "supports_vision": true - }, - "volcengine/doubao-seed-2-1-turbo-260628": { - "litellm_provider": "volcengine", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": false, - "supports_vision": true - }, "bedrock/us-east-1/zai.glm-5": { "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, From 211f5d2d102a5f18f43e529c4f8510974e06f9b4 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 20:19:31 +0000 Subject: [PATCH 06/88] test(savings): update gpt-5.5 priority baseline to the published 2.5x fast-mode rate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/proxy/spend_tracking/test_savings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 7dd18587df3..3f775d82b7f 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -852,7 +852,7 @@ def test_the_served_arm_is_read_from_the_record_not_repriced(): @pytest.mark.parametrize( "basis, expected_multiplier", [ - pytest.param({"service_tier": "priority"}, 2.0, id="priority tier doubles the baseline"), + pytest.param({"service_tier": "priority"}, 2.5, id="priority tier uplifts the baseline"), pytest.param({"data_residency": "eu"}, 1.1, id="eu residency uplifts the baseline"), pytest.param({}, 1.0, id="no basis recorded prices at standard"), pytest.param(None, 1.0, id="row predating the field prices at standard"), @@ -872,7 +872,8 @@ def test_the_baseline_is_priced_on_the_basis_the_request_was_billed_at(basis, ex """ gpt = litellm.get_model_info("gpt-5.5", "openai") haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") - assert gpt.get("input_cost_per_token_priority") == 2 * gpt["input_cost_per_token"] + assert gpt.get("input_cost_per_token_priority") == pytest.approx(2.5 * gpt["input_cost_per_token"]) + assert gpt.get("output_cost_per_token_priority") == pytest.approx(2.5 * gpt["output_cost_per_token"]) assert gpt.get("regional_processing_uplift_multiplier_eu") == 1.1 assert haiku.get("input_cost_per_token_priority") is None, "served model must not move with the basis" assert haiku.get("regional_processing_uplift_multiplier_eu") is None From 671559e591ccbc24d540e9bf9be76f8631f59f16 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 20:35:29 +0000 Subject: [PATCH 07/88] fix(model_prices): set watsonx max_tokens equal to max_output_tokens per registry convention Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 12 ++++++------ model_prices_and_context_window.json | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f6316fba884..2e717b52b28 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -46960,7 +46960,7 @@ "supports_vision": false }, "watsonx/ibm/granite-4-h-small": { - "max_tokens": 131072, + "max_tokens": 20480, "max_input_tokens": 131072, "max_output_tokens": 20480, "input_cost_per_token": 6.36e-08, @@ -47093,7 +47093,7 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-3-70b-instruct": { - "max_tokens": 131072, + "max_tokens": 128000, "max_input_tokens": 131072, "max_output_tokens": 128000, "input_cost_per_token": 7.526e-07, @@ -47106,7 +47106,7 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b": { - "max_tokens": 131072, + "max_tokens": 128000, "max_input_tokens": 131072, "max_output_tokens": 128000, "input_cost_per_token": 3.71e-07, @@ -47119,7 +47119,7 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b-128e-instruct-fp8": { - "max_tokens": 131072, + "max_tokens": 128000, "max_input_tokens": 131072, "max_output_tokens": 128000, "input_cost_per_token": 3.71e-07, @@ -47168,7 +47168,7 @@ "supports_vision": false }, "watsonx/mistralai/mistral-small-3-1-24b-instruct-2503": { - "max_tokens": 131072, + "max_tokens": 32000, "max_input_tokens": 131072, "max_output_tokens": 32000, "input_cost_per_token": 1.06e-07, @@ -47193,7 +47193,7 @@ "supports_vision": true }, "watsonx/openai/gpt-oss-120b": { - "max_tokens": 131072, + "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 1.59e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f6316fba884..2e717b52b28 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -46960,7 +46960,7 @@ "supports_vision": false }, "watsonx/ibm/granite-4-h-small": { - "max_tokens": 131072, + "max_tokens": 20480, "max_input_tokens": 131072, "max_output_tokens": 20480, "input_cost_per_token": 6.36e-08, @@ -47093,7 +47093,7 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-3-70b-instruct": { - "max_tokens": 131072, + "max_tokens": 128000, "max_input_tokens": 131072, "max_output_tokens": 128000, "input_cost_per_token": 7.526e-07, @@ -47106,7 +47106,7 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b": { - "max_tokens": 131072, + "max_tokens": 128000, "max_input_tokens": 131072, "max_output_tokens": 128000, "input_cost_per_token": 3.71e-07, @@ -47119,7 +47119,7 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b-128e-instruct-fp8": { - "max_tokens": 131072, + "max_tokens": 128000, "max_input_tokens": 131072, "max_output_tokens": 128000, "input_cost_per_token": 3.71e-07, @@ -47168,7 +47168,7 @@ "supports_vision": false }, "watsonx/mistralai/mistral-small-3-1-24b-instruct-2503": { - "max_tokens": 131072, + "max_tokens": 32000, "max_input_tokens": 131072, "max_output_tokens": 32000, "input_cost_per_token": 1.06e-07, @@ -47193,7 +47193,7 @@ "supports_vision": true }, "watsonx/openai/gpt-oss-120b": { - "max_tokens": 131072, + "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 1.59e-07, From e148868f0c773ef933bba9d84cc3e2d34c572554 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 20:38:45 +0000 Subject: [PATCH 08/88] fix(model_prices): set watsonx max_output_tokens from IBM's documented maximum new tokens Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 24 +++++++++---------- model_prices_and_context_window.json | 24 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2e717b52b28..1935bc97c3b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -46960,9 +46960,9 @@ "supports_vision": false }, "watsonx/ibm/granite-4-h-small": { - "max_tokens": 20480, + "max_tokens": 131072, "max_input_tokens": 131072, - "max_output_tokens": 20480, + "max_output_tokens": 131072, "input_cost_per_token": 6.36e-08, "output_cost_per_token": 2.65e-07, "litellm_provider": "watsonx", @@ -47093,9 +47093,9 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-3-70b-instruct": { - "max_tokens": 128000, + "max_tokens": 131072, "max_input_tokens": 131072, - "max_output_tokens": 128000, + "max_output_tokens": 131072, "input_cost_per_token": 7.526e-07, "output_cost_per_token": 7.526e-07, "litellm_provider": "watsonx", @@ -47106,9 +47106,9 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b": { - "max_tokens": 128000, + "max_tokens": 8192, "max_input_tokens": 131072, - "max_output_tokens": 128000, + "max_output_tokens": 8192, "input_cost_per_token": 3.71e-07, "output_cost_per_token": 1.484e-06, "litellm_provider": "watsonx", @@ -47119,9 +47119,9 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b-128e-instruct-fp8": { - "max_tokens": 128000, + "max_tokens": 8192, "max_input_tokens": 131072, - "max_output_tokens": 128000, + "max_output_tokens": 8192, "input_cost_per_token": 3.71e-07, "output_cost_per_token": 1.484e-06, "litellm_provider": "watsonx", @@ -47168,9 +47168,9 @@ "supports_vision": false }, "watsonx/mistralai/mistral-small-3-1-24b-instruct-2503": { - "max_tokens": 32000, + "max_tokens": 16384, "max_input_tokens": 131072, - "max_output_tokens": 32000, + "max_output_tokens": 16384, "input_cost_per_token": 1.06e-07, "output_cost_per_token": 3.18e-07, "litellm_provider": "watsonx", @@ -47193,9 +47193,9 @@ "supports_vision": true }, "watsonx/openai/gpt-oss-120b": { - "max_tokens": 8192, + "max_tokens": 131072, "max_input_tokens": 131072, - "max_output_tokens": 8192, + "max_output_tokens": 131072, "input_cost_per_token": 1.59e-07, "output_cost_per_token": 6.36e-07, "litellm_provider": "watsonx", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2e717b52b28..1935bc97c3b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -46960,9 +46960,9 @@ "supports_vision": false }, "watsonx/ibm/granite-4-h-small": { - "max_tokens": 20480, + "max_tokens": 131072, "max_input_tokens": 131072, - "max_output_tokens": 20480, + "max_output_tokens": 131072, "input_cost_per_token": 6.36e-08, "output_cost_per_token": 2.65e-07, "litellm_provider": "watsonx", @@ -47093,9 +47093,9 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-3-70b-instruct": { - "max_tokens": 128000, + "max_tokens": 131072, "max_input_tokens": 131072, - "max_output_tokens": 128000, + "max_output_tokens": 131072, "input_cost_per_token": 7.526e-07, "output_cost_per_token": 7.526e-07, "litellm_provider": "watsonx", @@ -47106,9 +47106,9 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b": { - "max_tokens": 128000, + "max_tokens": 8192, "max_input_tokens": 131072, - "max_output_tokens": 128000, + "max_output_tokens": 8192, "input_cost_per_token": 3.71e-07, "output_cost_per_token": 1.484e-06, "litellm_provider": "watsonx", @@ -47119,9 +47119,9 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b-128e-instruct-fp8": { - "max_tokens": 128000, + "max_tokens": 8192, "max_input_tokens": 131072, - "max_output_tokens": 128000, + "max_output_tokens": 8192, "input_cost_per_token": 3.71e-07, "output_cost_per_token": 1.484e-06, "litellm_provider": "watsonx", @@ -47168,9 +47168,9 @@ "supports_vision": false }, "watsonx/mistralai/mistral-small-3-1-24b-instruct-2503": { - "max_tokens": 32000, + "max_tokens": 16384, "max_input_tokens": 131072, - "max_output_tokens": 32000, + "max_output_tokens": 16384, "input_cost_per_token": 1.06e-07, "output_cost_per_token": 3.18e-07, "litellm_provider": "watsonx", @@ -47193,9 +47193,9 @@ "supports_vision": true }, "watsonx/openai/gpt-oss-120b": { - "max_tokens": 8192, + "max_tokens": 131072, "max_input_tokens": 131072, - "max_output_tokens": 8192, + "max_output_tokens": 131072, "input_cost_per_token": 1.59e-07, "output_cost_per_token": 6.36e-07, "litellm_provider": "watsonx", From 55a5f142e612d99931500091e4a4d7fa13676060 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 3 Sep 2026 13:23:02 +0000 Subject: [PATCH 09/88] fix(model_prices): add azure_ai Codestral-2501 and FW-Nemotron-Lightning-3.5, sync Azure and Vertex deprecation dates, fix novita gpt-oss vision flags Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 55 +++++++++++++++++-- model_prices_and_context_window.json | 55 +++++++++++++++++-- .../azure_ai/test_azure_ai_cost_calculator.py | 13 +++++ .../test_azure_ai_fw_models_metadata.py | 25 +++++++++ 4 files changed, 136 insertions(+), 12 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a3030d2e33d..4e1869a83a8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3264,7 +3264,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "deprecation_date": "2027-12-05" }, "azure_ai/claude-opus-5": { "deprecation_date": "2027-07-08", @@ -8813,7 +8814,7 @@ "supports_web_search": false }, "azure/us/gpt-4.1-nano-2025-04-14": { - "deprecation_date": "2026-10-14", + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1.1e-07, "input_cost_per_token_batches": 6e-08, @@ -9333,6 +9334,26 @@ "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", "supports_embedding_image_input": true }, + "azure_ai/Codestral-2501": { + "input_cost_per_token": 3e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 256000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 9e-07, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/mistral/", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_native_streaming": true + }, "azure_ai/FLUX-1.1-pro": { "litellm_provider": "azure_ai", "mode": "image_generation", @@ -9611,6 +9632,26 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure_ai/FW-Nemotron-Lightning-3.5-30B-A3B": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 6e-08, + "litellm_provider": "azure_ai", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.2e-07, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, "azure_ai/FW-Nemotron-3-Ultra-NVFP4": { "cache_read_input_token_cost": 1.19e-07, "input_cost_per_token": 6e-07, @@ -44923,7 +44964,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "deprecation_date": "2027-03-01" }, "vertex_ai/claude-fable-5@default": { "deprecation_date": "2027-06-08", @@ -44994,7 +45036,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "deprecation_date": "2027-03-01" }, "vertex_ai/claude-opus-5": { "deprecation_date": "2027-01-24", @@ -51227,7 +51270,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, - "supports_vision": true, + "supports_vision": false, "supports_system_messages": true, "supports_response_schema": true, "supports_reasoning": true @@ -51343,7 +51386,7 @@ "max_output_tokens": 32768, "max_tokens": 32768, "supports_tool_choice": true, - "supports_vision": true, + "supports_vision": false, "supports_system_messages": true, "supports_response_schema": true, "supports_reasoning": true diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a3030d2e33d..4e1869a83a8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3264,7 +3264,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "deprecation_date": "2027-12-05" }, "azure_ai/claude-opus-5": { "deprecation_date": "2027-07-08", @@ -8813,7 +8814,7 @@ "supports_web_search": false }, "azure/us/gpt-4.1-nano-2025-04-14": { - "deprecation_date": "2026-10-14", + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1.1e-07, "input_cost_per_token_batches": 6e-08, @@ -9333,6 +9334,26 @@ "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", "supports_embedding_image_input": true }, + "azure_ai/Codestral-2501": { + "input_cost_per_token": 3e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 256000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 9e-07, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/mistral/", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_native_streaming": true + }, "azure_ai/FLUX-1.1-pro": { "litellm_provider": "azure_ai", "mode": "image_generation", @@ -9611,6 +9632,26 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure_ai/FW-Nemotron-Lightning-3.5-30B-A3B": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 6e-08, + "litellm_provider": "azure_ai", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.2e-07, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, "azure_ai/FW-Nemotron-3-Ultra-NVFP4": { "cache_read_input_token_cost": 1.19e-07, "input_cost_per_token": 6e-07, @@ -44923,7 +44964,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "deprecation_date": "2027-03-01" }, "vertex_ai/claude-fable-5@default": { "deprecation_date": "2027-06-08", @@ -44994,7 +45036,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "deprecation_date": "2027-03-01" }, "vertex_ai/claude-opus-5": { "deprecation_date": "2027-01-24", @@ -51227,7 +51270,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, - "supports_vision": true, + "supports_vision": false, "supports_system_messages": true, "supports_response_schema": true, "supports_reasoning": true @@ -51343,7 +51386,7 @@ "max_output_tokens": 32768, "max_tokens": 32768, "supports_tool_choice": true, - "supports_vision": true, + "supports_vision": false, "supports_system_messages": true, "supports_response_schema": true, "supports_reasoning": true diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py index 20260c744f8..fd6fab6a521 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py @@ -499,3 +499,16 @@ class TestAzureAIServiceTierCostCalculation: assert flex_prompt < standard_prompt assert flex_completion < standard_completion + + +def test_codestral_2501_model_info_and_cost(): + model_info = get_model_info(model="Codestral-2501", custom_llm_provider="azure_ai") + usage = Usage(prompt_tokens=1_000_000, completion_tokens=1_000_000, total_tokens=2_000_000) + + prompt_cost, completion_cost = cost_per_token(model="Codestral-2501", usage=usage) + + assert model_info["mode"] == "chat" + assert model_info["max_input_tokens"] == 256000 + assert model_info["max_output_tokens"] == 4096 + assert prompt_cost == pytest.approx(0.3) + assert completion_cost == pytest.approx(0.9) diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_fw_models_metadata.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_fw_models_metadata.py index 9917ab41b42..f3618572622 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_fw_models_metadata.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_fw_models_metadata.py @@ -176,6 +176,7 @@ def test_azure_ai_fw_model_info(use_local_model_cost_map, model_key, expected): ("FW-MiniMax-M2.5", 0.33, 1.32), ("FW-Inkling", 1.0, 4.05), ("FW-Nemotron-3-Ultra-NVFP4", 0.6, 2.4), + ("FW-Nemotron-Lightning-3.5-30B-A3B", 0.06, 0.22), ], ) def test_azure_ai_fw_cost_per_token( @@ -196,6 +197,30 @@ def test_azure_ai_fw_cost_per_token( assert completion_cost == pytest.approx(expected_completion) +def test_azure_ai_fw_nemotron_lightning_model_info(use_local_model_cost_map): + model_info = use_local_model_cost_map.get_model_info(model="azure_ai/FW-Nemotron-Lightning-3.5-30B-A3B") + + assert model_info["litellm_provider"] == "azure_ai" + assert model_info["mode"] == "chat" + assert model_info["input_cost_per_token"] == pytest.approx(6e-08) + assert model_info["output_cost_per_token"] == pytest.approx(2.2e-07) + assert model_info["cache_read_input_token_cost"] == pytest.approx(1e-08) + assert model_info["max_input_tokens"] == 262144 + assert model_info["supports_function_calling"] is True + assert model_info["supports_reasoning"] is True + assert model_info["supports_tool_choice"] is True + assert model_info["supports_prompt_caching"] is True + assert model_info["supports_vision"] is False + + +def test_azure_ai_fw_nemotron_lightning_supports_tool_choice(use_local_model_cost_map): + from litellm.llms.azure_ai.chat.transformation import AzureAIStudioConfig + + supported_params = AzureAIStudioConfig().get_supported_openai_params("FW-Nemotron-Lightning-3.5-30B-A3B") + + assert "tool_choice" in supported_params + + def test_azure_ai_fw_kimi_k26_case_insensitive_lookup(use_local_model_cost_map): upper = use_local_model_cost_map.get_model_info(model="azure_ai/FW-Kimi-K2.6") lower = use_local_model_cost_map.get_model_info(model="azure_ai/fw-kimi-k2.6") From f26407aa8ceb54e623ab6629a4a6524ad1dd8907 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 3 Sep 2026 13:43:48 +0000 Subject: [PATCH 10/88] feat(registry): add azure_ai/MAI-Thinking-1 from Azure Retail Prices and Foundry docs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 24 +++++++++++++++++++ model_prices_and_context_window.json | 24 +++++++++++++++++++ .../azure_ai/test_azure_ai_cost_calculator.py | 18 +++++++++++++- 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4e1869a83a8..23ce80f9968 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9713,6 +9713,30 @@ "/v1/images/generations" ] }, + "azure_ai/MAI-Thinking-1": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 256000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 8e-06, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "azure_ai/Llama-3.2-11B-Vision-Instruct": { "deprecation_date": "2026-06-13", "input_cost_per_token": 3.7e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4e1869a83a8..23ce80f9968 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -9713,6 +9713,30 @@ "/v1/images/generations" ] }, + "azure_ai/MAI-Thinking-1": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 256000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 8e-06, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "azure_ai/Llama-3.2-11B-Vision-Instruct": { "deprecation_date": "2026-06-13", "input_cost_per_token": 3.7e-07, diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py index fd6fab6a521..9612d97d946 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py @@ -501,7 +501,7 @@ class TestAzureAIServiceTierCostCalculation: assert flex_completion < standard_completion -def test_codestral_2501_model_info_and_cost(): +def test_codestral_2501_model_info_and_cost(local_model_cost_map): model_info = get_model_info(model="Codestral-2501", custom_llm_provider="azure_ai") usage = Usage(prompt_tokens=1_000_000, completion_tokens=1_000_000, total_tokens=2_000_000) @@ -512,3 +512,19 @@ def test_codestral_2501_model_info_and_cost(): assert model_info["max_output_tokens"] == 4096 assert prompt_cost == pytest.approx(0.3) assert completion_cost == pytest.approx(0.9) + + +def test_mai_thinking_1_model_info_and_cost(local_model_cost_map): + model_info = get_model_info(model="MAI-Thinking-1", custom_llm_provider="azure_ai") + usage = Usage(prompt_tokens=1_000_000, completion_tokens=1_000_000, total_tokens=2_000_000) + + prompt_cost, completion_cost = cost_per_token(model="MAI-Thinking-1", usage=usage) + + assert model_info["mode"] == "chat" + assert model_info["max_input_tokens"] == 256000 + assert model_info["max_output_tokens"] == 64000 + assert model_info["cache_read_input_token_cost"] == pytest.approx(2e-07) + assert model_info["supports_reasoning"] is True + assert model_info["supports_function_calling"] is True + assert prompt_cost == pytest.approx(2.0) + assert completion_cost == pytest.approx(8.0) From 840173e7789cbd8d9aa8cc836d8827427daf0201 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 3 Sep 2026 13:48:25 +0000 Subject: [PATCH 11/88] feat(registry): add azure_ai/mistral-ocr-4-0 page and annotation prices from Azure Retail Prices Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../model_prices_and_context_window_backup.json | 10 ++++++++++ model_prices_and_context_window.json | 10 ++++++++++ .../llms/mistral/ocr/test_mistral_ocr_cost.py | 14 ++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 23ce80f9968..df25f312b04 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -10037,6 +10037,16 @@ ], "source": "https://ai.azure.com/catalog/models/mistral-document-ai-2512" }, + "azure_ai/mistral-ocr-4-0": { + "litellm_provider": "azure_ai", + "ocr_cost_per_page": 0.004, + "annotation_cost_per_page": 0.005, + "mode": "ocr", + "supported_endpoints": [ + "/v1/ocr" + ], + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/mistral/" + }, "azure_ai/doc-intelligence/prebuilt-read": { "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.0015, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 23ce80f9968..df25f312b04 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -10037,6 +10037,16 @@ ], "source": "https://ai.azure.com/catalog/models/mistral-document-ai-2512" }, + "azure_ai/mistral-ocr-4-0": { + "litellm_provider": "azure_ai", + "ocr_cost_per_page": 0.004, + "annotation_cost_per_page": 0.005, + "mode": "ocr", + "supported_endpoints": [ + "/v1/ocr" + ], + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/mistral/" + }, "azure_ai/doc-intelligence/prebuilt-read": { "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.0015, diff --git a/tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py b/tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py index a0e1616d4b2..40e54f71eeb 100644 --- a/tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py +++ b/tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py @@ -15,6 +15,7 @@ from litellm.cost_calculator import completion_cost from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo OCR4_COST_PER_PAGE = 0.004 +OCR4_ANNOTATION_COST_PER_PAGE = 0.005 REPO_ROOT = Path(__file__).parents[5] MAIN_COST_MAP = REPO_ROOT / "model_prices_and_context_window.json" @@ -133,3 +134,16 @@ def test_azure_doc_ai_annotation_pages_fall_back_to_ocr_rate(local_model_cost_ma call_type="ocr", ) assert cost == pytest.approx(AZURE_DOC_AI_COST_PER_PAGE) + + +def test_azure_ocr4_bills_ocr_and_annotation_pages_at_their_own_rates(local_model_cost_map) -> None: + info = litellm.get_model_info(model="azure_ai/mistral-ocr-4-0", custom_llm_provider="azure_ai") + assert info["ocr_cost_per_page"] == OCR4_COST_PER_PAGE + assert info["annotation_cost_per_page"] == OCR4_ANNOTATION_COST_PER_PAGE + cost = completion_cost( + completion_response=_annotated_ocr_response("mistral-ocr-4-0", 2, 3), + model="azure_ai/mistral-ocr-4-0", + custom_llm_provider="azure_ai", + call_type="ocr", + ) + assert cost == pytest.approx(2 * OCR4_COST_PER_PAGE + 3 * OCR4_ANNOTATION_COST_PER_PAGE) From 63579f1e355dabb4df2b31f6c654d1d6f7b0ad44 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 3 Sep 2026 14:12:57 +0000 Subject: [PATCH 12/88] fix(spend-tracking): keep batch spend keys joinable after v1.99 provenance gate Batch cost attribution and the legacy queue endpoint already store the VerificationToken hash in user_api_key, but omitted user_api_key_hash. Since v1.99 the spend-log writer re-hashes any key without that provenance flag, so DailyUserSpend.api_key no longer joins VerificationToken and Usage shows key-hash-... rows with null api_key_alias / user_email. Co-authored-by: Mateo Wang --- .../proxy/common_utils/check_batch_cost.py | 1 + litellm/proxy/proxy_server.py | 1 + .../proxy_unit_tests/test_check_batch_cost.py | 43 +++++++++++++++++++ .../test_spend_tracking_utils.py | 36 ++++++++++++++++ 4 files changed, 81 insertions(+) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 354a6ed2fd0..e6f00877a26 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -161,6 +161,7 @@ class CheckBatchCost: metadata: dict[str, object] = { "user_api_key_user_id": job.created_by, "user_api_key": api_key, + "user_api_key_hash": api_key, "user_api_key_team_id": team_id, **(await self._get_user_info(batch_id, job.created_by)), } diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 27132c90e05..222e79f7ca7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15197,6 +15197,7 @@ async def async_queue_request( # extra_body); see above for the same guard upstream. data["metadata"] = {} data["metadata"]["user_api_key"] = user_api_key_dict.api_key + data["metadata"]["user_api_key_hash"] = user_api_key_dict.api_key data["metadata"]["user_api_key_metadata"] = strip_callback_config(user_api_key_dict.metadata) _headers: Final = _safe_get_request_headers(request).copy() _headers.pop("authorization", None) # do not store the original `sk-..` api key in the db diff --git a/tests/proxy_unit_tests/test_check_batch_cost.py b/tests/proxy_unit_tests/test_check_batch_cost.py index ff5e8f89d64..9a6ab08e9b6 100644 --- a/tests/proxy_unit_tests/test_check_batch_cost.py +++ b/tests/proxy_unit_tests/test_check_batch_cost.py @@ -2445,6 +2445,7 @@ class TestBatchCostAttribution: metadata = await instance._build_creator_attribution_metadata(self._job(), "batch-1") assert metadata["user_api_key"] == "hash-alice" + assert metadata["user_api_key_hash"] == "hash-alice" assert metadata["user_api_key_user_id"] == "alice" assert metadata["user_api_key_team_id"] == "team-alpha" assert metadata["user_api_key_alias"] == "prod-key" @@ -2553,6 +2554,48 @@ class TestBatchCostAttribution: assert metadata["user_api_key_alias"] == "prod-key" + @pytest.mark.asyncio + async def test_metadata_provenance_keeps_spend_log_api_key_joinable(self): + """ + CheckBatchCost stores the VerificationToken hash on the managed object. The + spend-log writer must receive matching user_api_key_hash provenance so it + does not re-hash that value; otherwise DailyUserSpend.api_key no longer joins + VerificationToken and Usage shows key-hash-... with a null alias/email. + """ + from datetime import datetime, timezone + from types import SimpleNamespace + + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + from litellm.proxy.utils import hash_token + + token_hash = hash_token("sk-batch-creator-key") + instance = self._instance( + key_row=SimpleNamespace(key_alias="prod-key"), + user_row=SimpleNamespace(user_email="alice@example.com", user_alias=None), + ) + metadata = await instance._build_creator_attribution_metadata( + self._job(api_key=token_hash), "batch-1" + ) + + assert metadata["user_api_key"] == token_hash + assert metadata["user_api_key_hash"] == token_hash + + payload = get_logging_payload( + kwargs={ + "model": "gpt-4o", + "call_type": "aretrieve_batch", + "litellm_params": {"metadata": metadata}, + }, + response_obj={ + "id": "batch_123", + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + start_time=datetime.now(timezone.utc), + end_time=datetime.now(timezone.utc), + ) + assert payload["api_key"] == token_hash + assert payload["api_key"] != hash_token(token_hash) + class TestPollPageStarvation: """LIT-5462 regression: a row that can never be costed used to keep its slot in the diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 9e5917637a8..bea1f8e2d6c 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -2755,6 +2755,42 @@ def test_get_spend_logs_metadata_already_hashed_no_provenance_is_rehashed(): assert meta["user_api_key"] == hash_token(already_hashed) +def test_get_logging_payload_batch_attribution_keeps_verification_token_hash(): + """ + Batch cost rebuilds metadata with the managed object's already-hashed api_key. + That hash must land in SpendLogs.api_key unchanged so Usage/CloudZero can join + LiteLLM_VerificationToken for api_key_alias and user_email. Regression: without + user_api_key_hash provenance, v1.99+ re-hashed the token and broke the join. + """ + token_hash = hash_token("sk-batch-creator-key") + kwargs = { + "model": "gpt-4o", + "call_type": "aretrieve_batch", + "litellm_params": { + "metadata": { + "user_api_key": token_hash, + "user_api_key_hash": token_hash, + "user_api_key_alias": "batch-creator", + "user_api_key_user_id": "alice", + "user_api_key_user_email": "alice@example.com", + "user_api_key_team_id": "team-1", + } + }, + } + payload = get_logging_payload( + kwargs=kwargs, + response_obj={"id": "batch_123", "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + assert payload["api_key"] == token_hash + assert payload["api_key"] != hash_token(token_hash) + parsed_meta = json.loads(payload["metadata"]) + assert parsed_meta["user_api_key"] == token_hash + assert parsed_meta["user_api_key_alias"] == "batch-creator" + + def test_get_spend_logs_metadata_provenance_bypass_requires_hash_match(): already_hashed = hash_token("sk-some-key") different_hash = hash_token("sk-other-key") From e07d58a4efd09e5dc6e044bdc089ba7e626489df Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 3 Sep 2026 14:21:25 +0000 Subject: [PATCH 13/88] fix(usage): recover aliases for v1.99 double-hashed spend keys Callback log replay also omitted user_api_key_hash, so it could double-hash spend rows the same way batch costing did. On the read path, Usage key metadata now reverse-hashes orphaned DailyUserSpend.api_key values against VerificationToken and falls back to SpendLogs metadata so historical dirty rows show their api_key_alias again instead of key-hash-... Co-authored-by: Mateo Wang --- .../callback_logs_endpoints.py | 4 +- .../common_daily_activity.py | 163 +++++++++++++++++- .../test_callback_logs_endpoints.py | 1 + .../test_common_daily_activity.py | 62 +++++++ 4 files changed, 226 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/logging_endpoints/callback_logs_endpoints.py b/litellm/proxy/logging_endpoints/callback_logs_endpoints.py index 66057f0dc16..cecadc03d71 100644 --- a/litellm/proxy/logging_endpoints/callback_logs_endpoints.py +++ b/litellm/proxy/logging_endpoints/callback_logs_endpoints.py @@ -88,8 +88,10 @@ class CallbackLogsReplayer: ) metadata: Final[dict[str, Any]] = payload.get("metadata") or {} + user_api_key_hash: Final = metadata.get("user_api_key_hash") litellm_metadata: Final[dict[str, Any]] = { - "user_api_key": metadata.get("user_api_key_hash"), + "user_api_key": user_api_key_hash, + "user_api_key_hash": user_api_key_hash, "user_api_key_alias": metadata.get("user_api_key_alias"), "user_api_key_user_id": metadata.get("user_api_key_user_id"), "user_api_key_team_id": metadata.get("user_api_key_team_id"), diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 91cd80b3c81..c1fec88d43d 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -10,9 +10,10 @@ from typing_extensions import TypedDict from litellm._logging import verbose_proxy_logger from litellm.constants import PTU_SENTINEL_API_KEY +from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash from litellm.proxy._types import CommonProxyErrors from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attribution_enabled -from litellm.proxy.utils import PrismaClient +from litellm.proxy.utils import PrismaClient, hash_token from litellm.repositories.table_repositories import DeletedVerificationTokenRepository from litellm.repositories.verification_token_repository import ( VerificationTokenRepository, @@ -115,6 +116,25 @@ class _KeyMetadataDict(TypedDict, total=False): team_id: str | None +# Cap reverse-hash scans so a Usage page with orphaned double-hashed api_key +# values cannot pull an unbounded VerificationToken table into memory. +_MAX_DOUBLE_HASH_TOKEN_SCAN: Final = 10_000 + +_SPEND_LOGS_KEY_METADATA_SQL: Final = """ +SELECT DISTINCT ON (api_key) + api_key, + metadata->>'user_api_key_alias' AS key_alias, + metadata->>'user_api_key_team_id' AS team_id +FROM "LiteLLM_SpendLogs" +WHERE api_key = ANY($1::text[]) + AND ( + NULLIF(metadata->>'user_api_key_alias', '') IS NOT NULL + OR NULLIF(metadata->>'user_api_key_team_id', '') IS NOT NULL + ) +ORDER BY api_key, "startTime" DESC NULLS LAST +""" + + _WhereValue = str | dict[str, object] @@ -439,6 +459,136 @@ def update_breakdown_metrics( return breakdown +class _TokenAliasRecord(Protocol): + @property + def token(self) -> str: ... + + @property + def key_alias(self) -> str | None: ... + + @property + def team_id(self) -> str | None: ... + + +def _token_digest_metadata( + records: Sequence[_TokenAliasRecord], + wanted: AbstractSet[str], +) -> dict[str, _KeyMetadataDict]: + return { + digested: {"key_alias": record.key_alias, "team_id": record.team_id} + for record in records + for digested in (hash_token(record.token),) + if digested in wanted + } + + +async def _reverse_hash_active_key_metadata( + prisma_client: PrismaClient, + wanted: AbstractSet[str], +) -> dict[str, _KeyMetadataDict]: + try: + active_records: Final[Sequence[_TokenAliasRecord]] = await VerificationTokenRepository( + prisma_client + ).table.find_many(take=_MAX_DOUBLE_HASH_TOKEN_SCAN) + except Exception as e: + verbose_proxy_logger.warning( + "Failed reverse-hash recovery against active keys for %d missing keys: %s", + len(wanted), + e, + ) + return {} + return _token_digest_metadata(active_records, wanted) + + +async def _reverse_hash_deleted_key_metadata( + prisma_client: PrismaClient, + wanted: AbstractSet[str], +) -> dict[str, _KeyMetadataDict]: + try: + deleted_records: Final[Sequence[_TokenAliasRecord]] = await DeletedVerificationTokenRepository( + prisma_client + ).table.find_many( + take=_MAX_DOUBLE_HASH_TOKEN_SCAN, + order={"deleted_at": "desc"}, + ) + except Exception as e: + verbose_proxy_logger.warning( + "Failed reverse-hash recovery against deleted keys for %d missing keys: %s", + len(wanted), + e, + ) + return {} + return _token_digest_metadata(deleted_records, wanted) + + +async def _reverse_hash_key_metadata( + prisma_client: PrismaClient, + wanted: AbstractSet[str], +) -> dict[str, _KeyMetadataDict]: + from_active: Final = await _reverse_hash_active_key_metadata(prisma_client, wanted) + still_wanted: Final = wanted - frozenset(from_active) + if not still_wanted: + return from_active + return {**from_active, **(await _reverse_hash_deleted_key_metadata(prisma_client, still_wanted))} + + +async def _spend_logs_key_metadata( + prisma_client: PrismaClient, + wanted: AbstractSet[str], +) -> dict[str, _KeyMetadataDict]: + try: + spend_log_rows: Final = await prisma_client.db.query_raw( + _SPEND_LOGS_KEY_METADATA_SQL, + list(wanted), + ) + except Exception as e: + verbose_proxy_logger.warning( + "Failed SpendLogs metadata recovery for %d missing keys: %s", + len(wanted), + e, + ) + return {} + + if not isinstance(spend_log_rows, list): + return {} + + return { + row["api_key"]: { + "key_alias": row.get("key_alias"), + "team_id": row.get("team_id"), + } + for row in spend_log_rows + if isinstance(row, dict) + and isinstance(row.get("api_key"), str) + and row["api_key"] in wanted + } + + +async def _recover_double_hashed_key_metadata( + prisma_client: PrismaClient, + missing_keys: AbstractSet[str], +) -> dict[str, _KeyMetadataDict]: + """ + Recover key_alias/team_id for DailyUserSpend.api_key values that were + double-hashed by the v1.99 spend-log provenance gate. + + Those rows store hash(VerificationToken.token) instead of the token, so the + exact join misses. Prefer a bounded reverse-hash against active/deleted + tokens; fall back to the alias/team stamped into SpendLogs metadata (which + stayed correct even when api_key did not). + """ + sha_missing: Final = frozenset(key for key in missing_keys if is_valid_sha256_hash(key)) + if not sha_missing: + return {} + + from_tokens: Final = await _reverse_hash_key_metadata(prisma_client, sha_missing) + still_missing: Final = sha_missing - frozenset(from_tokens) + if not still_missing: + return from_tokens + + return {**from_tokens, **(await _spend_logs_key_metadata(prisma_client, still_missing))} + + async def get_api_key_metadata( prisma_client: PrismaClient, api_keys: AbstractSet[str], @@ -446,7 +596,8 @@ async def get_api_key_metadata( """Get api key metadata, falling back to deleted keys table for keys not found in active table. This ensures that key_alias and team_id are preserved in historical activity logs - even after a key is deleted or regenerated. + even after a key is deleted or regenerated. Also recovers aliases for api_key + values that were double-hashed by the v1.99 spend-log provenance gate. """ key_records: Sequence[PrismaVerificationToken] = await VerificationTokenRepository(prisma_client).table.find_many( where={"token": {"in": list(api_keys)}} @@ -479,7 +630,13 @@ async def get_api_key_metadata( e, ) - return result + still_missing: Final = api_keys - set(result.keys()) + if not still_missing: + return result + return { + **result, + **(await _recover_double_hashed_key_metadata(prisma_client, still_missing)), + } def _adjust_dates_for_timezone( diff --git a/tests/test_litellm/proxy/logging_endpoints/test_callback_logs_endpoints.py b/tests/test_litellm/proxy/logging_endpoints/test_callback_logs_endpoints.py index 40e89329b8d..590d63fd868 100644 --- a/tests/test_litellm/proxy/logging_endpoints/test_callback_logs_endpoints.py +++ b/tests/test_litellm/proxy/logging_endpoints/test_callback_logs_endpoints.py @@ -61,6 +61,7 @@ def test_build_logging_obj_seeds_model_call_details(): # Metadata is mapped to the keys the cost-tracking callback reads. md = details["litellm_params"]["metadata"] assert md["user_api_key"] == "rust-gateway-test-key" + assert md["user_api_key_hash"] == "rust-gateway-test-key" assert md["user_api_key_user_id"] == "user-cb-logs-test" assert md["user_api_key_team_id"] == "team-cb-logs-test" diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index a258127acff..78752495f25 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -454,6 +454,68 @@ async def test_get_api_key_metadata_regenerated_key_uses_most_recent_deleted_rec assert result["old-key-hash"]["team_id"] == "latest-team" +@pytest.mark.asyncio +async def test_get_api_key_metadata_recovers_double_hashed_key_via_reverse_hash(): + """ + v1.99 spend logging re-hashed already-hashed api_key values when provenance was + missing. Usage joins DailyUserSpend.api_key to VerificationToken.token, so those + rows looked like key-hash-... with a null alias. Reverse-hash recovery must map + hash(token) back to the key's alias for historical dirty spend. + """ + from litellm.proxy.utils import hash_token + + token = "a" * 64 + double_hashed = hash_token(token) + mock_prisma = MagicMock() + + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock( + side_effect=[ + [], # exact join miss + [SimpleNamespace(token=token, key_alias="batch-worker", team_id="team-1")], + ] + ) + mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma.db.query_raw = AsyncMock(return_value=[]) + + result = await get_api_key_metadata( + prisma_client=mock_prisma, + api_keys={double_hashed}, + ) + + assert result[double_hashed]["key_alias"] == "batch-worker" + assert result[double_hashed]["team_id"] == "team-1" + mock_prisma.db.query_raw.assert_not_called() + + +@pytest.mark.asyncio +async def test_get_api_key_metadata_recovers_double_hashed_key_via_spend_logs(): + """When the token tables cannot reverse-hash the dirty key, use SpendLogs metadata.""" + from litellm.proxy.utils import hash_token + + double_hashed = hash_token("b" * 64) + mock_prisma = MagicMock() + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma.db.query_raw = AsyncMock( + return_value=[ + { + "api_key": double_hashed, + "key_alias": "from-spend-log", + "team_id": "team-spend", + } + ] + ) + + result = await get_api_key_metadata( + prisma_client=mock_prisma, + api_keys={double_hashed}, + ) + + assert result[double_hashed]["key_alias"] == "from-spend-log" + assert result[double_hashed]["team_id"] == "team-spend" + mock_prisma.db.query_raw.assert_called_once() + + @pytest.mark.asyncio async def test_tag_daily_activity_metadata_totals_not_zero(): """Test that tag daily activity returns correct metadata totals. From 9dae07175cd4c3c8a795d9e198903259cb05b78b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 3 Sep 2026 14:24:08 +0000 Subject: [PATCH 14/88] fix(spend): share double-hash key alias recovery with CloudZero and Focus Extract the Usage reverse-hash / SpendLogs alias recovery into a shared helper and apply it when CloudZero and Focus export DailyUserSpend rows, so BI pulls get api_key_alias back for historical v1.99 double-hashed keys instead of null. Co-authored-by: Mateo Wang --- litellm/integrations/cloudzero/database.py | 11 +- litellm/integrations/focus/database.py | 9 +- .../common_daily_activity.py | 157 +----------- .../spend_tracking/key_metadata_recovery.py | 223 ++++++++++++++++++ .../test_key_metadata_recovery.py | 64 +++++ 5 files changed, 308 insertions(+), 156 deletions(-) create mode 100644 litellm/proxy/spend_tracking/key_metadata_recovery.py create mode 100644 tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py diff --git a/litellm/integrations/cloudzero/database.py b/litellm/integrations/cloudzero/database.py index b050ee8e1ed..8fedd4edac4 100644 --- a/litellm/integrations/cloudzero/database.py +++ b/litellm/integrations/cloudzero/database.py @@ -94,8 +94,13 @@ class LiteLLMDatabase: try: db_response: Final = await client.db.query_raw(query, *params) - # Convert the response to polars DataFrame with full schema inference - # This prevents schema mismatch errors when data types vary across rows - return pl.DataFrame(db_response, infer_schema_length=None) + from litellm.proxy.spend_tracking.key_metadata_recovery import ( + fill_missing_api_key_aliases, + ) + + # v1.99 double-hashed DailyUserSpend.api_key values miss the + # VerificationToken join above; recover alias/team for those rows. + recovered_rows: Final = await fill_missing_api_key_aliases(client, db_response) + return pl.DataFrame(list(recovered_rows), infer_schema_length=None) except Exception as e: raise Exception(f"Error retrieving usage data: {e}") diff --git a/litellm/integrations/focus/database.py b/litellm/integrations/focus/database.py index 815c38b9e9c..db9849bbbc9 100644 --- a/litellm/integrations/focus/database.py +++ b/litellm/integrations/focus/database.py @@ -96,7 +96,14 @@ class FocusLiteLLMDatabase: try: db_response: Final = await client.db.query_raw(query, *query_params) - return pl.DataFrame(db_response, infer_schema_length=None) + from litellm.proxy.spend_tracking.key_metadata_recovery import ( + fill_missing_api_key_aliases, + ) + + # v1.99 double-hashed DailyUserSpend.api_key values miss the + # VerificationToken join above; recover alias/team for those rows. + recovered_rows: Final = await fill_missing_api_key_aliases(client, db_response) + return pl.DataFrame(list(recovered_rows), infer_schema_length=None) except Exception as exc: raise RuntimeError(f"Error retrieving usage data: {exc}") from exc diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index c1fec88d43d..f687cedeee9 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -10,10 +10,12 @@ from typing_extensions import TypedDict from litellm._logging import verbose_proxy_logger from litellm.constants import PTU_SENTINEL_API_KEY -from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash from litellm.proxy._types import CommonProxyErrors +from litellm.proxy.spend_tracking.key_metadata_recovery import ( + recover_double_hashed_key_metadata, +) from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attribution_enabled -from litellm.proxy.utils import PrismaClient, hash_token +from litellm.proxy.utils import PrismaClient from litellm.repositories.table_repositories import DeletedVerificationTokenRepository from litellm.repositories.verification_token_repository import ( VerificationTokenRepository, @@ -116,25 +118,6 @@ class _KeyMetadataDict(TypedDict, total=False): team_id: str | None -# Cap reverse-hash scans so a Usage page with orphaned double-hashed api_key -# values cannot pull an unbounded VerificationToken table into memory. -_MAX_DOUBLE_HASH_TOKEN_SCAN: Final = 10_000 - -_SPEND_LOGS_KEY_METADATA_SQL: Final = """ -SELECT DISTINCT ON (api_key) - api_key, - metadata->>'user_api_key_alias' AS key_alias, - metadata->>'user_api_key_team_id' AS team_id -FROM "LiteLLM_SpendLogs" -WHERE api_key = ANY($1::text[]) - AND ( - NULLIF(metadata->>'user_api_key_alias', '') IS NOT NULL - OR NULLIF(metadata->>'user_api_key_team_id', '') IS NOT NULL - ) -ORDER BY api_key, "startTime" DESC NULLS LAST -""" - - _WhereValue = str | dict[str, object] @@ -459,136 +442,6 @@ def update_breakdown_metrics( return breakdown -class _TokenAliasRecord(Protocol): - @property - def token(self) -> str: ... - - @property - def key_alias(self) -> str | None: ... - - @property - def team_id(self) -> str | None: ... - - -def _token_digest_metadata( - records: Sequence[_TokenAliasRecord], - wanted: AbstractSet[str], -) -> dict[str, _KeyMetadataDict]: - return { - digested: {"key_alias": record.key_alias, "team_id": record.team_id} - for record in records - for digested in (hash_token(record.token),) - if digested in wanted - } - - -async def _reverse_hash_active_key_metadata( - prisma_client: PrismaClient, - wanted: AbstractSet[str], -) -> dict[str, _KeyMetadataDict]: - try: - active_records: Final[Sequence[_TokenAliasRecord]] = await VerificationTokenRepository( - prisma_client - ).table.find_many(take=_MAX_DOUBLE_HASH_TOKEN_SCAN) - except Exception as e: - verbose_proxy_logger.warning( - "Failed reverse-hash recovery against active keys for %d missing keys: %s", - len(wanted), - e, - ) - return {} - return _token_digest_metadata(active_records, wanted) - - -async def _reverse_hash_deleted_key_metadata( - prisma_client: PrismaClient, - wanted: AbstractSet[str], -) -> dict[str, _KeyMetadataDict]: - try: - deleted_records: Final[Sequence[_TokenAliasRecord]] = await DeletedVerificationTokenRepository( - prisma_client - ).table.find_many( - take=_MAX_DOUBLE_HASH_TOKEN_SCAN, - order={"deleted_at": "desc"}, - ) - except Exception as e: - verbose_proxy_logger.warning( - "Failed reverse-hash recovery against deleted keys for %d missing keys: %s", - len(wanted), - e, - ) - return {} - return _token_digest_metadata(deleted_records, wanted) - - -async def _reverse_hash_key_metadata( - prisma_client: PrismaClient, - wanted: AbstractSet[str], -) -> dict[str, _KeyMetadataDict]: - from_active: Final = await _reverse_hash_active_key_metadata(prisma_client, wanted) - still_wanted: Final = wanted - frozenset(from_active) - if not still_wanted: - return from_active - return {**from_active, **(await _reverse_hash_deleted_key_metadata(prisma_client, still_wanted))} - - -async def _spend_logs_key_metadata( - prisma_client: PrismaClient, - wanted: AbstractSet[str], -) -> dict[str, _KeyMetadataDict]: - try: - spend_log_rows: Final = await prisma_client.db.query_raw( - _SPEND_LOGS_KEY_METADATA_SQL, - list(wanted), - ) - except Exception as e: - verbose_proxy_logger.warning( - "Failed SpendLogs metadata recovery for %d missing keys: %s", - len(wanted), - e, - ) - return {} - - if not isinstance(spend_log_rows, list): - return {} - - return { - row["api_key"]: { - "key_alias": row.get("key_alias"), - "team_id": row.get("team_id"), - } - for row in spend_log_rows - if isinstance(row, dict) - and isinstance(row.get("api_key"), str) - and row["api_key"] in wanted - } - - -async def _recover_double_hashed_key_metadata( - prisma_client: PrismaClient, - missing_keys: AbstractSet[str], -) -> dict[str, _KeyMetadataDict]: - """ - Recover key_alias/team_id for DailyUserSpend.api_key values that were - double-hashed by the v1.99 spend-log provenance gate. - - Those rows store hash(VerificationToken.token) instead of the token, so the - exact join misses. Prefer a bounded reverse-hash against active/deleted - tokens; fall back to the alias/team stamped into SpendLogs metadata (which - stayed correct even when api_key did not). - """ - sha_missing: Final = frozenset(key for key in missing_keys if is_valid_sha256_hash(key)) - if not sha_missing: - return {} - - from_tokens: Final = await _reverse_hash_key_metadata(prisma_client, sha_missing) - still_missing: Final = sha_missing - frozenset(from_tokens) - if not still_missing: - return from_tokens - - return {**from_tokens, **(await _spend_logs_key_metadata(prisma_client, still_missing))} - - async def get_api_key_metadata( prisma_client: PrismaClient, api_keys: AbstractSet[str], @@ -635,7 +488,7 @@ async def get_api_key_metadata( return result return { **result, - **(await _recover_double_hashed_key_metadata(prisma_client, still_missing)), + **(await recover_double_hashed_key_metadata(prisma_client, still_missing)), } diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py new file mode 100644 index 00000000000..4d85357b7a9 --- /dev/null +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -0,0 +1,223 @@ +from collections.abc import Mapping, Sequence, Set as AbstractSet +from typing import Final, Protocol + +from typing_extensions import TypedDict + +from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash +from litellm.proxy.utils import PrismaClient, hash_token +from litellm.repositories.table_repositories import DeletedVerificationTokenRepository +from litellm.repositories.verification_token_repository import ( + VerificationTokenRepository, +) + +# Cap reverse-hash scans so a Usage page with orphaned double-hashed api_key +# values cannot pull an unbounded VerificationToken table into memory. +_MAX_DOUBLE_HASH_TOKEN_SCAN: Final = 10_000 + +_SPEND_LOGS_KEY_METADATA_SQL: Final = """ +SELECT DISTINCT ON (api_key) + api_key, + metadata->>'user_api_key_alias' AS key_alias, + metadata->>'user_api_key_team_id' AS team_id +FROM "LiteLLM_SpendLogs" +WHERE api_key = ANY($1::text[]) + AND ( + NULLIF(metadata->>'user_api_key_alias', '') IS NOT NULL + OR NULLIF(metadata->>'user_api_key_team_id', '') IS NOT NULL + ) +ORDER BY api_key, "startTime" DESC NULLS LAST +""" + + +class KeyMetadataDict(TypedDict, total=False): + key_alias: str | None + team_id: str | None + + +class _TokenAliasRecord(Protocol): + @property + def token(self) -> str: ... + + @property + def key_alias(self) -> str | None: ... + + @property + def team_id(self) -> str | None: ... + + +def _token_digest_metadata( + records: Sequence[_TokenAliasRecord], + wanted: AbstractSet[str], +) -> dict[str, KeyMetadataDict]: + return { + digested: {"key_alias": record.key_alias, "team_id": record.team_id} + for record in records + for digested in (hash_token(record.token),) + if digested in wanted + } + + +async def _reverse_hash_active_key_metadata( + prisma_client: PrismaClient, + wanted: AbstractSet[str], +) -> dict[str, KeyMetadataDict]: + try: + active_records: Final[Sequence[_TokenAliasRecord]] = await VerificationTokenRepository( + prisma_client + ).table.find_many(take=_MAX_DOUBLE_HASH_TOKEN_SCAN) + except Exception as e: + verbose_proxy_logger.warning( + "Failed reverse-hash recovery against active keys for %d missing keys: %s", + len(wanted), + e, + ) + return {} + return _token_digest_metadata(active_records, wanted) + + +async def _reverse_hash_deleted_key_metadata( + prisma_client: PrismaClient, + wanted: AbstractSet[str], +) -> dict[str, KeyMetadataDict]: + try: + deleted_records: Final[Sequence[_TokenAliasRecord]] = await DeletedVerificationTokenRepository( + prisma_client + ).table.find_many( + take=_MAX_DOUBLE_HASH_TOKEN_SCAN, + order={"deleted_at": "desc"}, + ) + except Exception as e: + verbose_proxy_logger.warning( + "Failed reverse-hash recovery against deleted keys for %d missing keys: %s", + len(wanted), + e, + ) + return {} + return _token_digest_metadata(deleted_records, wanted) + + +async def _reverse_hash_key_metadata( + prisma_client: PrismaClient, + wanted: AbstractSet[str], +) -> dict[str, KeyMetadataDict]: + from_active: Final = await _reverse_hash_active_key_metadata(prisma_client, wanted) + still_wanted: Final = wanted - frozenset(from_active) + if not still_wanted: + return from_active + return {**from_active, **(await _reverse_hash_deleted_key_metadata(prisma_client, still_wanted))} + + +async def _spend_logs_key_metadata( + prisma_client: PrismaClient, + wanted: AbstractSet[str], +) -> dict[str, KeyMetadataDict]: + try: + spend_log_rows: Final = await prisma_client.db.query_raw( + _SPEND_LOGS_KEY_METADATA_SQL, + list(wanted), + ) + except Exception as e: + verbose_proxy_logger.warning( + "Failed SpendLogs metadata recovery for %d missing keys: %s", + len(wanted), + e, + ) + return {} + + if not isinstance(spend_log_rows, list): + return {} + + return { + row["api_key"]: { + "key_alias": row.get("key_alias"), + "team_id": row.get("team_id"), + } + for row in spend_log_rows + if isinstance(row, dict) + and isinstance(row.get("api_key"), str) + and row["api_key"] in wanted + } + + +async def recover_double_hashed_key_metadata( + prisma_client: PrismaClient, + missing_keys: AbstractSet[str], +) -> dict[str, KeyMetadataDict]: + """ + Recover key_alias/team_id for DailyUserSpend.api_key values that were + double-hashed by the v1.99 spend-log provenance gate. + + Those rows store hash(VerificationToken.token) instead of the token, so the + exact join misses. Prefer a bounded reverse-hash against active/deleted + tokens; fall back to the alias/team stamped into SpendLogs metadata (which + stayed correct even when api_key did not). + """ + sha_missing: Final = frozenset(key for key in missing_keys if is_valid_sha256_hash(key)) + if not sha_missing: + return {} + + from_tokens: Final = await _reverse_hash_key_metadata(prisma_client, sha_missing) + still_missing: Final = sha_missing - frozenset(from_tokens) + if not still_missing: + return from_tokens + + return {**from_tokens, **(await _spend_logs_key_metadata(prisma_client, still_missing))} + + +def _row_with_recovered_alias( + row: Mapping[str, object], + recovered: Mapping[str, KeyMetadataDict], + *, + api_key_field: str, + alias_field: str, + team_id_field: str, +) -> Mapping[str, object]: + api_key: Final = row.get(api_key_field) + if not isinstance(api_key, str) or api_key not in recovered: + return row + meta: Final = recovered[api_key] + return { + **row, + alias_field: meta.get("key_alias") or row.get(alias_field), + team_id_field: meta.get("team_id") or row.get(team_id_field), + } + + +async def fill_missing_api_key_aliases( + prisma_client: PrismaClient, + rows: Sequence[Mapping[str, object]], + *, + api_key_field: str = "api_key", + alias_field: str = "api_key_alias", + team_id_field: str = "team_id", +) -> tuple[Mapping[str, object], ...]: + """ + Fill null api_key_alias / team_id on export rows whose api_key was double-hashed. + + Used by CloudZero and Focus, which join DailyUserSpend.api_key to + VerificationToken.token and otherwise export null aliases for those rows. + """ + missing_keys: Final = frozenset( + key + for row in rows + for key in (row.get(api_key_field),) + if isinstance(key, str) and key and row.get(alias_field) in (None, "") + ) + if not missing_keys: + return tuple(rows) + + recovered: Final = await recover_double_hashed_key_metadata(prisma_client, missing_keys) + if not recovered: + return tuple(rows) + + return tuple( + _row_with_recovered_alias( + row, + recovered, + api_key_field=api_key_field, + alias_field=alias_field, + team_id_field=team_id_field, + ) + for row in rows + ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py new file mode 100644 index 00000000000..6b12676e82b --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py @@ -0,0 +1,64 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.proxy.spend_tracking.key_metadata_recovery import ( + fill_missing_api_key_aliases, + recover_double_hashed_key_metadata, +) +from litellm.proxy.utils import hash_token + + +@pytest.mark.asyncio +async def test_recover_double_hashed_key_metadata_via_reverse_hash(): + token = "a" * 64 + double_hashed = hash_token(token) + mock_prisma = MagicMock() + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[SimpleNamespace(token=token, key_alias="batch-worker", team_id="team-1")] + ) + mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma.db.query_raw = AsyncMock(return_value=[]) + + result = await recover_double_hashed_key_metadata(mock_prisma, {double_hashed}) + + assert result[double_hashed]["key_alias"] == "batch-worker" + assert result[double_hashed]["team_id"] == "team-1" + mock_prisma.db.query_raw.assert_not_called() + + +@pytest.mark.asyncio +async def test_fill_missing_api_key_aliases_updates_null_alias_rows(): + token = "c" * 64 + double_hashed = hash_token(token) + mock_prisma = MagicMock() + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[SimpleNamespace(token=token, key_alias="recovered-alias", team_id="team-9")] + ) + mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma.db.query_raw = AsyncMock(return_value=[]) + + rows = ( + { + "api_key": double_hashed, + "api_key_alias": None, + "team_id": None, + "user_email": "owner@example.com", + "spend": 12.5, + }, + { + "api_key": "already-joined-token", + "api_key_alias": "named-key", + "team_id": "team-ok", + "user_email": "other@example.com", + "spend": 1.0, + }, + ) + + filled = await fill_missing_api_key_aliases(mock_prisma, rows) + + assert filled[0]["api_key_alias"] == "recovered-alias" + assert filled[0]["team_id"] == "team-9" + assert filled[0]["user_email"] == "owner@example.com" + assert filled[1]["api_key_alias"] == "named-key" From 6f0f2fcc8d00158fa7eb55b0b5781af6fd279acc Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 3 Sep 2026 14:27:12 +0000 Subject: [PATCH 15/88] fix(spend): restore user_email for double-hashed keys and persist it in spend logs Recovery now resolves the key owner's email from UserTable via the recovered token user_id, and SpendLogsMetadata keeps user_api_key_user_email so new batch/export consumers see email without a separate user join. Co-authored-by: Mateo Wang --- litellm/proxy/_types.py | 1 + .../spend_tracking/key_metadata_recovery.py | 101 +++++++++++++++--- .../spend_tracking/spend_tracking_utils.py | 1 + .../test_common_daily_activity.py | 13 ++- .../test_key_metadata_recovery.py | 31 +++++- 5 files changed, 127 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 849e54c65aa..54f1bfcad50 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3645,6 +3645,7 @@ class SpendLogsMetadata(TypedDict): user_api_key_project_alias: str | None user_api_key_org_id: str | None user_api_key_user_id: str | None + user_api_key_user_email: str | None user_api_key_team_alias: str | None spend_logs_metadata: dict | None # special param to log k,v pairs to spendlogs for a call requester_ip_address: str | None diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index 4d85357b7a9..7ad1c77ff57 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -7,6 +7,7 @@ from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash from litellm.proxy.utils import PrismaClient, hash_token from litellm.repositories.table_repositories import DeletedVerificationTokenRepository +from litellm.repositories.user_repository import UserRepository from litellm.repositories.verification_token_repository import ( VerificationTokenRepository, ) @@ -19,12 +20,15 @@ _SPEND_LOGS_KEY_METADATA_SQL: Final = """ SELECT DISTINCT ON (api_key) api_key, metadata->>'user_api_key_alias' AS key_alias, - metadata->>'user_api_key_team_id' AS team_id + metadata->>'user_api_key_team_id' AS team_id, + metadata->>'user_api_key_user_id' AS user_id, + metadata->>'user_api_key_user_email' AS user_email FROM "LiteLLM_SpendLogs" WHERE api_key = ANY($1::text[]) AND ( NULLIF(metadata->>'user_api_key_alias', '') IS NOT NULL OR NULLIF(metadata->>'user_api_key_team_id', '') IS NOT NULL + OR NULLIF(metadata->>'user_api_key_user_email', '') IS NOT NULL ) ORDER BY api_key, "startTime" DESC NULLS LAST """ @@ -33,6 +37,8 @@ ORDER BY api_key, "startTime" DESC NULLS LAST class KeyMetadataDict(TypedDict, total=False): key_alias: str | None team_id: str | None + user_id: str | None + user_email: str | None class _TokenAliasRecord(Protocol): @@ -45,13 +51,20 @@ class _TokenAliasRecord(Protocol): @property def team_id(self) -> str | None: ... + @property + def user_id(self) -> str | None: ... + def _token_digest_metadata( records: Sequence[_TokenAliasRecord], wanted: AbstractSet[str], ) -> dict[str, KeyMetadataDict]: return { - digested: {"key_alias": record.key_alias, "team_id": record.team_id} + digested: { + "key_alias": record.key_alias, + "team_id": record.team_id, + "user_id": getattr(record, "user_id", None), + } for record in records for digested in (hash_token(record.token),) if digested in wanted @@ -132,6 +145,8 @@ async def _spend_logs_key_metadata( row["api_key"]: { "key_alias": row.get("key_alias"), "team_id": row.get("team_id"), + "user_id": row.get("user_id"), + "user_email": row.get("user_email"), } for row in spend_log_rows if isinstance(row, dict) @@ -140,18 +155,67 @@ async def _spend_logs_key_metadata( } +async def _emails_for_user_ids( + prisma_client: PrismaClient, + user_ids: AbstractSet[str], +) -> Mapping[str, str]: + if not user_ids: + return {} + try: + users: Final = await UserRepository(prisma_client).table.find_many( + where={"user_id": {"in": list(user_ids)}} + ) + except Exception as e: + verbose_proxy_logger.warning( + "Failed user_email recovery for %d user ids: %s", + len(user_ids), + e, + ) + return {} + return { + user.user_id: user.user_email + for user in users + if getattr(user, "user_id", None) and getattr(user, "user_email", None) + } + + +def _meta_with_email(meta: KeyMetadataDict, emails: Mapping[str, str]) -> KeyMetadataDict: + if meta.get("user_email"): + return meta + user_id: Final = meta.get("user_id") + if not isinstance(user_id, str) or user_id not in emails: + return meta + return {**meta, "user_email": emails[user_id]} + + +async def _with_user_emails( + prisma_client: PrismaClient, + recovered: Mapping[str, KeyMetadataDict], +) -> dict[str, KeyMetadataDict]: + needing_email: Final = frozenset( + user_id + for meta in recovered.values() + for user_id in (meta.get("user_id"),) + if isinstance(user_id, str) and user_id and not meta.get("user_email") + ) + emails: Final = await _emails_for_user_ids(prisma_client, needing_email) + if not emails: + return dict(recovered) + return {api_key: _meta_with_email(meta, emails) for api_key, meta in recovered.items()} + + async def recover_double_hashed_key_metadata( prisma_client: PrismaClient, missing_keys: AbstractSet[str], ) -> dict[str, KeyMetadataDict]: """ - Recover key_alias/team_id for DailyUserSpend.api_key values that were - double-hashed by the v1.99 spend-log provenance gate. + Recover key_alias/team_id/user_email for DailyUserSpend.api_key values that + were double-hashed by the v1.99 spend-log provenance gate. Those rows store hash(VerificationToken.token) instead of the token, so the exact join misses. Prefer a bounded reverse-hash against active/deleted - tokens; fall back to the alias/team stamped into SpendLogs metadata (which - stayed correct even when api_key did not). + tokens; fall back to SpendLogs metadata. Emails come from SpendLogs when + present, otherwise from UserTable via the recovered key's user_id. """ sha_missing: Final = frozenset(key for key in missing_keys if is_valid_sha256_hash(key)) if not sha_missing: @@ -159,19 +223,22 @@ async def recover_double_hashed_key_metadata( from_tokens: Final = await _reverse_hash_key_metadata(prisma_client, sha_missing) still_missing: Final = sha_missing - frozenset(from_tokens) - if not still_missing: - return from_tokens - - return {**from_tokens, **(await _spend_logs_key_metadata(prisma_client, still_missing))} + recovered: Final = ( + from_tokens + if not still_missing + else {**from_tokens, **(await _spend_logs_key_metadata(prisma_client, still_missing))} + ) + return await _with_user_emails(prisma_client, recovered) -def _row_with_recovered_alias( +def _row_with_recovered_fields( row: Mapping[str, object], recovered: Mapping[str, KeyMetadataDict], *, api_key_field: str, alias_field: str, team_id_field: str, + user_email_field: str, ) -> Mapping[str, object]: api_key: Final = row.get(api_key_field) if not isinstance(api_key, str) or api_key not in recovered: @@ -181,6 +248,7 @@ def _row_with_recovered_alias( **row, alias_field: meta.get("key_alias") or row.get(alias_field), team_id_field: meta.get("team_id") or row.get(team_id_field), + user_email_field: meta.get("user_email") or row.get(user_email_field), } @@ -191,9 +259,11 @@ async def fill_missing_api_key_aliases( api_key_field: str = "api_key", alias_field: str = "api_key_alias", team_id_field: str = "team_id", + user_email_field: str = "user_email", ) -> tuple[Mapping[str, object], ...]: """ - Fill null api_key_alias / team_id on export rows whose api_key was double-hashed. + Fill null api_key_alias / team_id / user_email on export rows whose api_key + was double-hashed. Used by CloudZero and Focus, which join DailyUserSpend.api_key to VerificationToken.token and otherwise export null aliases for those rows. @@ -202,7 +272,9 @@ async def fill_missing_api_key_aliases( key for row in rows for key in (row.get(api_key_field),) - if isinstance(key, str) and key and row.get(alias_field) in (None, "") + if isinstance(key, str) + and key + and (row.get(alias_field) in (None, "") or row.get(user_email_field) in (None, "")) ) if not missing_keys: return tuple(rows) @@ -212,12 +284,13 @@ async def fill_missing_api_key_aliases( return tuple(rows) return tuple( - _row_with_recovered_alias( + _row_with_recovered_fields( row, recovered, api_key_field=api_key_field, alias_field=alias_field, team_id_field=team_id_field, + user_email_field=user_email_field, ) for row in rows ) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 7442d71bd96..93d5dd09b0c 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -138,6 +138,7 @@ def _get_spend_logs_metadata( user_api_key_project_alias=None, user_api_key_org_id=None, user_api_key_user_id=None, + user_api_key_user_email=None, user_api_key_team_alias=None, spend_logs_metadata=None, requester_ip_address=None, diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index 78752495f25..2418a1cf245 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -471,10 +471,20 @@ async def test_get_api_key_metadata_recovers_double_hashed_key_via_reverse_hash( mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock( side_effect=[ [], # exact join miss - [SimpleNamespace(token=token, key_alias="batch-worker", team_id="team-1")], + [ + SimpleNamespace( + token=token, + key_alias="batch-worker", + team_id="team-1", + user_id="alice", + ) + ], ] ) mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_usertable.find_many = AsyncMock( + return_value=[SimpleNamespace(user_id="alice", user_email="alice@example.com")] + ) mock_prisma.db.query_raw = AsyncMock(return_value=[]) result = await get_api_key_metadata( @@ -484,6 +494,7 @@ async def test_get_api_key_metadata_recovers_double_hashed_key_via_reverse_hash( assert result[double_hashed]["key_alias"] == "batch-worker" assert result[double_hashed]["team_id"] == "team-1" + assert result[double_hashed]["user_email"] == "alice@example.com" mock_prisma.db.query_raw.assert_not_called() diff --git a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py index 6b12676e82b..3d08b2909c4 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py +++ b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py @@ -16,27 +16,48 @@ async def test_recover_double_hashed_key_metadata_via_reverse_hash(): double_hashed = hash_token(token) mock_prisma = MagicMock() mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock( - return_value=[SimpleNamespace(token=token, key_alias="batch-worker", team_id="team-1")] + return_value=[ + SimpleNamespace( + token=token, + key_alias="batch-worker", + team_id="team-1", + user_id="alice", + ) + ] ) mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_usertable.find_many = AsyncMock( + return_value=[SimpleNamespace(user_id="alice", user_email="alice@example.com")] + ) mock_prisma.db.query_raw = AsyncMock(return_value=[]) result = await recover_double_hashed_key_metadata(mock_prisma, {double_hashed}) assert result[double_hashed]["key_alias"] == "batch-worker" assert result[double_hashed]["team_id"] == "team-1" + assert result[double_hashed]["user_email"] == "alice@example.com" mock_prisma.db.query_raw.assert_not_called() @pytest.mark.asyncio -async def test_fill_missing_api_key_aliases_updates_null_alias_rows(): +async def test_fill_missing_api_key_aliases_updates_null_alias_and_email_rows(): token = "c" * 64 double_hashed = hash_token(token) mock_prisma = MagicMock() mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock( - return_value=[SimpleNamespace(token=token, key_alias="recovered-alias", team_id="team-9")] + return_value=[ + SimpleNamespace( + token=token, + key_alias="recovered-alias", + team_id="team-9", + user_id="bob", + ) + ] ) mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_usertable.find_many = AsyncMock( + return_value=[SimpleNamespace(user_id="bob", user_email="bob@example.com")] + ) mock_prisma.db.query_raw = AsyncMock(return_value=[]) rows = ( @@ -44,7 +65,7 @@ async def test_fill_missing_api_key_aliases_updates_null_alias_rows(): "api_key": double_hashed, "api_key_alias": None, "team_id": None, - "user_email": "owner@example.com", + "user_email": None, "spend": 12.5, }, { @@ -60,5 +81,5 @@ async def test_fill_missing_api_key_aliases_updates_null_alias_rows(): assert filled[0]["api_key_alias"] == "recovered-alias" assert filled[0]["team_id"] == "team-9" - assert filled[0]["user_email"] == "owner@example.com" + assert filled[0]["user_email"] == "bob@example.com" assert filled[1]["api_key_alias"] == "named-key" From fd520e53abba19b35eaaa212515764022a253b05 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 3 Sep 2026 14:34:44 +0000 Subject: [PATCH 16/88] style(spend): satisfy ruff format on key metadata recovery Lint CI failed because ruff format splits the Set alias import and collapses a couple of long lines. Co-authored-by: Mateo Wang --- litellm/proxy/spend_tracking/key_metadata_recovery.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index 7ad1c77ff57..33339f16d52 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -1,4 +1,5 @@ -from collections.abc import Mapping, Sequence, Set as AbstractSet +from collections.abc import Mapping, Sequence +from collections.abc import Set as AbstractSet from typing import Final, Protocol from typing_extensions import TypedDict @@ -149,9 +150,7 @@ async def _spend_logs_key_metadata( "user_email": row.get("user_email"), } for row in spend_log_rows - if isinstance(row, dict) - and isinstance(row.get("api_key"), str) - and row["api_key"] in wanted + if isinstance(row, dict) and isinstance(row.get("api_key"), str) and row["api_key"] in wanted } @@ -162,9 +161,7 @@ async def _emails_for_user_ids( if not user_ids: return {} try: - users: Final = await UserRepository(prisma_client).table.find_many( - where={"user_id": {"in": list(user_ids)}} - ) + users: Final = await UserRepository(prisma_client).table.find_many(where={"user_id": {"in": list(user_ids)}}) except Exception as e: verbose_proxy_logger.warning( "Failed user_email recovery for %d user ids: %s", From bbf4d1dc304827538e84ba66415466719136dcca Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 3 Sep 2026 14:44:02 +0000 Subject: [PATCH 17/88] fix(spend): catch PrismaError instead of bare Exception in key recovery The Usage recovery path was adding four BLE001 hits and failing the strict-rule budget. Soft-fail only on PrismaError so a down token table still falls through to SpendLogs. Co-authored-by: Mateo Wang --- .../spend_tracking/key_metadata_recovery.py | 84 +++++++++---------- .../test_key_metadata_recovery.py | 28 +++++++ 2 files changed, 70 insertions(+), 42 deletions(-) diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index 33339f16d52..78169283dff 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -1,7 +1,8 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from collections.abc import Set as AbstractSet -from typing import Final, Protocol +from typing import Final, Protocol, TypeVar +from prisma.errors import PrismaError from typing_extensions import TypedDict from litellm._logging import verbose_proxy_logger @@ -13,6 +14,8 @@ from litellm.repositories.verification_token_repository import ( VerificationTokenRepository, ) +_T = TypeVar("_T") + # Cap reverse-hash scans so a Usage page with orphaned double-hashed api_key # values cannot pull an unbounded VerificationToken table into memory. _MAX_DOUBLE_HASH_TOKEN_SCAN: Final = 10_000 @@ -56,6 +59,18 @@ class _TokenAliasRecord(Protocol): def user_id(self) -> str | None: ... +async def _db_or_empty( + load: Callable[[], Awaitable[_T]], + warning: str, + count: int, +) -> _T | None: + try: + return await load() + except PrismaError as e: + verbose_proxy_logger.warning(warning, count, e) + return None + + def _token_digest_metadata( records: Sequence[_TokenAliasRecord], wanted: AbstractSet[str], @@ -76,16 +91,12 @@ async def _reverse_hash_active_key_metadata( prisma_client: PrismaClient, wanted: AbstractSet[str], ) -> dict[str, KeyMetadataDict]: - try: - active_records: Final[Sequence[_TokenAliasRecord]] = await VerificationTokenRepository( - prisma_client - ).table.find_many(take=_MAX_DOUBLE_HASH_TOKEN_SCAN) - except Exception as e: - verbose_proxy_logger.warning( - "Failed reverse-hash recovery against active keys for %d missing keys: %s", - len(wanted), - e, - ) + active_records: Final = await _db_or_empty( + lambda: VerificationTokenRepository(prisma_client).table.find_many(take=_MAX_DOUBLE_HASH_TOKEN_SCAN), + "Failed reverse-hash recovery against active keys for %d missing keys: %s", + len(wanted), + ) + if active_records is None: return {} return _token_digest_metadata(active_records, wanted) @@ -94,19 +105,15 @@ async def _reverse_hash_deleted_key_metadata( prisma_client: PrismaClient, wanted: AbstractSet[str], ) -> dict[str, KeyMetadataDict]: - try: - deleted_records: Final[Sequence[_TokenAliasRecord]] = await DeletedVerificationTokenRepository( - prisma_client - ).table.find_many( + deleted_records: Final = await _db_or_empty( + lambda: DeletedVerificationTokenRepository(prisma_client).table.find_many( take=_MAX_DOUBLE_HASH_TOKEN_SCAN, order={"deleted_at": "desc"}, - ) - except Exception as e: - verbose_proxy_logger.warning( - "Failed reverse-hash recovery against deleted keys for %d missing keys: %s", - len(wanted), - e, - ) + ), + "Failed reverse-hash recovery against deleted keys for %d missing keys: %s", + len(wanted), + ) + if deleted_records is None: return {} return _token_digest_metadata(deleted_records, wanted) @@ -126,19 +133,14 @@ async def _spend_logs_key_metadata( prisma_client: PrismaClient, wanted: AbstractSet[str], ) -> dict[str, KeyMetadataDict]: - try: - spend_log_rows: Final = await prisma_client.db.query_raw( + spend_log_rows: Final = await _db_or_empty( + lambda: prisma_client.db.query_raw( _SPEND_LOGS_KEY_METADATA_SQL, list(wanted), - ) - except Exception as e: - verbose_proxy_logger.warning( - "Failed SpendLogs metadata recovery for %d missing keys: %s", - len(wanted), - e, - ) - return {} - + ), + "Failed SpendLogs metadata recovery for %d missing keys: %s", + len(wanted), + ) if not isinstance(spend_log_rows, list): return {} @@ -160,14 +162,12 @@ async def _emails_for_user_ids( ) -> Mapping[str, str]: if not user_ids: return {} - try: - users: Final = await UserRepository(prisma_client).table.find_many(where={"user_id": {"in": list(user_ids)}}) - except Exception as e: - verbose_proxy_logger.warning( - "Failed user_email recovery for %d user ids: %s", - len(user_ids), - e, - ) + users: Final = await _db_or_empty( + lambda: UserRepository(prisma_client).table.find_many(where={"user_id": {"in": list(user_ids)}}), + "Failed user_email recovery for %d user ids: %s", + len(user_ids), + ) + if users is None: return {} return { user.user_id: user.user_email diff --git a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py index 3d08b2909c4..6f0e912c5b1 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py +++ b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py @@ -2,6 +2,7 @@ from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock import pytest +from prisma.errors import PrismaError from litellm.proxy.spend_tracking.key_metadata_recovery import ( fill_missing_api_key_aliases, @@ -83,3 +84,30 @@ async def test_fill_missing_api_key_aliases_updates_null_alias_and_email_rows(): assert filled[0]["team_id"] == "team-9" assert filled[0]["user_email"] == "bob@example.com" assert filled[1]["api_key_alias"] == "named-key" + + +@pytest.mark.asyncio +async def test_recover_falls_back_to_spend_logs_when_token_scan_raises_prisma_error(): + token = "b" * 64 + double_hashed = hash_token(token) + mock_prisma = MagicMock() + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(side_effect=PrismaError("db down")) + mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(side_effect=PrismaError("db down")) + mock_prisma.db.litellm_usertable.find_many = AsyncMock(return_value=[]) + mock_prisma.db.query_raw = AsyncMock( + return_value=[ + { + "api_key": double_hashed, + "key_alias": "from-spend-logs", + "team_id": "team-sl", + "user_id": "carol", + "user_email": "carol@example.com", + } + ] + ) + + result = await recover_double_hashed_key_metadata(mock_prisma, {double_hashed}) + + assert result[double_hashed]["key_alias"] == "from-spend-logs" + assert result[double_hashed]["team_id"] == "team-sl" + assert result[double_hashed]["user_email"] == "carol@example.com" From f610d205431a35f0bce1e19e6f2ebd8775a4af39 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 3 Sep 2026 14:52:01 +0000 Subject: [PATCH 18/88] fix(spend): keep key recovery import-safe and LIT-clean The Python 3.10 smoke check imports the proxy without prisma, so PrismaError is loaded only inside the DB helper. Recovery now returns frozen mappings and ReadOnly TypedDict fields so the type-discipline budget stays put. Co-authored-by: Mateo Wang --- litellm/integrations/cloudzero/database.py | 2 +- litellm/integrations/focus/database.py | 2 +- litellm/proxy/_types.py | 2 +- .../common_daily_activity.py | 12 +- .../spend_tracking/key_metadata_recovery.py | 142 +++++++++++------- 5 files changed, 92 insertions(+), 68 deletions(-) diff --git a/litellm/integrations/cloudzero/database.py b/litellm/integrations/cloudzero/database.py index 8fedd4edac4..87f0c8bd160 100644 --- a/litellm/integrations/cloudzero/database.py +++ b/litellm/integrations/cloudzero/database.py @@ -101,6 +101,6 @@ class LiteLLMDatabase: # v1.99 double-hashed DailyUserSpend.api_key values miss the # VerificationToken join above; recover alias/team for those rows. recovered_rows: Final = await fill_missing_api_key_aliases(client, db_response) - return pl.DataFrame(list(recovered_rows), infer_schema_length=None) + return pl.DataFrame(tuple(recovered_rows), infer_schema_length=None) except Exception as e: raise Exception(f"Error retrieving usage data: {e}") diff --git a/litellm/integrations/focus/database.py b/litellm/integrations/focus/database.py index db9849bbbc9..96a32046e81 100644 --- a/litellm/integrations/focus/database.py +++ b/litellm/integrations/focus/database.py @@ -103,7 +103,7 @@ class FocusLiteLLMDatabase: # v1.99 double-hashed DailyUserSpend.api_key values miss the # VerificationToken join above; recover alias/team for those rows. recovered_rows: Final = await fill_missing_api_key_aliases(client, db_response) - return pl.DataFrame(list(recovered_rows), infer_schema_length=None) + return pl.DataFrame(tuple(recovered_rows), infer_schema_length=None) except Exception as exc: raise RuntimeError(f"Error retrieving usage data: {exc}") from exc diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 54f1bfcad50..a4519e175ee 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3645,7 +3645,7 @@ class SpendLogsMetadata(TypedDict): user_api_key_project_alias: str | None user_api_key_org_id: str | None user_api_key_user_id: str | None - user_api_key_user_email: str | None + user_api_key_user_email: ReadOnly[str | None] user_api_key_team_alias: str | None spend_logs_metadata: dict | None # special param to log k,v pairs to spendlogs for a call requester_ip_address: str | None diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index f687cedeee9..9853f05a068 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -2,7 +2,7 @@ import asyncio from collections.abc import Awaitable, Callable, Mapping, Sequence from collections.abc import Set as AbstractSet from datetime import datetime, timedelta, timezone -from types import SimpleNamespace +from types import MappingProxyType, SimpleNamespace from typing import TYPE_CHECKING, Final, Protocol from fastapi import HTTPException, status @@ -445,7 +445,7 @@ def update_breakdown_metrics( async def get_api_key_metadata( prisma_client: PrismaClient, api_keys: AbstractSet[str], -) -> dict[str, _KeyMetadataDict]: +) -> Mapping[str, _KeyMetadataDict]: """Get api key metadata, falling back to deleted keys table for keys not found in active table. This ensures that key_alias and team_id are preserved in historical activity logs @@ -483,13 +483,11 @@ async def get_api_key_metadata( e, ) - still_missing: Final = api_keys - set(result.keys()) + still_missing: Final = api_keys - frozenset(result) if not still_missing: return result - return { - **result, - **(await recover_double_hashed_key_metadata(prisma_client, still_missing)), - } + recovered: Final = await recover_double_hashed_key_metadata(prisma_client, still_missing) + return MappingProxyType({**result, **recovered}) def _adjust_dates_for_timezone( diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index 78169283dff..10ce4548214 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -1,9 +1,9 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence from collections.abc import Set as AbstractSet +from types import MappingProxyType from typing import Final, Protocol, TypeVar -from prisma.errors import PrismaError -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash @@ -16,8 +16,6 @@ from litellm.repositories.verification_token_repository import ( _T = TypeVar("_T") -# Cap reverse-hash scans so a Usage page with orphaned double-hashed api_key -# values cannot pull an unbounded VerificationToken table into memory. _MAX_DOUBLE_HASH_TOKEN_SCAN: Final = 10_000 _SPEND_LOGS_KEY_METADATA_SQL: Final = """ @@ -39,10 +37,14 @@ ORDER BY api_key, "startTime" DESC NULLS LAST class KeyMetadataDict(TypedDict, total=False): - key_alias: str | None - team_id: str | None - user_id: str | None - user_email: str | None + key_alias: ReadOnly[str | None] + team_id: ReadOnly[str | None] + user_id: ReadOnly[str | None] + user_email: ReadOnly[str | None] + + +_EMPTY_KEY_METADATA: Final[Mapping[str, KeyMetadataDict]] = MappingProxyType({}) +_EMPTY_EMAILS: Final[Mapping[str, str]] = MappingProxyType({}) class _TokenAliasRecord(Protocol): @@ -64,6 +66,8 @@ async def _db_or_empty( warning: str, count: int, ) -> _T | None: + from prisma.errors import PrismaError + try: return await load() except PrismaError as e: @@ -71,89 +75,104 @@ async def _db_or_empty( return None +def _record_metadata(record: _TokenAliasRecord) -> KeyMetadataDict: + meta: Final[KeyMetadataDict] = { + "key_alias": record.key_alias, + "team_id": record.team_id, + "user_id": getattr(record, "user_id", None), + } + return meta + + +def _spend_log_row_metadata(row: Mapping[str, object]) -> KeyMetadataDict: + meta: Final[KeyMetadataDict] = { + "key_alias": row.get("key_alias") if isinstance(row.get("key_alias"), str) else None, + "team_id": row.get("team_id") if isinstance(row.get("team_id"), str) else None, + "user_id": row.get("user_id") if isinstance(row.get("user_id"), str) else None, + "user_email": row.get("user_email") if isinstance(row.get("user_email"), str) else None, + } + return meta + + def _token_digest_metadata( records: Sequence[_TokenAliasRecord], wanted: AbstractSet[str], -) -> dict[str, KeyMetadataDict]: - return { - digested: { - "key_alias": record.key_alias, - "team_id": record.team_id, - "user_id": getattr(record, "user_id", None), +) -> Mapping[str, KeyMetadataDict]: + return MappingProxyType( + { + digested: _record_metadata(record) + for record in records + for digested in (hash_token(record.token),) + if digested in wanted } - for record in records - for digested in (hash_token(record.token),) - if digested in wanted - } + ) async def _reverse_hash_active_key_metadata( prisma_client: PrismaClient, wanted: AbstractSet[str], -) -> dict[str, KeyMetadataDict]: +) -> Mapping[str, KeyMetadataDict]: active_records: Final = await _db_or_empty( lambda: VerificationTokenRepository(prisma_client).table.find_many(take=_MAX_DOUBLE_HASH_TOKEN_SCAN), "Failed reverse-hash recovery against active keys for %d missing keys: %s", len(wanted), ) if active_records is None: - return {} + return _EMPTY_KEY_METADATA return _token_digest_metadata(active_records, wanted) async def _reverse_hash_deleted_key_metadata( prisma_client: PrismaClient, wanted: AbstractSet[str], -) -> dict[str, KeyMetadataDict]: +) -> Mapping[str, KeyMetadataDict]: deleted_records: Final = await _db_or_empty( lambda: DeletedVerificationTokenRepository(prisma_client).table.find_many( take=_MAX_DOUBLE_HASH_TOKEN_SCAN, - order={"deleted_at": "desc"}, + order={"deleted_at": "desc"}, # mutable-ok: Prisma find_many order= is a dict ), "Failed reverse-hash recovery against deleted keys for %d missing keys: %s", len(wanted), ) if deleted_records is None: - return {} + return _EMPTY_KEY_METADATA return _token_digest_metadata(deleted_records, wanted) async def _reverse_hash_key_metadata( prisma_client: PrismaClient, wanted: AbstractSet[str], -) -> dict[str, KeyMetadataDict]: +) -> Mapping[str, KeyMetadataDict]: from_active: Final = await _reverse_hash_active_key_metadata(prisma_client, wanted) still_wanted: Final = wanted - frozenset(from_active) if not still_wanted: return from_active - return {**from_active, **(await _reverse_hash_deleted_key_metadata(prisma_client, still_wanted))} + from_deleted: Final = await _reverse_hash_deleted_key_metadata(prisma_client, still_wanted) + return MappingProxyType({**from_active, **from_deleted}) async def _spend_logs_key_metadata( prisma_client: PrismaClient, wanted: AbstractSet[str], -) -> dict[str, KeyMetadataDict]: +) -> Mapping[str, KeyMetadataDict]: spend_log_rows: Final = await _db_or_empty( lambda: prisma_client.db.query_raw( _SPEND_LOGS_KEY_METADATA_SQL, - list(wanted), + tuple(wanted), ), "Failed SpendLogs metadata recovery for %d missing keys: %s", len(wanted), ) if not isinstance(spend_log_rows, list): - return {} + return _EMPTY_KEY_METADATA - return { - row["api_key"]: { - "key_alias": row.get("key_alias"), - "team_id": row.get("team_id"), - "user_id": row.get("user_id"), - "user_email": row.get("user_email"), + return MappingProxyType( + { + row["api_key"]: _spend_log_row_metadata(row) + for row in spend_log_rows + if isinstance(row, dict) and isinstance(row.get("api_key"), str) and row["api_key"] in wanted } - for row in spend_log_rows - if isinstance(row, dict) and isinstance(row.get("api_key"), str) and row["api_key"] in wanted - } + ) async def _emails_for_user_ids( @@ -161,19 +180,23 @@ async def _emails_for_user_ids( user_ids: AbstractSet[str], ) -> Mapping[str, str]: if not user_ids: - return {} + return _EMPTY_EMAILS users: Final = await _db_or_empty( - lambda: UserRepository(prisma_client).table.find_many(where={"user_id": {"in": list(user_ids)}}), + lambda: UserRepository(prisma_client).table.find_many( + where={"user_id": {"in": tuple(user_ids)}}, # mutable-ok: Prisma find_many where= is a dict + ), "Failed user_email recovery for %d user ids: %s", len(user_ids), ) if users is None: - return {} - return { - user.user_id: user.user_email - for user in users - if getattr(user, "user_id", None) and getattr(user, "user_email", None) - } + return _EMPTY_EMAILS + return MappingProxyType( + { + user.user_id: user.user_email + for user in users + if getattr(user, "user_id", None) and getattr(user, "user_email", None) + } + ) def _meta_with_email(meta: KeyMetadataDict, emails: Mapping[str, str]) -> KeyMetadataDict: @@ -182,13 +205,14 @@ def _meta_with_email(meta: KeyMetadataDict, emails: Mapping[str, str]) -> KeyMet user_id: Final = meta.get("user_id") if not isinstance(user_id, str) or user_id not in emails: return meta - return {**meta, "user_email": emails[user_id]} + updated: Final[KeyMetadataDict] = {**meta, "user_email": emails[user_id]} + return updated async def _with_user_emails( prisma_client: PrismaClient, recovered: Mapping[str, KeyMetadataDict], -) -> dict[str, KeyMetadataDict]: +) -> Mapping[str, KeyMetadataDict]: needing_email: Final = frozenset( user_id for meta in recovered.values() @@ -197,14 +221,14 @@ async def _with_user_emails( ) emails: Final = await _emails_for_user_ids(prisma_client, needing_email) if not emails: - return dict(recovered) - return {api_key: _meta_with_email(meta, emails) for api_key, meta in recovered.items()} + return recovered + return MappingProxyType({api_key: _meta_with_email(meta, emails) for api_key, meta in recovered.items()}) async def recover_double_hashed_key_metadata( prisma_client: PrismaClient, missing_keys: AbstractSet[str], -) -> dict[str, KeyMetadataDict]: +) -> Mapping[str, KeyMetadataDict]: """ Recover key_alias/team_id/user_email for DailyUserSpend.api_key values that were double-hashed by the v1.99 spend-log provenance gate. @@ -216,14 +240,14 @@ async def recover_double_hashed_key_metadata( """ sha_missing: Final = frozenset(key for key in missing_keys if is_valid_sha256_hash(key)) if not sha_missing: - return {} + return _EMPTY_KEY_METADATA from_tokens: Final = await _reverse_hash_key_metadata(prisma_client, sha_missing) still_missing: Final = sha_missing - frozenset(from_tokens) recovered: Final = ( from_tokens if not still_missing - else {**from_tokens, **(await _spend_logs_key_metadata(prisma_client, still_missing))} + else MappingProxyType({**from_tokens, **(await _spend_logs_key_metadata(prisma_client, still_missing))}) ) return await _with_user_emails(prisma_client, recovered) @@ -241,12 +265,14 @@ def _row_with_recovered_fields( if not isinstance(api_key, str) or api_key not in recovered: return row meta: Final = recovered[api_key] - return { - **row, - alias_field: meta.get("key_alias") or row.get(alias_field), - team_id_field: meta.get("team_id") or row.get(team_id_field), - user_email_field: meta.get("user_email") or row.get(user_email_field), - } + return MappingProxyType( + { + **row, + alias_field: meta.get("key_alias") or row.get(alias_field), + team_id_field: meta.get("team_id") or row.get(team_id_field), + user_email_field: meta.get("user_email") or row.get(user_email_field), + } + ) async def fill_missing_api_key_aliases( From e76a18ca3251489af9954e7a8ea30b3b763d6c01 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 3 Sep 2026 14:56:26 +0000 Subject: [PATCH 19/88] fix(usage): return user_email on key activity metadata Recovery already resolved the owner email for double-hashed spend keys, then Key Activity dropped it. The Usage payload now carries user_email and the key label falls back to that email before key-hash-... Co-authored-by: Mateo Wang --- .../common_daily_activity.py | 29 ++++++++++++++----- .../spend_tracking/key_metadata_recovery.py | 4 +-- .../common_daily_activity.py | 1 + .../test_common_daily_activity.py | 19 ++++++++++++ .../src/components/UsagePage/types.ts | 1 + .../src/components/activity_metrics.test.tsx | 13 ++++++++- .../src/components/activity_metrics.tsx | 2 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 ++ 8 files changed, 60 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 9853f05a068..833f463621a 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -6,12 +6,13 @@ from types import MappingProxyType, SimpleNamespace from typing import TYPE_CHECKING, Final, Protocol from fastapi import HTTPException, status -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.constants import PTU_SENTINEL_API_KEY from litellm.proxy._types import CommonProxyErrors from litellm.proxy.spend_tracking.key_metadata_recovery import ( + attach_user_emails, recover_double_hashed_key_metadata, ) from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attribution_enabled @@ -116,6 +117,8 @@ class DailySpendRecord(Protocol): class _KeyMetadataDict(TypedDict, total=False): key_alias: str | None team_id: str | None + user_id: ReadOnly[str | None] + user_email: ReadOnly[str | None] _WhereValue = str | dict[str, object] @@ -456,7 +459,12 @@ async def get_api_key_metadata( where={"token": {"in": list(api_keys)}} ) result: Final[dict[str, _KeyMetadataDict]] = { - k.token: {"key_alias": k.key_alias, "team_id": k.team_id} for k in key_records + k.token: { + "key_alias": k.key_alias, + "team_id": k.team_id, + "user_id": getattr(k, "user_id", None), + } + for k in key_records } # For any keys not found in the active table, check the deleted keys table @@ -475,6 +483,7 @@ async def get_api_key_metadata( result[k.token] = { "key_alias": k.key_alias, "team_id": k.team_id, + "user_id": getattr(k, "user_id", None), } except Exception as e: verbose_proxy_logger.warning( @@ -484,10 +493,12 @@ async def get_api_key_metadata( ) still_missing: Final = api_keys - frozenset(result) - if not still_missing: - return result - recovered: Final = await recover_double_hashed_key_metadata(prisma_client, still_missing) - return MappingProxyType({**result, **recovered}) + combined: Final = ( + result + if not still_missing + else MappingProxyType({**result, **(await recover_double_hashed_key_metadata(prisma_client, still_missing))}) + ) + return await attach_user_emails(prisma_client, combined) def _adjust_dates_for_timezone( @@ -961,7 +972,11 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics: def _key_metadata(api_key_metadata: Mapping[str, _KeyMetadataDict], api_key: str) -> KeyMetadata: meta: Final = api_key_metadata.get(api_key, {}) - return KeyMetadata(key_alias=meta.get("key_alias"), team_id=meta.get("team_id")) + return KeyMetadata( + key_alias=meta.get("key_alias"), + team_id=meta.get("team_id"), + user_email=meta.get("user_email"), + ) def _aggregate_grouping_sets_records_sync( diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index 10ce4548214..fae091d4948 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -209,7 +209,7 @@ def _meta_with_email(meta: KeyMetadataDict, emails: Mapping[str, str]) -> KeyMet return updated -async def _with_user_emails( +async def attach_user_emails( prisma_client: PrismaClient, recovered: Mapping[str, KeyMetadataDict], ) -> Mapping[str, KeyMetadataDict]: @@ -249,7 +249,7 @@ async def recover_double_hashed_key_metadata( if not still_missing else MappingProxyType({**from_tokens, **(await _spend_logs_key_metadata(prisma_client, still_missing))}) ) - return await _with_user_emails(prisma_client, recovered) + return await attach_user_emails(prisma_client, recovered) def _row_with_recovered_fields( diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 101405abf50..2b39c5dbb9b 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -43,6 +43,7 @@ class KeyMetadata(BaseModel): key_alias: str | None = None team_id: str | None = None + user_email: str | None = None class KeyMetricWithMetadata(MetricBase): diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index 2418a1cf245..f7ad01c3bba 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -516,6 +516,7 @@ async def test_get_api_key_metadata_recovers_double_hashed_key_via_spend_logs(): } ] ) + mock_prisma.db.litellm_usertable.find_many = AsyncMock(return_value=[]) result = await get_api_key_metadata( prisma_client=mock_prisma, @@ -527,6 +528,24 @@ async def test_get_api_key_metadata_recovers_double_hashed_key_via_spend_logs(): mock_prisma.db.query_raw.assert_called_once() +def test_key_metadata_includes_recovered_user_email(): + from litellm.proxy.management_endpoints.common_daily_activity import _key_metadata + + meta = _key_metadata( + { + "dirty-key": { + "key_alias": "batch-worker", + "team_id": "team-1", + "user_email": "alice@example.com", + } + }, + "dirty-key", + ) + + assert meta.key_alias == "batch-worker" + assert meta.user_email == "alice@example.com" + + @pytest.mark.asyncio async def test_tag_daily_activity_metadata_totals_not_zero(): """Test that tag daily activity returns correct metadata totals. diff --git a/ui/litellm-dashboard/src/components/UsagePage/types.ts b/ui/litellm-dashboard/src/components/UsagePage/types.ts index 8e7c1869df2..a10e9e68c4d 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/types.ts +++ b/ui/litellm-dashboard/src/components/UsagePage/types.ts @@ -46,6 +46,7 @@ export interface KeyMetricWithMetadata { export interface KeyMetadata { key_alias: string | null; team_id: string | null; + user_email?: string | null; tags?: { tag: string; usage: number }[]; } diff --git a/ui/litellm-dashboard/src/components/activity_metrics.test.tsx b/ui/litellm-dashboard/src/components/activity_metrics.test.tsx index 74d258e2bd0..b0fc8dc7866 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.test.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.test.tsx @@ -101,7 +101,7 @@ const createMockDailyData = ( }); const createMockKeyMetricWithMetadata = ( - metadata: { key_alias: string | null; team_id: string | null }, + metadata: { key_alias: string | null; team_id: string | null; user_email?: string | null }, metrics: typeof EMPTY_SPEND_METRICS = EMPTY_SPEND_METRICS, ): KeyMetricWithMetadata => ({ metrics, @@ -1450,6 +1450,17 @@ describe("formatKeyLabel", () => { expect(result).toBe("key-hash-actual-key (team: Test Team 1)"); }); + it("should use user_email when key_alias is null", () => { + const modelData = createMockKeyMetricWithMetadata({ + key_alias: null, + team_id: "team1", + user_email: "alice@example.com", + }); + + const result = formatKeyLabel(modelData, "actual-key", MOCK_TEAMS); + expect(result).toBe("alice@example.com (team: Test Team 1)"); + }); + it("should return key_alias with team_id when teams array is empty", () => { const modelData = createMockKeyMetricWithMetadata({ key_alias: "my-key", diff --git a/ui/litellm-dashboard/src/components/activity_metrics.tsx b/ui/litellm-dashboard/src/components/activity_metrics.tsx index a3fff08faae..e17263ab078 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.tsx @@ -433,7 +433,7 @@ export const ActivityMetrics: React.FC = ({ modelMetrics, // Helper function to format key label export const formatKeyLabel = (modelData: KeyMetricWithMetadata, model: string, teams: Team[]): string => { - const keyAlias = modelData.metadata.key_alias || `key-hash-${model}`; + const keyAlias = modelData.metadata.key_alias || modelData.metadata.user_email || `key-hash-${model}`; const teamId = modelData.metadata.team_id; if (teamId) { const teamAlias = resolveTeamAliasFromTeamID(teamId, teams); diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index b0f8e618645..8085967cf6a 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27994,6 +27994,8 @@ export interface components { key_alias?: string | null; /** Team Id */ team_id?: string | null; + /** User Email */ + user_email?: string | null; }; /** * KeyMetricWithMetadata From 6f4ea2d296311c93559f2f8d60d8690f35386e39 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 3 Sep 2026 14:58:05 +0000 Subject: [PATCH 20/88] fix(usage): label key activity tables with email when alias is missing Key Activity charts already fell back to user_email. The top-keys tables and usage export still printed '-' or a truncated hash. They now use the same alias-then-email label. Co-authored-by: Mateo Wang --- .../EntityUsage/entityUsageAggregations.ts | 4 +++- .../_components/components/UsagePageView.tsx | 4 +++- .../src/components/EntityUsageExport/utils.ts | 3 ++- .../components/UsagePage/keyActivityLabel.test.ts | 15 +++++++++++++++ .../src/components/UsagePage/keyActivityLabel.ts | 8 ++++++++ .../src/components/activity_metrics.tsx | 5 +++-- 6 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/UsagePage/keyActivityLabel.test.ts create mode 100644 ui/litellm-dashboard/src/components/UsagePage/keyActivityLabel.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/entityUsageAggregations.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/entityUsageAggregations.ts index a53b1d2827b..d482a5576ae 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/entityUsageAggregations.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/entityUsageAggregations.ts @@ -1,3 +1,4 @@ +import { keyActivityLabel } from "@/components/UsagePage/keyActivityLabel"; import { BreakdownMetrics, DailyData, KeyMetricWithMetadata, TagUsage } from "@/components/UsagePage/types"; export type ExtendedDailyData = DailyData & { @@ -118,6 +119,7 @@ export const getTopAPIKeys = (results: ExtendedDailyData[], topKeysLimit: number metadata: { key_alias: metrics.metadata.key_alias, team_id: metrics.metadata.team_id || null, + user_email: metrics.metadata.user_email, tags: tagDictionary[key] || [], }, }; @@ -137,7 +139,7 @@ export const getTopAPIKeys = (results: ExtendedDailyData[], topKeysLimit: number return Object.entries(keySpend) .map(([api_key, metrics]) => ({ api_key, - key_alias: metrics.metadata.key_alias || "-", // Using truncated key as alias + key_alias: keyActivityLabel(metrics.metadata), tags: metrics.metadata.tags || "-", spend: metrics.metrics.spend, })) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx index cbdfc8f39e6..29a81e1ae3f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx @@ -44,6 +44,7 @@ import { Tag } from "@/components/tag_management/types"; import UserAgentActivity from "@/components/user_agent_activity"; import ViewUserSpend from "@/components/view_user_spend"; import { usePaginatedDailyActivity } from "../hooks/usePaginatedDailyActivity"; +import { keyActivityLabel } from "@/components/UsagePage/keyActivityLabel"; import { DailyData, KeyMetricWithMetadata, MetricWithMetadata } from "@/components/UsagePage/types"; import { valueFormatterSpend } from "@/components/UsagePage/utils/value_formatters"; import { @@ -426,6 +427,7 @@ const UsagePage: React.FC = ({ teams, organizations }) => { metadata: { key_alias: metrics.metadata.key_alias, team_id: null, + user_email: metrics.metadata.user_email, tags: metrics.metadata.tags || [], }, }; @@ -445,7 +447,7 @@ const UsagePage: React.FC = ({ teams, organizations }) => { return Object.entries(keySpend) .map(([api_key, metrics]) => ({ api_key, - key_alias: metrics.metadata.key_alias || "-", + key_alias: keyActivityLabel(metrics.metadata), tags: metrics.metadata.tags || [], spend: metrics.metrics.spend, })) diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts index de637d5d627..8fd75134bcc 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts @@ -1,6 +1,7 @@ import { formatNumberWithCommas } from "@/utils/dataUtils"; import type { DateRangePickerValue } from "@/components/shared/date_picker_types"; import Papa from "papaparse"; +import { keyActivityLabel } from "@/components/UsagePage/keyActivityLabel"; import type { EntityBreakdown, EntitySpendData, EntityType, ExportMetadata, ExportScope } from "./types"; const resolveEntityDisplay = ( @@ -186,7 +187,7 @@ export const generateDailyWithKeysData = ( // Iterate through each API key in the breakdown Object.entries(apiKeyBreakdown).forEach(([keyId, keyData]: [string, any]) => { - const keyAlias = keyData?.metadata?.key_alias || null; + const keyAlias = keyActivityLabel(keyData?.metadata, "") || null; // Create unique key for aggregation: Date_EntityID_KeyID const uniqueKey = `${day.date}_${entityId}_${keyId}`; diff --git a/ui/litellm-dashboard/src/components/UsagePage/keyActivityLabel.test.ts b/ui/litellm-dashboard/src/components/UsagePage/keyActivityLabel.test.ts new file mode 100644 index 00000000000..eaf1985c5fa --- /dev/null +++ b/ui/litellm-dashboard/src/components/UsagePage/keyActivityLabel.test.ts @@ -0,0 +1,15 @@ +import { keyActivityLabel } from "./keyActivityLabel"; + +describe("keyActivityLabel", () => { + it("prefers key_alias", () => { + expect(keyActivityLabel({ key_alias: "batch-worker", user_email: "alice@example.com" })).toBe("batch-worker"); + }); + + it("falls back to user_email when alias is missing", () => { + expect(keyActivityLabel({ key_alias: null, user_email: "alice@example.com" })).toBe("alice@example.com"); + }); + + it("uses the fallback when both alias and email are missing", () => { + expect(keyActivityLabel({ key_alias: null, user_email: null }, "key-hash-abc")).toBe("key-hash-abc"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/UsagePage/keyActivityLabel.ts b/ui/litellm-dashboard/src/components/UsagePage/keyActivityLabel.ts new file mode 100644 index 00000000000..8b3a7eec916 --- /dev/null +++ b/ui/litellm-dashboard/src/components/UsagePage/keyActivityLabel.ts @@ -0,0 +1,8 @@ +import type { KeyMetadata } from "./types"; + +export function keyActivityLabel( + metadata: Pick | null | undefined, + fallback = "-", +): string { + return metadata?.key_alias || metadata?.user_email || fallback; +} diff --git a/ui/litellm-dashboard/src/components/activity_metrics.tsx b/ui/litellm-dashboard/src/components/activity_metrics.tsx index e17263ab078..f4348fb65ae 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.tsx @@ -7,6 +7,7 @@ import { ChevronDown } from "lucide-react"; import React, { useState } from "react"; import { Team } from "./key_team_helpers/key_list"; import KeyModelUsageView from "./UsagePage/components/KeyModelUsageView"; +import { keyActivityLabel } from "./UsagePage/keyActivityLabel"; import { DailyData, KeyMetricWithMetadata, ModelActivityData, TopApiKeyData, TopModelData } from "./UsagePage/types"; import { valueFormatter } from "./UsagePage/utils/value_formatters"; @@ -433,7 +434,7 @@ export const ActivityMetrics: React.FC = ({ modelMetrics, // Helper function to format key label export const formatKeyLabel = (modelData: KeyMetricWithMetadata, model: string, teams: Team[]): string => { - const keyAlias = modelData.metadata.key_alias || modelData.metadata.user_email || `key-hash-${model}`; + const keyAlias = keyActivityLabel(modelData.metadata, `key-hash-${model}`); const teamId = modelData.metadata.team_id; if (teamId) { const teamAlias = resolveTeamAliasFromTeamID(teamId, teams); @@ -516,7 +517,7 @@ export const processActivityData = ( if (!apiKeyBreakdown[apiKey]) { apiKeyBreakdown[apiKey] = { api_key: apiKey, - key_alias: keyData.metadata.key_alias, + key_alias: keyActivityLabel(keyData.metadata, "") || null, team_id: keyData.metadata.team_id, spend: 0, requests: 0, From eeb8b4f7f39afdd9f2148881d89d6c5a6fa951bb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 3 Sep 2026 15:00:31 +0000 Subject: [PATCH 21/88] test(spend): assert batch spend metadata keeps user email Co-authored-by: Mateo Wang --- .../proxy/spend_tracking/test_spend_tracking_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index bea1f8e2d6c..6cdf9e3fe7e 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -2789,6 +2789,7 @@ def test_get_logging_payload_batch_attribution_keeps_verification_token_hash(): parsed_meta = json.loads(payload["metadata"]) assert parsed_meta["user_api_key"] == token_hash assert parsed_meta["user_api_key_alias"] == "batch-creator" + assert parsed_meta["user_api_key_user_email"] == "alice@example.com" def test_get_spend_logs_metadata_provenance_bypass_requires_hash_match(): From e109d89c2041373a1744805e0397a181f49f427b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 3 Sep 2026 15:11:45 +0000 Subject: [PATCH 22/88] fix(usage): include user_email on daily activity key breakdowns Co-authored-by: Mateo Wang --- .../common_daily_activity.py | 57 ++++++----------- .../test_common_daily_activity.py | 63 +++++++++++++++++++ 2 files changed, 81 insertions(+), 39 deletions(-) diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 833f463621a..ce6a97708ab 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -115,12 +115,21 @@ class DailySpendRecord(Protocol): class _KeyMetadataDict(TypedDict, total=False): - key_alias: str | None - team_id: str | None + key_alias: ReadOnly[str | None] + team_id: ReadOnly[str | None] user_id: ReadOnly[str | None] user_email: ReadOnly[str | None] +def _key_metadata(api_key_metadata: Mapping[str, _KeyMetadataDict], api_key: str) -> KeyMetadata: + meta: Final = api_key_metadata.get(api_key, {}) + return KeyMetadata( + key_alias=meta.get("key_alias"), + team_id=meta.get("team_id"), + user_email=meta.get("user_email"), + ) + + _WhereValue = str | dict[str, object] @@ -289,10 +298,7 @@ def update_breakdown_metrics( if record.api_key not in breakdown.models[model_key].api_key_breakdown: breakdown.models[model_key].api_key_breakdown[record.api_key] = KeyMetricWithMetadata( metrics=SpendMetrics(), - metadata=KeyMetadata( - key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None), - team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None), - ), + metadata=_key_metadata(api_key_metadata, record.api_key), ) breakdown.models[model_key].api_key_breakdown[record.api_key].metrics = update_metrics( breakdown.models[model_key].api_key_breakdown[record.api_key].metrics, @@ -316,10 +322,7 @@ def update_breakdown_metrics( if record.api_key not in breakdown.model_groups[model_group_key].api_key_breakdown: breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key] = KeyMetricWithMetadata( metrics=SpendMetrics(), - metadata=KeyMetadata( - key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None), - team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None), - ), + metadata=_key_metadata(api_key_metadata, record.api_key), ) breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key].metrics = update_metrics( breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key].metrics, @@ -341,10 +344,7 @@ def update_breakdown_metrics( breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[record.api_key] = ( KeyMetricWithMetadata( metrics=SpendMetrics(), - metadata=KeyMetadata( - key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None), - team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None), - ), + metadata=_key_metadata(api_key_metadata, record.api_key), ) ) @@ -369,10 +369,7 @@ def update_breakdown_metrics( if record.api_key not in breakdown.providers[provider].api_key_breakdown: breakdown.providers[provider].api_key_breakdown[record.api_key] = KeyMetricWithMetadata( metrics=SpendMetrics(), - metadata=KeyMetadata( - key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None), - team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None), - ), + metadata=_key_metadata(api_key_metadata, record.api_key), ) breakdown.providers[provider].api_key_breakdown[record.api_key].metrics = update_metrics( breakdown.providers[provider].api_key_breakdown[record.api_key].metrics, @@ -394,10 +391,7 @@ def update_breakdown_metrics( if record.api_key not in breakdown.endpoints[record.endpoint].api_key_breakdown: breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key] = KeyMetricWithMetadata( metrics=SpendMetrics(), - metadata=KeyMetadata( - key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None), - team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None), - ), + metadata=_key_metadata(api_key_metadata, record.api_key), ) breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key].metrics = update_metrics( breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key].metrics, @@ -409,10 +403,7 @@ def update_breakdown_metrics( if record.api_key not in breakdown.api_keys: breakdown.api_keys[record.api_key] = KeyMetricWithMetadata( metrics=SpendMetrics(), - metadata=KeyMetadata( - key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None), - team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None), - ), # Add any api_key-specific metadata here + metadata=_key_metadata(api_key_metadata, record.api_key), ) breakdown.api_keys[record.api_key].metrics = update_metrics(breakdown.api_keys[record.api_key].metrics, record) @@ -432,10 +423,7 @@ def update_breakdown_metrics( if record.api_key not in breakdown.entities[entity_value].api_key_breakdown: breakdown.entities[entity_value].api_key_breakdown[record.api_key] = KeyMetricWithMetadata( metrics=SpendMetrics(), - metadata=KeyMetadata( - key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None), - team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None), - ), + metadata=_key_metadata(api_key_metadata, record.api_key), ) breakdown.entities[entity_value].api_key_breakdown[record.api_key].metrics = update_metrics( breakdown.entities[entity_value].api_key_breakdown[record.api_key].metrics, @@ -970,15 +958,6 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics: ) -def _key_metadata(api_key_metadata: Mapping[str, _KeyMetadataDict], api_key: str) -> KeyMetadata: - meta: Final = api_key_metadata.get(api_key, {}) - return KeyMetadata( - key_alias=meta.get("key_alias"), - team_id=meta.get("team_id"), - user_email=meta.get("user_email"), - ) - - def _aggregate_grouping_sets_records_sync( *, records: Sequence[_GroupingSetsRow], diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index f7ad01c3bba..b9c0d953086 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -546,6 +546,69 @@ def test_key_metadata_includes_recovered_user_email(): assert meta.user_email == "alice@example.com" +def test_update_breakdown_metrics_includes_user_email(): + from litellm.proxy.management_endpoints.common_daily_activity import update_breakdown_metrics + from litellm.types.proxy.management_endpoints.common_daily_activity import BreakdownMetrics + + breakdown = BreakdownMetrics() + record = SimpleNamespace( + api_key="dirty-key", + model="gpt-4o-mini", + model_group="grp", + mcp_namespaced_tool_name="srv/tool", + custom_llm_provider="openai", + endpoint="/v1/chat/completions", + spend=1.23, + prompt_tokens=1, + completion_tokens=1, + cache_read_input_tokens=0, + cache_creation_input_tokens=0, + compression_saved_tokens=0, + compression_savings_spend=0, + prompt_caching_savings_spend=0, + gateway_injected_caching_savings_spend=0, + autorouter_savings_spend=0, + total_tokens=2, + api_requests=1, + successful_requests=1, + failed_requests=0, + ptu_flat_cost=0.0, + user_id="alice", + ) + api_key_metadata = { + "dirty-key": { + "key_alias": "batch-worker", + "team_id": "team-1", + "user_email": "alice@example.com", + } + } + + update_breakdown_metrics( + breakdown, + record, + {}, + {}, + api_key_metadata, + entity_id_field="user_id", + ) + + expected = ("batch-worker", "alice@example.com") + top = breakdown.api_keys["dirty-key"].metadata + assert (top.key_alias, top.user_email) == expected + assert ( + breakdown.models["gpt-4o-mini"].api_key_breakdown["dirty-key"].metadata.key_alias, + breakdown.models["gpt-4o-mini"].api_key_breakdown["dirty-key"].metadata.user_email, + ) == expected + assert ( + breakdown.providers["openai"].api_key_breakdown["dirty-key"].metadata.key_alias, + breakdown.providers["openai"].api_key_breakdown["dirty-key"].metadata.user_email, + ) == expected + assert ( + breakdown.entities["alice"].api_key_breakdown["dirty-key"].metadata.key_alias, + breakdown.entities["alice"].api_key_breakdown["dirty-key"].metadata.user_email, + ) == expected + + @pytest.mark.asyncio async def test_tag_daily_activity_metadata_totals_not_zero(): """Test that tag daily activity returns correct metadata totals. From 7e4032cfcc319cf646537ee288fc3291e332af98 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 3 Sep 2026 15:16:37 +0000 Subject: [PATCH 23/88] chore(openapi): sync lazy snapshot with KeyMetadata.user_email Co-authored-by: Mateo Wang --- litellm/proxy/_lazy_openapi_snapshot.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 13c7a4c7cfa..e8e1f53b473 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -3151,6 +3151,17 @@ } ], "title": "Team Id" + }, + "user_email": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User Email" } }, "title": "KeyMetadata", From d3c839147edf831efc6ccc83ce8d927f751d4077 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 3 Sep 2026 15:21:35 +0000 Subject: [PATCH 24/88] fix(spend): keep CloudZero export and spend-log snapshots compatible with email recovery Co-authored-by: Mateo Wang --- litellm/integrations/cloudzero/database.py | 7 ++++++- litellm/integrations/focus/database.py | 7 ++++++- .../test_litellm/integrations/cloudzero/test_cloudzero.py | 5 +++++ .../spend_tracking/test_spend_management_endpoints.py | 1 + 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/cloudzero/database.py b/litellm/integrations/cloudzero/database.py index 87f0c8bd160..6a63868a08b 100644 --- a/litellm/integrations/cloudzero/database.py +++ b/litellm/integrations/cloudzero/database.py @@ -98,9 +98,14 @@ class LiteLLMDatabase: fill_missing_api_key_aliases, ) + usage_rows: Final = ( + db_response.to_dicts() + if isinstance(db_response, pl.DataFrame) + else db_response if isinstance(db_response, list) else [] + ) # v1.99 double-hashed DailyUserSpend.api_key values miss the # VerificationToken join above; recover alias/team for those rows. - recovered_rows: Final = await fill_missing_api_key_aliases(client, db_response) + recovered_rows: Final = await fill_missing_api_key_aliases(client, usage_rows) return pl.DataFrame(tuple(recovered_rows), infer_schema_length=None) except Exception as e: raise Exception(f"Error retrieving usage data: {e}") diff --git a/litellm/integrations/focus/database.py b/litellm/integrations/focus/database.py index 96a32046e81..da6458c9369 100644 --- a/litellm/integrations/focus/database.py +++ b/litellm/integrations/focus/database.py @@ -100,9 +100,14 @@ class FocusLiteLLMDatabase: fill_missing_api_key_aliases, ) + usage_rows: Final = ( + db_response.to_dicts() + if isinstance(db_response, pl.DataFrame) + else db_response if isinstance(db_response, list) else [] + ) # v1.99 double-hashed DailyUserSpend.api_key values miss the # VerificationToken join above; recover alias/team for those rows. - recovered_rows: Final = await fill_missing_api_key_aliases(client, db_response) + recovered_rows: Final = await fill_missing_api_key_aliases(client, usage_rows) return pl.DataFrame(tuple(recovered_rows), infer_schema_length=None) except Exception as exc: raise RuntimeError(f"Error retrieving usage data: {exc}") from exc diff --git a/tests/test_litellm/integrations/cloudzero/test_cloudzero.py b/tests/test_litellm/integrations/cloudzero/test_cloudzero.py index 2d51eeb9944..c543156eedd 100644 --- a/tests/test_litellm/integrations/cloudzero/test_cloudzero.py +++ b/tests/test_litellm/integrations/cloudzero/test_cloudzero.py @@ -74,6 +74,8 @@ class TestCloudZeroHourlyExport: fake_db = MagicMock() async def query_raw_mock(query: str, *params): + if "LiteLLM_SpendLogs" in query: + return [] start_time_utc = params[0] if len(params) > 0 else None end_time_utc = params[1] if len(params) > 1 else None limit = params[2] if len(params) > 2 else None @@ -146,6 +148,9 @@ class TestCloudZeroHourlyExport: return joined fake_db.query_raw = AsyncMock(side_effect=query_raw_mock) + fake_db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + fake_db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) + fake_db.litellm_usertable.find_many = AsyncMock(return_value=[]) fake_client.db = fake_db mock_prisma_client_getter.return_value = fake_client diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 30b086bab61..b62c1c076c8 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -483,6 +483,7 @@ ignored_keys = [ "metadata.user_api_key_project_alias", "metadata.user_api_key_org_id", "metadata.user_api_key_user_id", + "metadata.user_api_key_user_email", "metadata.user_api_key_team_alias", "metadata.spend_logs_metadata", "metadata.requester_ip_address", From 4db370e85194bded3df0ec73a71a777ff7f15ce5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 3 Sep 2026 15:25:46 +0000 Subject: [PATCH 25/88] style(spend): format CloudZero and Focus recovery row coercion Required lint failed ruff format on the ternary that unwraps a polars DataFrame or list before alias recovery Co-authored-by: Mateo Wang --- litellm/integrations/cloudzero/database.py | 4 +++- litellm/integrations/focus/database.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/cloudzero/database.py b/litellm/integrations/cloudzero/database.py index 6a63868a08b..e630bd85114 100644 --- a/litellm/integrations/cloudzero/database.py +++ b/litellm/integrations/cloudzero/database.py @@ -101,7 +101,9 @@ class LiteLLMDatabase: usage_rows: Final = ( db_response.to_dicts() if isinstance(db_response, pl.DataFrame) - else db_response if isinstance(db_response, list) else [] + else db_response + if isinstance(db_response, list) + else [] ) # v1.99 double-hashed DailyUserSpend.api_key values miss the # VerificationToken join above; recover alias/team for those rows. diff --git a/litellm/integrations/focus/database.py b/litellm/integrations/focus/database.py index da6458c9369..02b1e9e944b 100644 --- a/litellm/integrations/focus/database.py +++ b/litellm/integrations/focus/database.py @@ -103,7 +103,9 @@ class FocusLiteLLMDatabase: usage_rows: Final = ( db_response.to_dicts() if isinstance(db_response, pl.DataFrame) - else db_response if isinstance(db_response, list) else [] + else db_response + if isinstance(db_response, list) + else [] ) # v1.99 double-hashed DailyUserSpend.api_key values miss the # VerificationToken join above; recover alias/team for those rows. From d426b99f562672c930a473ec545dc8167e6345a0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 3 Sep 2026 15:44:18 +0000 Subject: [PATCH 26/88] fix(spend): page reverse-hash recovery past the first 10k keys Historical dirty spend on large installs was still unlabeled when the matching token sat past the first page. Keep scanning until the digest matches or the table ends. Co-authored-by: Mateo Wang --- .../spend_tracking/key_metadata_recovery.py | 95 ++++++++++++++----- .../test_key_metadata_recovery.py | 73 ++++++++++++++ 2 files changed, 142 insertions(+), 26 deletions(-) diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index fae091d4948..895f97e1a05 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -16,7 +16,7 @@ from litellm.repositories.verification_token_repository import ( _T = TypeVar("_T") -_MAX_DOUBLE_HASH_TOKEN_SCAN: Final = 10_000 +_TOKEN_SCAN_PAGE: Final = 10_000 _SPEND_LOGS_KEY_METADATA_SQL: Final = """ SELECT DISTINCT ON (api_key) @@ -108,46 +108,86 @@ def _token_digest_metadata( ) +async def _paginate_token_digest_metadata( + load_page: Callable[[int], Awaitable[Sequence[_TokenAliasRecord] | None]], + wanted: AbstractSet[str], + *, + page_size: int, + skip: int = 0, + accumulated: Mapping[str, KeyMetadataDict] = _EMPTY_KEY_METADATA, +) -> Mapping[str, KeyMetadataDict]: + if not wanted: + return accumulated + records: Final = await load_page(skip) + if records is None: + return accumulated + page_hits: Final = _token_digest_metadata(records, wanted) + combined: Final[Mapping[str, KeyMetadataDict]] = ( + MappingProxyType({**accumulated, **page_hits}) if page_hits else accumulated + ) + still_wanted: Final = wanted - frozenset(page_hits) + if not still_wanted or len(records) < page_size: + return combined + return await _paginate_token_digest_metadata( + load_page, + still_wanted, + page_size=page_size, + skip=skip + page_size, + accumulated=combined, + ) + + async def _reverse_hash_active_key_metadata( prisma_client: PrismaClient, wanted: AbstractSet[str], + *, + page_size: int, ) -> Mapping[str, KeyMetadataDict]: - active_records: Final = await _db_or_empty( - lambda: VerificationTokenRepository(prisma_client).table.find_many(take=_MAX_DOUBLE_HASH_TOKEN_SCAN), - "Failed reverse-hash recovery against active keys for %d missing keys: %s", - len(wanted), - ) - if active_records is None: - return _EMPTY_KEY_METADATA - return _token_digest_metadata(active_records, wanted) + async def load_page(skip: int) -> Sequence[_TokenAliasRecord] | None: + return await _db_or_empty( + lambda: VerificationTokenRepository(prisma_client).table.find_many( + take=page_size, + skip=skip, + order={"token": "asc"}, # mutable-ok: Prisma find_many order= is a dict + ), + "Failed reverse-hash recovery against active keys for %d missing keys: %s", + len(wanted), + ) + + return await _paginate_token_digest_metadata(load_page, wanted, page_size=page_size) async def _reverse_hash_deleted_key_metadata( prisma_client: PrismaClient, wanted: AbstractSet[str], + *, + page_size: int, ) -> Mapping[str, KeyMetadataDict]: - deleted_records: Final = await _db_or_empty( - lambda: DeletedVerificationTokenRepository(prisma_client).table.find_many( - take=_MAX_DOUBLE_HASH_TOKEN_SCAN, - order={"deleted_at": "desc"}, # mutable-ok: Prisma find_many order= is a dict - ), - "Failed reverse-hash recovery against deleted keys for %d missing keys: %s", - len(wanted), - ) - if deleted_records is None: - return _EMPTY_KEY_METADATA - return _token_digest_metadata(deleted_records, wanted) + async def load_page(skip: int) -> Sequence[_TokenAliasRecord] | None: + return await _db_or_empty( + lambda: DeletedVerificationTokenRepository(prisma_client).table.find_many( + take=page_size, + skip=skip, + order=[{"deleted_at": "desc"}, {"id": "asc"}], # mutable-ok: Prisma find_many order= is a dict + ), + "Failed reverse-hash recovery against deleted keys for %d missing keys: %s", + len(wanted), + ) + + return await _paginate_token_digest_metadata(load_page, wanted, page_size=page_size) async def _reverse_hash_key_metadata( prisma_client: PrismaClient, wanted: AbstractSet[str], + *, + page_size: int, ) -> Mapping[str, KeyMetadataDict]: - from_active: Final = await _reverse_hash_active_key_metadata(prisma_client, wanted) + from_active: Final = await _reverse_hash_active_key_metadata(prisma_client, wanted, page_size=page_size) still_wanted: Final = wanted - frozenset(from_active) if not still_wanted: return from_active - from_deleted: Final = await _reverse_hash_deleted_key_metadata(prisma_client, still_wanted) + from_deleted: Final = await _reverse_hash_deleted_key_metadata(prisma_client, still_wanted, page_size=page_size) return MappingProxyType({**from_active, **from_deleted}) @@ -228,21 +268,24 @@ async def attach_user_emails( async def recover_double_hashed_key_metadata( prisma_client: PrismaClient, missing_keys: AbstractSet[str], + *, + token_scan_page_size: int = _TOKEN_SCAN_PAGE, ) -> Mapping[str, KeyMetadataDict]: """ Recover key_alias/team_id/user_email for DailyUserSpend.api_key values that were double-hashed by the v1.99 spend-log provenance gate. Those rows store hash(VerificationToken.token) instead of the token, so the - exact join misses. Prefer a bounded reverse-hash against active/deleted - tokens; fall back to SpendLogs metadata. Emails come from SpendLogs when - present, otherwise from UserTable via the recovered key's user_id. + exact join misses. Page through active then deleted tokens until every + wanted digest is found or the table ends; fall back to SpendLogs metadata. + Emails come from SpendLogs when present, otherwise from UserTable via the + recovered key's user_id. """ sha_missing: Final = frozenset(key for key in missing_keys if is_valid_sha256_hash(key)) if not sha_missing: return _EMPTY_KEY_METADATA - from_tokens: Final = await _reverse_hash_key_metadata(prisma_client, sha_missing) + from_tokens: Final = await _reverse_hash_key_metadata(prisma_client, sha_missing, page_size=token_scan_page_size) still_missing: Final = sha_missing - frozenset(from_tokens) recovered: Final = ( from_tokens diff --git a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py index 6f0e912c5b1..704de049fc8 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py +++ b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py @@ -111,3 +111,76 @@ async def test_recover_falls_back_to_spend_logs_when_token_scan_raises_prisma_er assert result[double_hashed]["key_alias"] == "from-spend-logs" assert result[double_hashed]["team_id"] == "team-sl" assert result[double_hashed]["user_email"] == "carol@example.com" + + +@pytest.mark.asyncio +async def test_recover_double_hashed_key_metadata_scans_past_first_page(): + token = "z" * 64 + double_hashed = hash_token(token) + decoys = ( + SimpleNamespace(token="1" * 64, key_alias="decoy-1", team_id=None, user_id=None), + SimpleNamespace(token="2" * 64, key_alias="decoy-2", team_id=None, user_id=None), + ) + match = SimpleNamespace(token=token, key_alias="late-key", team_id="team-late", user_id="dana") + + async def find_many(*, take: int | None = None, skip: int | None = None, order: object = None): + if skip == 0: + return list(decoys) + if skip == 2: + return [match] + return [] + + mock_prisma = MagicMock() + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(side_effect=find_many) + mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_usertable.find_many = AsyncMock( + return_value=[SimpleNamespace(user_id="dana", user_email="dana@example.com")] + ) + mock_prisma.db.query_raw = AsyncMock(return_value=[]) + + result = await recover_double_hashed_key_metadata(mock_prisma, {double_hashed}, token_scan_page_size=2) + + assert result[double_hashed]["key_alias"] == "late-key" + assert result[double_hashed]["team_id"] == "team-late" + assert result[double_hashed]["user_email"] == "dana@example.com" + assert [call.kwargs["skip"] for call in mock_prisma.db.litellm_verificationtoken.find_many.call_args_list] == [0, 2] + mock_prisma.db.litellm_deletedverificationtoken.find_many.assert_not_called() + mock_prisma.db.query_raw.assert_not_called() + + +@pytest.mark.asyncio +async def test_recover_double_hashed_key_metadata_pages_deleted_tokens(): + token = "y" * 64 + double_hashed = hash_token(token) + decoys = ( + SimpleNamespace(token="3" * 64, key_alias="deleted-decoy-1", team_id=None, user_id=None), + SimpleNamespace(token="4" * 64, key_alias="deleted-decoy-2", team_id=None, user_id=None), + ) + match = SimpleNamespace(token=token, key_alias="deleted-late-key", team_id="team-del", user_id="erin") + + async def find_deleted(*, take: int | None = None, skip: int | None = None, order: object = None): + if skip == 0: + return list(decoys) + if skip == 2: + return [match] + return [] + + mock_prisma = MagicMock() + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(side_effect=find_deleted) + mock_prisma.db.litellm_usertable.find_many = AsyncMock( + return_value=[SimpleNamespace(user_id="erin", user_email="erin@example.com")] + ) + mock_prisma.db.query_raw = AsyncMock(return_value=[]) + + result = await recover_double_hashed_key_metadata(mock_prisma, {double_hashed}, token_scan_page_size=2) + + assert result[double_hashed]["key_alias"] == "deleted-late-key" + assert result[double_hashed]["user_email"] == "erin@example.com" + assert [ + call.kwargs["skip"] for call in mock_prisma.db.litellm_deletedverificationtoken.find_many.call_args_list + ] == [ + 0, + 2, + ] + mock_prisma.db.query_raw.assert_not_called() From cf7abf81367ccf800c561e855ddcb5bad3730a86 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 10:25:19 -0700 Subject: [PATCH 27/88] fix(proxy): log mid-stream /v1/messages failures as failures with partial usage A provider read timeout after the 200 was already committed on a streamed /v1/messages request used to run the success logging path, so the failure callbacks never fired and the failure metrics stayed flat. The pass-through stream handler and the Bedrock relay iterator now dispatch the failure handlers instead, with the usage and cost of the chunks already delivered stashed on the logging object so the failure row still bills them. --- litellm/litellm_core_utils/litellm_logging.py | 5 + .../messages/streaming_iterator.py | 49 ++-- .../anthropic_passthrough_logging_handler.py | 217 +++++++++++------- .../streaming_handler.py | 39 +++- .../messages/test_streaming_iterator.py | 77 +++++-- ...t_anthropic_passthrough_logging_handler.py | 75 ++++++ .../test_streaming_handler_interrupt.py | 109 +++++++++ 7 files changed, 431 insertions(+), 140 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index f54eeca5178..e3941d6fbe1 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1888,6 +1888,11 @@ class Logging(LiteLLMLoggingBaseClass): **kwargs, ) + def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None: + """Stash what an interrupted stream already consumed so the failure log bills it instead of zero.""" + self.model_call_details["combined_usage_object"] = usage + self.model_call_details["response_cost"] = response_cost + async def dispatch_failure_handlers( self, exception: Exception, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py index 66e36dab2ba..34286e2171f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py @@ -1,6 +1,6 @@ import asyncio import json -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Mapping, Sequence from datetime import datetime from typing import Any, Final, Protocol, runtime_checkable @@ -176,17 +176,6 @@ def _try_claim_detached_drain_slot() -> bool: return True -def _exception_left_unconsumed(queue: "asyncio.Queue[bytes | None | BaseException]", exc: BaseException) -> bool: - """After client detach the relay never reads the queue again, so drain it here. - - The forwarded exception still sitting in the queue means the relay tore - down before re-raising it, so the proxy's failure handling never ran and - the caller must salvage spend itself. - """ - remaining: Final = tuple(queue.get_nowait() for _ in range(queue.qsize())) - return any(item is exc for item in remaining) - - def _sse_event(event_type: str, payload: Mapping[str, object]) -> bytes: return f"event: {event_type}\ndata: {json.dumps(payload)}\n\n".encode() @@ -671,27 +660,25 @@ class BaseAnthropicMessagesStreamingIterator: self, queue: "asyncio.Queue[bytes | None | BaseException]", client_detached: "asyncio.Event", - collected_chunks: list[bytes], # mutable-ok: SSE buffer forwarded to list-typed _bill_collected_chunks - exc: BaseException, + collected_chunks: Sequence[bytes], + exc: Exception, ) -> None: - """Forward a provider error to a still-connected client, else salvage partial spend. + """Forward a provider error to a still-connected client and log the request as failed. - Handing the original exception to the client-facing generator lets it - re-raise so the proxy's failure handling keeps the provider status and - owns logging (no success-bill). If the client already went away, or - disconnects before ever consuming the queued exception, no failure hook - runs, so bill the partial instead of dropping the request. + The relay re-raises the forwarded exception so the proxy's failure hook + keeps the provider status; the logging object's failure handlers fire + here either way, carrying the partial usage the provider already + billed, so a client that left before consuming the exception still + gets a failure row rather than a success one. """ - from litellm._logging import verbose_proxy_logger + from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler - if not client_detached.is_set() and await self._enqueue_for_client(queue, client_detached, exc): - await client_detached.wait() - if not _exception_left_unconsumed(queue, exc): - return - verbose_proxy_logger.warning( - "async_sse_wrapper upstream pump failed after client disconnect (%d chunks): %s(%s)", - len(collected_chunks), - type(exc).__name__, - exc, + if not client_detached.is_set(): + await self._enqueue_for_client(queue, client_detached, exc) + PassThroughStreamingHandler.schedule_stream_failure_logging( + litellm_logging_obj=self.litellm_logging_obj, + endpoint_type=EndpointType.ANTHROPIC, + request_body=self.request_body or {}, + raw_bytes=collected_chunks, + exception=exc, ) - await self._bill_collected_chunks(collected_chunks, stream_teardown=True) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index a36a365f39a..4ce840ce6f9 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -37,6 +37,7 @@ from litellm.types.utils import ( Message, ModelResponse, TextCompletionResponse, + Usage, ) if TYPE_CHECKING: @@ -147,6 +148,134 @@ class AnthropicPassthroughLoggingHandler: return model_group.removeprefix("passthrough/") return model + @staticmethod + def _resolve_logged_model( + litellm_logging_obj: LiteLLMLoggingObj, + request_body: Mapping[str, object], + all_chunks: Sequence[str | bytes], + ) -> str: + request_model: Final = request_body.get("model") + logged_model: Final = ( + request_model + if isinstance(request_model, str) and request_model + else str(litellm_logging_obj.model_call_details.get("model") or "") + ) + if logged_model and logged_model != "unknown": + return logged_model + return AnthropicPassthroughLoggingHandler._extract_model_from_anthropic_chunks(all_chunks) or logged_model + + @staticmethod + def _usage_only_response_or_none( + all_chunks: Sequence[str | bytes], model: str, speed: str | None + ) -> ModelResponse | None: + try: + return AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks( + all_chunks=all_chunks, model=model, speed=speed + ) + except Exception as e: + verbose_proxy_logger.warning("Anthropic passthrough: usage-only fallback failed (model=%s): %s", model, e) + return None + + @staticmethod + def _assemble_streaming_response( + all_chunks: Sequence[str | bytes], + litellm_logging_obj: LiteLLMLoggingObj, + model: str, + speed: str | None, + ) -> ModelResponse | TextCompletionResponse | None: + try: + assembled: Final = AnthropicPassthroughLoggingHandler._build_complete_streaming_response( + all_chunks=all_chunks, + litellm_logging_obj=litellm_logging_obj, + model=model, + speed=speed, + ) + except Exception as e: + verbose_proxy_logger.warning( + "Anthropic passthrough: stream assembly raised (model=%s): %s; falling " + "back to usage-only cost from raw SSE events.", + model, + e, + ) + return AnthropicPassthroughLoggingHandler._usage_only_response_or_none(all_chunks, model, speed) + if assembled is not None: + return assembled + return AnthropicPassthroughLoggingHandler._usage_only_response_or_none(all_chunks, model, speed) + + @staticmethod + def _build_streaming_response_for_logging( + litellm_logging_obj: LiteLLMLoggingObj, + request_body: Mapping[str, object], + all_chunks: Sequence[str | bytes], + model: str, + ) -> ModelResponse | TextCompletionResponse | None: + response: Final = AnthropicPassthroughLoggingHandler._assemble_streaming_response( + all_chunks=all_chunks, + litellm_logging_obj=litellm_logging_obj, + model=model, + speed=AnthropicPassthroughLoggingHandler._cost_relevant_speed(request_body), + ) + if response is None: + return None + AnthropicPassthroughLoggingHandler._recover_interrupted_stream_output_tokens( + response=response, all_chunks=all_chunks, model=model + ) + return response + + @staticmethod + def record_partial_usage_for_failure( + litellm_logging_obj: LiteLLMLoggingObj, + request_body: Mapping[str, object], + all_chunks: Sequence[str | bytes], + ) -> None: + if not all_chunks: + return + model: Final = AnthropicPassthroughLoggingHandler._resolve_logged_model( + litellm_logging_obj, request_body, all_chunks + ) + partial_response: Final = AnthropicPassthroughLoggingHandler._build_streaming_response_for_logging( + litellm_logging_obj=litellm_logging_obj, request_body=request_body, all_chunks=all_chunks, model=model + ) + usage: Final = cast(Usage | None, getattr(partial_response, "usage", None)) + if partial_response is None or usage is None: + return + try: + response_cost: Final = AnthropicPassthroughLoggingHandler._compute_response_cost( + litellm_model_response=partial_response, + model=AnthropicPassthroughLoggingHandler._resolve_costing_model(model, litellm_logging_obj), + logging_obj=litellm_logging_obj, + ) + except Exception as e: + verbose_proxy_logger.warning( + "Anthropic passthrough: could not cost the partial usage of a failed stream (model=%s): %s", model, e + ) + return + litellm_logging_obj.record_partial_usage_for_failure(usage=usage, response_cost=response_cost) + + @staticmethod + def _compute_response_cost( + litellm_model_response: ModelResponse | TextCompletionResponse, + model: str, + logging_obj: LiteLLMLoggingObj, + ) -> float: + if logging_obj.model_call_details.get("cache_hit") is True: + return 0.0 + custom_llm_provider: Final = logging_obj.model_call_details.get("custom_llm_provider") + model_for_cost: Final = ( + f"{custom_llm_provider}/{model}" + if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/") + else model + ) + return litellm.completion_cost( + completion_response=litellm_model_response, + model=model_for_cost, + custom_llm_provider=custom_llm_provider, + custom_pricing=use_custom_pricing_for_model( + litellm_params=(logging_obj.litellm_params if hasattr(logging_obj, "litellm_params") else None) + ), + router_model_id=logging_obj.get_router_model_id(), + ) + @staticmethod def _extract_model_from_anthropic_chunks( all_chunks: Sequence[str | bytes], @@ -263,31 +392,9 @@ class AnthropicPassthroughLoggingHandler: if logging_obj.model_call_details.get("stream") is True: logging_obj.model_call_details["complete_streaming_response"] = litellm_model_response try: - # Get custom_llm_provider from logging object if available (e.g., azure_ai for Azure Anthropic) - custom_llm_provider: Final = logging_obj.model_call_details.get("custom_llm_provider") - model = AnthropicPassthroughLoggingHandler._resolve_costing_model(model, logging_obj) - - # Prepend custom_llm_provider to model if not already present - model_for_cost = model - if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"): - model_for_cost = f"{custom_llm_provider}/{model}" - - router_model_id: Final = logging_obj.get_router_model_id() - custom_pricing: Final = use_custom_pricing_for_model( - litellm_params=(logging_obj.litellm_params if hasattr(logging_obj, "litellm_params") else None) - ) - - response_cost: Final = ( - 0.0 - if logging_obj.model_call_details.get("cache_hit") is True - else litellm.completion_cost( - completion_response=litellm_model_response, - model=model_for_cost, - custom_llm_provider=custom_llm_provider, - custom_pricing=custom_pricing, - router_model_id=router_model_id, - ) + response_cost: Final = AnthropicPassthroughLoggingHandler._compute_response_cost( + litellm_model_response=litellm_model_response, model=model, logging_obj=logging_obj ) kwargs["response_cost"] = response_cost @@ -342,57 +449,12 @@ class AnthropicPassthroughLoggingHandler: - Logs in litellm callbacks """ - speed: Final = AnthropicPassthroughLoggingHandler._cost_relevant_speed(request_body) - model = request_body.get("model", "") - # Check if it's available in the logging object - if ( - not model - and hasattr(litellm_logging_obj, "model_call_details") - and litellm_logging_obj.model_call_details.get("model") - ): - model = cast(str, litellm_logging_obj.model_call_details.get("model")) - - if not model or model == "unknown": - chunk_model: Final = AnthropicPassthroughLoggingHandler._extract_model_from_anthropic_chunks(all_chunks) - if chunk_model: - model = chunk_model - - try: - complete_streaming_response = AnthropicPassthroughLoggingHandler._build_complete_streaming_response( - all_chunks=all_chunks, - litellm_logging_obj=litellm_logging_obj, - model=model, - speed=speed, - ) - except Exception as e: - # stream_chunk_builder re-raises assembly failures (as litellm.APIError) - # on large agentic tool-use / thinking streams; treat that the same as a - # None result so the usage-only fallback below still recovers cost - verbose_proxy_logger.warning( - "Anthropic passthrough: stream assembly raised (model=%s): %s; falling " - "back to usage-only cost from raw SSE events.", - model, - e, - ) - complete_streaming_response = None - if complete_streaming_response is None: - # stream_chunk_builder cannot always reassemble large agentic streams, but - # Anthropic still emits token usage in the message_start / message_delta SSE - # events regardless of content shape; recover usage-only so cost is tracked. - # Guard it too: a raise here would defeat the point and drop the request - try: - complete_streaming_response = AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks( - all_chunks=all_chunks, - model=model, - speed=speed, - ) - except Exception as e: - verbose_proxy_logger.warning( - "Anthropic passthrough: usage-only fallback failed (model=%s): %s", - model, - e, - ) - complete_streaming_response = None + model: Final = AnthropicPassthroughLoggingHandler._resolve_logged_model( + litellm_logging_obj, request_body, all_chunks + ) + complete_streaming_response: Final = AnthropicPassthroughLoggingHandler._build_streaming_response_for_logging( + litellm_logging_obj=litellm_logging_obj, request_body=request_body, all_chunks=all_chunks, model=model + ) if complete_streaming_response is None: verbose_proxy_logger.error( "Unable to build complete streaming response for Anthropic passthrough endpoint, not logging..." @@ -401,11 +463,6 @@ class AnthropicPassthroughLoggingHandler: "result": None, "kwargs": {}, } - AnthropicPassthroughLoggingHandler._recover_interrupted_stream_output_tokens( - response=complete_streaming_response, - all_chunks=all_chunks, - model=model, - ) kwargs: Final = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( litellm_model_response=complete_streaming_response, model=model, diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index 022a1ecbac4..ba2717ef119 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -1,4 +1,5 @@ -from collections.abc import Coroutine +import traceback +from collections.abc import Coroutine, Mapping, Sequence from datetime import datetime from typing import Final, Protocol @@ -50,6 +51,27 @@ class PassThroughStreamingHandler: if litellm_logging_obj.completion_start_time is None: litellm_logging_obj._update_completion_start_time(completion_start_time=datetime.now()) + @staticmethod + def schedule_stream_failure_logging( + litellm_logging_obj: LiteLLMLoggingObj, + endpoint_type: EndpointType, + request_body: Mapping[str, object], + raw_bytes: Sequence[bytes], + exception: Exception, + ) -> None: + if endpoint_type == EndpointType.ANTHROPIC: + AnthropicPassthroughLoggingHandler.record_partial_usage_for_failure( + litellm_logging_obj=litellm_logging_obj, request_body=request_body, all_chunks=raw_bytes + ) + try: + GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( + async_coroutine=litellm_logging_obj.dispatch_failure_handlers( + exception, traceback.format_exc(), prefer_async_handlers=True + ) + ) + except Exception as e: + verbose_proxy_logger.error("Error scheduling stream failure logging: %s", e) + @staticmethod async def chunk_processor( response: httpx.Response, @@ -132,9 +154,9 @@ class PassThroughStreamingHandler: # coroutine on logging_obj instead of enqueueing now, so # ProxyLogging._fire_deferred_stream_logging fires it after # guardrail end-of-stream blocks populate guardrail_information. - # Disconnect/exception paths skip this and fall through to the - # immediate enqueue in ``finally`` to keep partial billing - # (LIT-2642). + # Disconnect paths skip this and fall through to the immediate + # enqueue in ``finally`` to keep partial billing (LIT-2642); + # upstream exceptions log a failure instead (LIT-3798). if ( getattr(litellm_logging_obj, "_on_deferred_stream_complete", None) is not None and raw_bytes @@ -144,6 +166,15 @@ class PassThroughStreamingHandler: litellm_logging_obj._deferred_stream_complete_args = (_build_logging_coroutine(),) except Exception as e: verbose_proxy_logger.error("Error in chunk_processor: %s", e) + if response.status_code < 400: + logging_scheduled = True + PassThroughStreamingHandler.schedule_stream_failure_logging( + litellm_logging_obj=litellm_logging_obj, + endpoint_type=endpoint_type, + request_body=request_body or {}, + raw_bytes=raw_bytes, + exception=e, + ) raise finally: # GeneratorExit (raised on client disconnect) is not caught by diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py index 11a048edc1f..3d41d0942e5 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py @@ -5,6 +5,7 @@ from datetime import datetime import pytest +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.experimental_pass_through.messages import streaming_iterator as streaming_iterator_module from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import ( @@ -32,7 +33,16 @@ class _RecordingLoggingIterator(BaseAnthropicMessagesStreamingIterator): self.logging_call_count += 1 -def _make_logging_obj(test_name: str) -> LiteLLMLoggingObj: +class _FailureRecorder(CustomLogger): + def __init__(self): + super().__init__() + self.failure_kwargs: list = [] + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.failure_kwargs.append(kwargs) + + +def _make_logging_obj(test_name: str, failure_recorder: _FailureRecorder | None = None) -> LiteLLMLoggingObj: return LiteLLMLoggingObj( model="bedrock/invoke/anthropic.claude-3-sonnet-20240229-v1:0", messages=[{"role": "user", "content": "hi"}], @@ -41,9 +51,19 @@ def _make_logging_obj(test_name: str) -> LiteLLMLoggingObj: start_time=datetime.now(), litellm_call_id=test_name, function_id=test_name, + dynamic_async_failure_callbacks=[failure_recorder] if failure_recorder is not None else None, ) +async def _wait_for_failure_event(recorder: _FailureRecorder) -> dict: + for _ in range(300): + if recorder.failure_kwargs: + break + await asyncio.sleep(0.01) + assert len(recorder.failure_kwargs) == 1, "expected exactly one failure event" + return recorder.failure_kwargs[0] + + def _make_iterator(test_name: str) -> BaseAnthropicMessagesStreamingIterator: return BaseAnthropicMessagesStreamingIterator( litellm_logging_obj=_make_logging_obj(test_name), @@ -539,7 +559,8 @@ async def test_async_sse_wrapper_reraises_upstream_error_to_connected_client(): before message_stop must propagate the ORIGINAL provider exception to a still-connected client, so the proxy's failure handling keeps the provider-specific status. The pump must not swallow it into a generic - api_error event + normal termination. + api_error event + normal termination, and the request is logged as a + failure carrying the partial usage, never as a success. """ async def _failing_stream(): @@ -547,8 +568,9 @@ async def test_async_sse_wrapper_reraises_upstream_error_to_connected_client(): yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "partial"}} raise _ProviderStreamError("bedrock stream blew up", status_code=529) + recorder = _FailureRecorder() iterator = _RecordingLoggingIterator( - litellm_logging_obj=_make_logging_obj("test_reraises_upstream_error"), + litellm_logging_obj=_make_logging_obj("test_reraises_upstream_error", recorder), request_body={}, ) @@ -561,18 +583,23 @@ async def test_async_sse_wrapper_reraises_upstream_error_to_connected_client(): with pytest.raises(_ProviderStreamError) as excinfo: await _drain() + failure_kwargs = await _wait_for_failure_event(recorder) + assert excinfo.value.status_code == 529 assert received assert not any(c.startswith(b"event: error\n") for c in received) assert iterator.logged_chunks == [] + assert failure_kwargs["standard_logging_object"]["status"] == "failure" + assert failure_kwargs["standard_logging_object"]["prompt_tokens"] == 52 @pytest.mark.asyncio -async def test_async_sse_wrapper_salvages_partial_spend_on_upstream_error_after_disconnect(): +async def test_async_sse_wrapper_logs_failure_on_upstream_error_after_disconnect(): """ When the upstream errors AFTER the client has already disconnected there is - no live client to re-raise to and no failure hook will run, so the pump - salvages partial spend from what it collected instead of dropping the row. + no live client to re-raise to and no proxy failure hook will run, so the + pump logs the failure itself with the partial usage it collected; it must + never bill the broken stream as a success. """ tail_gated = asyncio.Event() @@ -582,8 +609,9 @@ async def test_async_sse_wrapper_salvages_partial_spend_on_upstream_error_after_ await tail_gated.wait() raise _ProviderStreamError("late failure", status_code=500) + recorder = _FailureRecorder() iterator = _RecordingLoggingIterator( - litellm_logging_obj=_make_logging_obj("test_salvage_partial_on_late_error"), + litellm_logging_obj=_make_logging_obj("test_failure_logged_on_late_error", recorder), request_body={}, ) @@ -592,24 +620,23 @@ async def test_async_sse_wrapper_salvages_partial_spend_on_upstream_error_after_ await gen.aclose() # client disconnects before the upstream error tail_gated.set() # let the upstream raise now, after disconnect - for _ in range(100): - if iterator.logged_chunks: - break - await asyncio.sleep(0.01) + failure_kwargs = await _wait_for_failure_event(recorder) assert len(received) == 2 - assert iterator.logged_chunks == received + assert iterator.logging_call_count == 0 + assert failure_kwargs["standard_logging_object"]["status"] == "failure" + assert failure_kwargs["standard_logging_object"]["prompt_tokens"] == 52 + assert isinstance(failure_kwargs["exception"], _ProviderStreamError) @pytest.mark.asyncio -async def test_async_sse_wrapper_salvages_spend_when_queued_error_is_never_consumed(): +async def test_async_sse_wrapper_logs_failure_when_queued_error_is_never_consumed(): """ When the upstream errors while the client is still connected, the pump - forwards the exception through the queue expecting the relay to re-raise it - into the proxy's failure handling. If the client disconnects before - consuming that queued exception, the handoff never happens and no failure - hook runs, so the pump must notice the unconsumed exception at teardown and - salvage partial spend instead of dropping the row entirely. + forwards the exception through the queue for the relay to re-raise. If the + client disconnects before consuming that queued exception, no proxy failure + hook runs, so the failure logged by the pump itself is the only record of + the request; it must be a failure row, not a salvaged success. """ upstream_errored = asyncio.Event() @@ -619,8 +646,9 @@ async def test_async_sse_wrapper_salvages_spend_when_queued_error_is_never_consu upstream_errored.set() raise _ProviderStreamError("mid-stream failure", status_code=500) + recorder = _FailureRecorder() iterator = _RecordingLoggingIterator( - litellm_logging_obj=_make_logging_obj("test_salvage_on_unconsumed_queued_error"), + litellm_logging_obj=_make_logging_obj("test_failure_logged_on_unconsumed_queued_error", recorder), request_body={}, ) @@ -629,13 +657,12 @@ async def test_async_sse_wrapper_salvages_spend_when_queued_error_is_never_consu await upstream_errored.wait() # exception is now queued behind the consumed chunks await gen.aclose() # client disconnects without ever consuming the queued exception - for _ in range(100): - if iterator.logged_chunks: - break - await asyncio.sleep(0.01) + failure_kwargs = await _wait_for_failure_event(recorder) - assert iterator.logging_call_count == 1 - assert iterator.logged_chunks == received + assert len(received) == 2 + assert iterator.logging_call_count == 0 + assert failure_kwargs["standard_logging_object"]["status"] == "failure" + assert failure_kwargs["standard_logging_object"]["prompt_tokens"] == 52 @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index 19bca05fb84..480da1d040e 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -2441,3 +2441,78 @@ class TestAnthropicPassthroughFastMode: assert served_standard.usage.speed == "standard" assert self._cost(served_standard) == pytest.approx(self._cost(standard)) + + +class TestRecordPartialUsageForFailure: + """A stream that dies mid-way still carries the usage the provider billed in + message_start; the failure row must keep it and its cost instead of logging + a zero-cost failure (or, worse, a success).""" + + @staticmethod + def _sse(event, data): + return f"event: {event}\ndata: {json.dumps(data)}\n\n".encode() + + @staticmethod + def _make_logging_obj() -> LiteLLMLoggingObj: + return LiteLLMLoggingObj( + model="claude-sonnet-5", + messages=[{"role": "user", "content": "hello"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="test-partial-usage-failure", + function_id="test-partial-usage-failure", + ) + + def _interrupted_chunks(self): + return [ + self._sse( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_abc", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 52, "output_tokens": 1}, + }, + }, + ), + self._sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + self._sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "partial"}}, + ), + ] + + def test_stashes_partial_usage_and_cost_from_interrupted_stream(self): + logging_obj = self._make_logging_obj() + + AnthropicPassthroughLoggingHandler.record_partial_usage_for_failure( + litellm_logging_obj=logging_obj, + request_body={"model": "claude-sonnet-5", "stream": True}, + all_chunks=self._interrupted_chunks(), + ) + + usage = logging_obj.model_call_details["combined_usage_object"] + assert usage.prompt_tokens == 52 + assert logging_obj.model_call_details["response_cost"] > 0 + + def test_leaves_logging_obj_untouched_when_nothing_streamed(self): + logging_obj = self._make_logging_obj() + + AnthropicPassthroughLoggingHandler.record_partial_usage_for_failure( + litellm_logging_obj=logging_obj, + request_body={"model": "claude-sonnet-5", "stream": True}, + all_chunks=[], + ) + + assert "combined_usage_object" not in logging_obj.model_call_details + assert "response_cost" not in logging_obj.model_call_details diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py b/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py index 56c89fed79a..c4ae0c81d6e 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py @@ -9,6 +9,8 @@ import httpx import pytest import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.proxy.pass_through_endpoints.streaming_handler import ( PassThroughStreamingHandler, @@ -632,3 +634,110 @@ async def test_chunk_processor_enqueues_immediately_on_disconnect_even_when_arme mock_enqueue.assert_called_once() assert logging_obj._deferred_stream_complete_args is None + + +class _EventRecorder(CustomLogger): + def __init__(self): + super().__init__() + self.failure_kwargs = [] + self.success_kwargs = [] + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.failure_kwargs.append(kwargs) + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.success_kwargs.append(kwargs) + + +def _anthropic_sse(event: str, payload: dict) -> bytes: + return f"event: {event}\ndata: {json.dumps(payload)}\n\n".encode() + + +def _anthropic_stream_that_times_out_mid_stream(): + mock = MagicMock(spec=httpx.Response) + mock.status_code = 200 + + async def _aiter_bytes(): + yield _anthropic_sse( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 52, "output_tokens": 1}, + }, + }, + ) + yield _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ) + yield _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "partial"}}, + ) + raise httpx.ReadTimeout("Timeout on reading data from socket") + + mock.aiter_bytes = _aiter_bytes + return mock + + +@pytest.mark.asyncio +async def test_chunk_processor_logs_failure_not_success_on_mid_stream_exception(): + """A stream that dies after the first chunks is a failed request: the failure + callbacks must fire once with the partial usage and cost, and the success + routing must never run for it.""" + recorder = _EventRecorder() + logging_obj = LiteLLMLoggingObj( + model="claude-sonnet-5", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="test-mid-stream-timeout", + function_id="test-mid-stream-timeout", + dynamic_async_success_callbacks=[recorder], + dynamic_async_failure_callbacks=[recorder], + ) + success_routes = [] + + async def _record_success_route(**kwargs): + success_routes.append(kwargs) + + received = [] + + async def _consume_stream(): + async for chunk in PassThroughStreamingHandler.chunk_processor( + response=_anthropic_stream_that_times_out_mid_stream(), + request_body={"model": "claude-sonnet-5", "stream": True}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.ANTHROPIC, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route="/v1/messages", + route_streaming_logging=_record_success_route, + ): + received.append(chunk) + + with pytest.raises(httpx.ReadTimeout): + await _consume_stream() + + for _ in range(300): + if recorder.failure_kwargs: + break + await asyncio.sleep(0.01) + + assert len(received) == 3 + assert success_routes == [] + assert recorder.success_kwargs == [] + assert len(recorder.failure_kwargs) == 1 + failure_payload = recorder.failure_kwargs[0]["standard_logging_object"] + assert failure_payload["status"] == "failure" + assert failure_payload["prompt_tokens"] == 52 + assert failure_payload["response_cost"] > 0 + assert isinstance(recorder.failure_kwargs[0]["exception"], httpx.ReadTimeout) From 501f47ba2fa59ad950c4e122fe9cfddf890ec106 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:31:35 -0700 Subject: [PATCH 28/88] fix(proxy): run the failure hook when a pass-through stream dies mid-body --- .../pass_through_endpoints.py | 84 +++++++++++++++---- .../test_pass_through_endpoints.py | 72 ++++++++++++++++ 2 files changed, 140 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 79d5d0a016f..37c8cfb09d6 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -868,6 +868,38 @@ async def _log_passthrough_upstream_failure( ) +async def _relay_reporting_failures( + stream: AsyncGenerator[bytes, None], + upstream_status: int, + user_api_key_dict: UserAPIKeyAuth, + request_payload: dict, # mutable-ok: post_call_failure_hook lifts fields onto request_data in place +) -> AsyncGenerator[bytes, None]: + """An upstream that dies mid-stream leaves the client a truncated body and the proxy no record, so run + ``post_call_failure_hook`` (spend row, alerting, failure metric) the way the unified endpoints' generators do. + Error statuses were already reported by ``_log_passthrough_upstream_failure`` and relay untouched.""" + from litellm.proxy.proxy_server import proxy_logging_obj + + try: + async for chunk in stream: + yield chunk + except Exception as e: + if upstream_status >= 400: + raise + try: + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=e, + request_data=request_payload, + traceback_str=traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG), + ) + except Exception: # noqa: BLE001 - a failing logging callback must never mask the upstream error + verbose_proxy_logger.warning( + "pass_through_endpoint: post_call_failure_hook raised for a mid-stream upstream error", + exc_info=True, + ) + raise + + from litellm.passthrough.timeout_utils import ( DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS, # noqa: F401 - re-exported for backward compat resolve_llm_passthrough_timeout, # noqa: F401 - re-exported for backward compat @@ -1291,14 +1323,24 @@ async def pass_through_request( return StreamingResponse( wrap_passthrough_sse_bytes_with_keepalive_pings( stream=_own_streamed_managed_ids( - stream=PassThroughStreamingHandler.chunk_processor( - response=response, - request_body=_parsed_body, - litellm_logging_obj=logging_obj, - endpoint_type=endpoint_type, - start_time=start_time, - passthrough_success_handler_obj=pass_through_endpoint_logging, - url_route=str(url), + stream=_relay_reporting_failures( + stream=PassThroughStreamingHandler.chunk_processor( + response=response, + request_body=_parsed_body, + litellm_logging_obj=logging_obj, + endpoint_type=endpoint_type, + start_time=start_time, + passthrough_success_handler_obj=pass_through_endpoint_logging, + url_route=str(url), + ), + upstream_status=response.status_code, + user_api_key_dict=user_api_key_dict, + request_payload=_build_passthrough_failure_request_payload( + parsed_body=_parsed_body, + kwargs=kwargs, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + ), ), managed_id_provider=_managed_id_provider, request=request, @@ -1372,14 +1414,24 @@ async def pass_through_request( return StreamingResponse( wrap_passthrough_sse_bytes_with_keepalive_pings( stream=_own_streamed_managed_ids( - stream=PassThroughStreamingHandler.chunk_processor( - response=response, - request_body=_parsed_body, - litellm_logging_obj=logging_obj, - endpoint_type=endpoint_type, - start_time=start_time, - passthrough_success_handler_obj=pass_through_endpoint_logging, - url_route=str(url), + stream=_relay_reporting_failures( + stream=PassThroughStreamingHandler.chunk_processor( + response=response, + request_body=_parsed_body, + litellm_logging_obj=logging_obj, + endpoint_type=endpoint_type, + start_time=start_time, + passthrough_success_handler_obj=pass_through_endpoint_logging, + url_route=str(url), + ), + upstream_status=response.status_code, + user_api_key_dict=user_api_key_dict, + request_payload=_build_passthrough_failure_request_payload( + parsed_body=_parsed_body, + kwargs=kwargs, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + ), ), managed_id_provider=_managed_id_provider, request=request, diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index d3f17c73499..442adbca08c 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -3988,6 +3988,78 @@ async def test_pass_through_request_streaming_upstream_error_returned_unchanged( assert failure_call_kwargs["original_exception"].status_code == 403 +class _UpstreamDroppingMidStream(httpx.AsyncByteStream): + async def __aiter__(self): + yield b'data: {"id": "chatcmpl-1", "choices": [{"delta": {"content": "hi"}}]}\n\n' + raise httpx.ReadError("upstream dropped the connection mid-stream") + + +async def _relay_everything(body_iterator) -> list: + return [chunk async for chunk in body_iterator] + + +@pytest.mark.asyncio +async def test_pass_through_request_mid_stream_upstream_drop_fires_failure_hook(): + """ + Regression: a 200 stream whose upstream dies mid-body used to end with no + proxy-level failure hook at all, so the request left no spend row, no + failure metric, and no alert; the pre-stream 4xx/5xx path already fires it. + """ + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.llms.custom_http import httpxSpecialProvider + + def transport_handler(upstream_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, stream=_UpstreamDroppingMidStream(), headers={"content-type": "text/event-stream"}) + + real_handler = get_async_httpx_client( + llm_provider=httpxSpecialProvider.PassThroughEndpoint, + params={"timeout": resolve_pass_through_request_timeout(None)}, + ) + cache_dict = litellm.in_memory_llm_clients_cache.cache_dict + cache_key = next(key for key, cached in cache_dict.items() if cached is real_handler) + cache_dict[cache_key] = SimpleNamespace(client=httpx.AsyncClient(transport=httpx.MockTransport(transport_handler))) + + mock_proxy_logging = MagicMock() + mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=lambda user_api_key_dict, data, call_type: data) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value=None) + mock_proxy_logging.get_proxy_hook = MagicMock(return_value=None) + + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.scope = {"path": "/relay-chat"} + mock_request.url = MagicMock() + mock_request.url.path = "/relay-chat" + mock_request.body = AsyncMock(return_value=b'{"model": "gpt-5.6", "stream": true}') + mock_request.headers = Headers({"content-type": "application/json"}) + mock_request.query_params = QueryParams({}) + + try: + with patch( # test-quality-ok: proxy_logging_obj is a proxy_server module global read inside pass_through_request; there is no injection seam + "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging + ): + response = await pass_through_request( + request=mock_request, + target="http://target-api.com/v1/chat/completions", + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + stream=True, + ) + with pytest.raises(httpx.ReadError): + await _relay_everything(response.body_iterator) + await asyncio.sleep(0) + finally: + cache_dict[cache_key] = real_handler + + mock_proxy_logging.post_call_failure_hook.assert_awaited_once() + failure_call_kwargs = mock_proxy_logging.post_call_failure_hook.call_args.kwargs + assert isinstance(failure_call_kwargs["original_exception"], httpx.ReadError) + request_data = failure_call_kwargs["request_data"] + assert request_data["litellm_call_id"] + assert request_data["model"] == "gpt-5.6" + assert isinstance(request_data["litellm_logging_obj"], LiteLLMLoggingObj) + + @pytest.mark.asyncio async def test_pass_through_request_non_streaming_success_unchanged(): """Success (2xx) passthrough behavior must remain unchanged by the error fix.""" From f3021937c62a24e743adbef004a1e45942951c77 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:29:49 -0700 Subject: [PATCH 29/88] refactor(proxy): drop the docstring restating the pass-through failure relay --- litellm/proxy/pass_through_endpoints/pass_through_endpoints.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 37c8cfb09d6..51a579b27cd 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -874,9 +874,6 @@ async def _relay_reporting_failures( user_api_key_dict: UserAPIKeyAuth, request_payload: dict, # mutable-ok: post_call_failure_hook lifts fields onto request_data in place ) -> AsyncGenerator[bytes, None]: - """An upstream that dies mid-stream leaves the client a truncated body and the proxy no record, so run - ``post_call_failure_hook`` (spend row, alerting, failure metric) the way the unified endpoints' generators do. - Error statuses were already reported by ``_log_passthrough_upstream_failure`` and relay untouched.""" from litellm.proxy.proxy_server import proxy_logging_obj try: From 32a3a653124cd0b71982248732bd284e4272ebd0 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 3 Sep 2026 19:35:30 +0000 Subject: [PATCH 30/88] fix(model_prices): add Lyria 3.5, Perplexity Agent API and OpenRouter first-party models, fix Nebius, Mistral, OpenRouter metadata Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 1051 ++++++++++++++++- model_prices_and_context_window.json | 1051 ++++++++++++++++- 2 files changed, 2002 insertions(+), 100 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 50e38072e80..724929a48f5 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -26299,7 +26299,8 @@ "supports_response_schema": false, "supports_system_messages": false, "supports_vision": false, - "supports_web_search": false + "supports_web_search": false, + "output_cost_per_image": 0.08 }, "gemini/veo-2.0-generate-001": { "deprecation_date": "2026-06-30", @@ -33887,7 +33888,8 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 2e-08 }, "mistral/ministral-14b-latest": { "input_cost_per_token": 2e-07, @@ -33902,7 +33904,8 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 2e-08 }, "mistral/ministral-3b-2512": { "input_cost_per_token": 1e-07, @@ -33917,7 +33920,8 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1e-08 }, "mistral/ministral-3b-latest": { "input_cost_per_token": 1e-07, @@ -33932,7 +33936,8 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1e-08 }, "mistral/mistral-embed-2312": { "input_cost_per_token": 1e-07, @@ -35395,9 +35400,9 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/google%2Fgemma-3-27b-it" }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, "input_cost_per_token": 1.3e-07, "output_cost_per_token": 4e-07, "litellm_provider": "nebius", @@ -35715,9 +35720,9 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/moonshotai%2FKimi-K2.7-Code" }, "nebius/moonshotai/Kimi-K3": { - "max_tokens": 1048576, - "max_input_tokens": 1048576, - "max_output_tokens": 1048576, + "max_tokens": 1024000, + "max_input_tokens": 1024000, + "max_output_tokens": 1024000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, "litellm_provider": "nebius", @@ -37585,8 +37590,8 @@ "input_cost_per_token": 5e-06, "litellm_provider": "openrouter", "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": true, @@ -37597,7 +37602,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://openrouter.ai/anthropic/claude-opus-4.5" }, "openrouter/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, @@ -37632,8 +37638,8 @@ "cache_read_input_token_cost_above_200k_tokens": 6e-07, "litellm_provider": "openrouter", "max_input_tokens": 1000000, - "max_output_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -37643,7 +37649,8 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://openrouter.ai/anthropic/claude-sonnet-4.5" }, "openrouter/anthropic/claude-haiku-4.5": { "cache_creation_input_token_cost": 1.25e-06, @@ -37651,8 +37658,8 @@ "input_cost_per_token": 1e-06, "litellm_provider": "openrouter", "max_input_tokens": 200000, - "max_output_tokens": 200000, - "max_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -37662,7 +37669,8 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://openrouter.ai/anthropic/claude-haiku-4.5" }, "openrouter/anthropic/claude-opus-4.7": { "supports_adaptive_thinking": true, @@ -37875,8 +37883,8 @@ "input_cost_per_token": 3e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 65535, + "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 2.5e-06, "supports_audio_output": true, @@ -37885,15 +37893,18 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_image_size": false + "supports_image_size": false, + "cache_read_input_token_cost": 3e-08, + "supports_prompt_caching": true, + "source": "https://openrouter.ai/google/gemini-2.5-flash" }, "openrouter/google/gemini-2.5-pro": { "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "supports_audio_output": true, @@ -37901,7 +37912,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1.25e-07, + "supports_prompt_caching": true, + "source": "https://openrouter.ai/google/gemini-2.5-pro" }, "openrouter/google/gemini-3-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -38292,14 +38306,15 @@ "supports_tool_choice": true }, "openrouter/openai/gpt-3.5-turbo": { - "input_cost_per_token": 1.5e-06, + "input_cost_per_token": 5e-07, "litellm_provider": "openrouter", "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 2e-06, + "output_cost_per_token": 1.5e-06, "supports_tool_choice": true, "max_input_tokens": 16385, - "max_output_tokens": 4096 + "max_output_tokens": 4096, + "source": "https://openrouter.ai/openai/gpt-3.5-turbo" }, "openrouter/openai/gpt-3.5-turbo-16k": { "input_cost_per_token": 3e-06, @@ -38376,14 +38391,17 @@ "input_cost_per_token": 2.5e-06, "litellm_provider": "openrouter", "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1.25e-06, + "supports_prompt_caching": true, + "source": "https://openrouter.ai/openai/gpt-4o" }, "openrouter/openai/gpt-4o-2024-05-13": { "input_cost_per_token": 5e-06, @@ -38631,30 +38649,36 @@ "openrouter/openai/o3-mini": { "input_cost_per_token": 1.1e-06, "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "cache_read_input_token_cost": 5.5e-07, + "supports_prompt_caching": true, + "source": "https://openrouter.ai/openai/o3-mini" }, "openrouter/openai/o3-mini-high": { "input_cost_per_token": 1.1e-06, "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "cache_read_input_token_cost": 5.5e-07, + "supports_prompt_caching": true, + "source": "https://openrouter.ai/openai/o3-mini-high" }, "openrouter/qwen/qwen-2.5-coder-32b-instruct": { "input_cost_per_token": 1.8e-07, @@ -39672,21 +39696,33 @@ "mode": "responses", "supports_web_search": true, "supports_reasoning": true, - "supports_function_calling": true + "supports_function_calling": true, + "input_cost_per_token": 1.75e-06, + "output_cost_per_token": 1.4e-05, + "cache_read_input_token_cost": 1.75e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/openai/gpt-5.1": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/openai/gpt-5-mini": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 2.5e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/anthropic/claude-opus-4-6": { "supports_adaptive_thinking": true, @@ -39696,7 +39732,11 @@ "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, - "supports_output_config": true + "supports_output_config": true, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "cache_read_input_token_cost": 5e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/anthropic/claude-opus-4-7": { "supports_adaptive_thinking": true, @@ -39705,7 +39745,11 @@ "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, - "supports_output_config": true + "supports_output_config": true, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "cache_read_input_token_cost": 5e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/anthropic/claude-opus-4-5": { "litellm_provider": "perplexity", @@ -39713,21 +39757,33 @@ "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, - "supports_output_config": true + "supports_output_config": true, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "cache_read_input_token_cost": 5e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/anthropic/claude-sonnet-4-5": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/anthropic/claude-haiku-4-5": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 5e-06, + "cache_read_input_token_cost": 1e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/google/gemini-3-pro-preview": { "litellm_provider": "perplexity", @@ -39741,7 +39797,11 @@ "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 5e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/google/gemini-2.5-pro": { "litellm_provider": "perplexity", @@ -39770,7 +39830,11 @@ "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 6.25e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/perplexity/deepseek-v4-flash-0731": { "cache_read_input_token_cost": 2.8e-08, @@ -59010,5 +59074,892 @@ "supported_endpoints": [ "/v1/audio/transcriptions" ] + }, + "gemini/lyria-3.5-clip-preview": { + "input_cost_per_token": 0, + "litellm_provider": "gemini", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_image": 0.04, + "output_cost_per_token": 0, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_prompt_caching": false, + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_web_search": false + }, + "gemini/lyria-3.5-pro-preview": { + "input_cost_per_token": 0, + "litellm_provider": "gemini", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_prompt_caching": false, + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_web_search": false, + "output_cost_per_image": 0.08 + }, + "perplexity/anthropic/claude-fable-5": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "cache_read_input_token_cost": 1e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/anthropic/claude-opus-5": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "cache_read_input_token_cost": 5e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/anthropic/claude-opus-4-8": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "cache_read_input_token_cost": 5e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/anthropic/claude-sonnet-5": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 2e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/anthropic/claude-sonnet-4-6": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/openai/gpt-5.6-sol": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 3e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token_above_272k_tokens": 1e-05, + "output_cost_per_token_above_272k_tokens": 4.5e-05, + "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/openai/gpt-5.6-terra": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token_above_272k_tokens": 4e-06, + "output_cost_per_token_above_272k_tokens": 1.8e-05, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/openai/gpt-5.6-luna": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token_above_272k_tokens": 4e-07, + "output_cost_per_token_above_272k_tokens": 1.8e-06, + "cache_read_input_token_cost_above_272k_tokens": 4e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/openai/gpt-5.5": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 3e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token_above_272k_tokens": 1e-05, + "output_cost_per_token_above_272k_tokens": 4.5e-05, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/openai/gpt-5.4": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token_above_272k_tokens": 5e-06, + "output_cost_per_token_above_272k_tokens": 2.25e-05, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/openai/gpt-5.4-mini": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 4.5e-06, + "cache_read_input_token_cost": 7.5e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/openai/gpt-5.4-nano": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.25e-06, + "cache_read_input_token_cost": 2e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/openai/gpt-5": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/google/gemini-3.1-pro-preview": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token_above_200k_tokens": 4e-06, + "output_cost_per_token_above_200k_tokens": 1.8e-05, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/google/gemini-3.1-flash-lite": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.5e-06, + "cache_read_input_token_cost": 2.5e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/google/gemini-3.5-flash": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 9e-06, + "cache_read_input_token_cost": 1.5e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/google/gemini-3.5-flash-lite": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 3e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/google/gemini-3.6-flash": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 7.5e-06, + "cache_read_input_token_cost": 1.5e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/google/gemini-3.7-flash": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 3.75e-06, + "cache_read_input_token_cost": 7.5e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/xai/grok-4.6": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token_above_200k_tokens": 4e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/xai/grok-4.5": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token_above_200k_tokens": 4e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/xai/grok-4.3": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/xai/grok-4.20-reasoning": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/xai/grok-4.20-non-reasoning": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/xai/grok-4.20-multi-agent": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/perplexity/glm-5.3": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "cache_read_input_token_cost": 2.6e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/perplexity/glm-5.3-flash": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 3e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/perplexity/nemotron-3.5-lightning-30b-a3b": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.15e-08, + "output_cost_per_token": 1.7e-07, + "cache_read_input_token_cost": 1.15e-09, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/perplexity/nemotron-3-ultra-550b-a55b": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2.5e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "openrouter/anthropic/claude-fable-5": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/anthropic/claude-fable-5", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 1e-06, + "supports_prompt_caching": true, + "cache_creation_input_token_cost": 1.25e-05 + }, + "openrouter/anthropic/claude-fable-5.1": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/anthropic/claude-fable-5.1", + "supports_function_calling": true, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2.5e-07, + "supports_prompt_caching": true, + "cache_creation_input_token_cost": 1.25e-05 + }, + "openrouter/anthropic/claude-opus-4.8": { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/anthropic/claude-opus-4.8", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 5e-07, + "supports_prompt_caching": true, + "cache_creation_input_token_cost": 6.25e-06 + }, + "openrouter/anthropic/claude-sonnet-5": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/anthropic/claude-sonnet-5", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true, + "cache_creation_input_token_cost": 2.5e-06 + }, + "openrouter/google/gemini-2.5-flash-lite": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_tokens": 65535, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-2.5-flash-lite", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "cache_read_input_token_cost": 1e-08, + "supports_prompt_caching": true + }, + "openrouter/google/gemini-3.5-flash": { + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 9e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.5-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "cache_read_input_token_cost": 1.5e-07, + "supports_prompt_caching": true + }, + "openrouter/google/gemini-3.5-flash-lite": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2.5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.5-flash-lite", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "cache_read_input_token_cost": 3e-08, + "supports_prompt_caching": true + }, + "openrouter/google/gemini-3.6-flash": { + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 3.75e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.6-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "cache_read_input_token_cost": 7.5e-08, + "supports_prompt_caching": true + }, + "openrouter/google/gemini-3.7-flash": { + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 3.75e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.7-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "cache_read_input_token_cost": 7.5e-08, + "supports_prompt_caching": true + }, + "openrouter/google/gemini-3.8-flash": { + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 3.75e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.8-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "cache_read_input_token_cost": 7.5e-08, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-4o-mini": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-4o-mini", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 7.5e-08, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.1": { + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.1", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 1.25e-07, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.3-codex": { + "input_cost_per_token": 1.75e-06, + "output_cost_per_token": 1.4e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.3-codex", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 1.75e-07, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.4": { + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1.5e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.4", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2.5e-07, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.4-mini": { + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 4.5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.4-mini", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 7.5e-08, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.4-nano": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.25e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.4-nano", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2e-08, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.5": { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 3e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.5", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 5e-07, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.6-luna": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.6-luna", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2e-08, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.6-terra": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.6-terra", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true + }, + "openrouter/openai/o3": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "source": "https://openrouter.ai/openai/o3", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 5e-07, + "supports_prompt_caching": true + }, + "openrouter/openai/o4-mini": { + "input_cost_per_token": 1.1e-06, + "output_cost_per_token": 4.4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "source": "https://openrouter.ai/openai/o4-mini", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2.75e-07, + "supports_prompt_caching": true + }, + "openrouter/x-ai/grok-4.20": { + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 2.5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 2000000, + "max_output_tokens": 1800000, + "max_tokens": 1800000, + "mode": "chat", + "source": "https://openrouter.ai/x-ai/grok-4.20", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true + }, + "openrouter/x-ai/grok-4.20-multi-agent": { + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 2.5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 2000000, + "max_output_tokens": 1800000, + "max_tokens": 1800000, + "mode": "chat", + "source": "https://openrouter.ai/x-ai/grok-4.20-multi-agent", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true + }, + "openrouter/x-ai/grok-4.3": { + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 2.5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 900000, + "max_tokens": 900000, + "mode": "chat", + "source": "https://openrouter.ai/x-ai/grok-4.3", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true + }, + "openrouter/x-ai/grok-4.5": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 500000, + "max_output_tokens": 450000, + "max_tokens": 450000, + "mode": "chat", + "source": "https://openrouter.ai/x-ai/grok-4.5", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 3e-07, + "supports_prompt_caching": true + }, + "openrouter/x-ai/grok-4.6": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 500000, + "max_output_tokens": 450000, + "max_tokens": 450000, + "mode": "chat", + "source": "https://openrouter.ai/x-ai/grok-4.6", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 5e-07, + "supports_prompt_caching": true + }, + "openrouter/x-ai/grok-build-0.1": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 256000, + "max_output_tokens": 230400, + "max_tokens": 230400, + "mode": "chat", + "source": "https://openrouter.ai/x-ai/grok-build-0.1", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 50e38072e80..724929a48f5 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -26299,7 +26299,8 @@ "supports_response_schema": false, "supports_system_messages": false, "supports_vision": false, - "supports_web_search": false + "supports_web_search": false, + "output_cost_per_image": 0.08 }, "gemini/veo-2.0-generate-001": { "deprecation_date": "2026-06-30", @@ -33887,7 +33888,8 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 2e-08 }, "mistral/ministral-14b-latest": { "input_cost_per_token": 2e-07, @@ -33902,7 +33904,8 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 2e-08 }, "mistral/ministral-3b-2512": { "input_cost_per_token": 1e-07, @@ -33917,7 +33920,8 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1e-08 }, "mistral/ministral-3b-latest": { "input_cost_per_token": 1e-07, @@ -33932,7 +33936,8 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1e-08 }, "mistral/mistral-embed-2312": { "input_cost_per_token": 1e-07, @@ -35395,9 +35400,9 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/google%2Fgemma-3-27b-it" }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, "input_cost_per_token": 1.3e-07, "output_cost_per_token": 4e-07, "litellm_provider": "nebius", @@ -35715,9 +35720,9 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/moonshotai%2FKimi-K2.7-Code" }, "nebius/moonshotai/Kimi-K3": { - "max_tokens": 1048576, - "max_input_tokens": 1048576, - "max_output_tokens": 1048576, + "max_tokens": 1024000, + "max_input_tokens": 1024000, + "max_output_tokens": 1024000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, "litellm_provider": "nebius", @@ -37585,8 +37590,8 @@ "input_cost_per_token": 5e-06, "litellm_provider": "openrouter", "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": true, @@ -37597,7 +37602,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://openrouter.ai/anthropic/claude-opus-4.5" }, "openrouter/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, @@ -37632,8 +37638,8 @@ "cache_read_input_token_cost_above_200k_tokens": 6e-07, "litellm_provider": "openrouter", "max_input_tokens": 1000000, - "max_output_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -37643,7 +37649,8 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://openrouter.ai/anthropic/claude-sonnet-4.5" }, "openrouter/anthropic/claude-haiku-4.5": { "cache_creation_input_token_cost": 1.25e-06, @@ -37651,8 +37658,8 @@ "input_cost_per_token": 1e-06, "litellm_provider": "openrouter", "max_input_tokens": 200000, - "max_output_tokens": 200000, - "max_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -37662,7 +37669,8 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://openrouter.ai/anthropic/claude-haiku-4.5" }, "openrouter/anthropic/claude-opus-4.7": { "supports_adaptive_thinking": true, @@ -37875,8 +37883,8 @@ "input_cost_per_token": 3e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 65535, + "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 2.5e-06, "supports_audio_output": true, @@ -37885,15 +37893,18 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_image_size": false + "supports_image_size": false, + "cache_read_input_token_cost": 3e-08, + "supports_prompt_caching": true, + "source": "https://openrouter.ai/google/gemini-2.5-flash" }, "openrouter/google/gemini-2.5-pro": { "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "supports_audio_output": true, @@ -37901,7 +37912,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1.25e-07, + "supports_prompt_caching": true, + "source": "https://openrouter.ai/google/gemini-2.5-pro" }, "openrouter/google/gemini-3-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -38292,14 +38306,15 @@ "supports_tool_choice": true }, "openrouter/openai/gpt-3.5-turbo": { - "input_cost_per_token": 1.5e-06, + "input_cost_per_token": 5e-07, "litellm_provider": "openrouter", "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 2e-06, + "output_cost_per_token": 1.5e-06, "supports_tool_choice": true, "max_input_tokens": 16385, - "max_output_tokens": 4096 + "max_output_tokens": 4096, + "source": "https://openrouter.ai/openai/gpt-3.5-turbo" }, "openrouter/openai/gpt-3.5-turbo-16k": { "input_cost_per_token": 3e-06, @@ -38376,14 +38391,17 @@ "input_cost_per_token": 2.5e-06, "litellm_provider": "openrouter", "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1.25e-06, + "supports_prompt_caching": true, + "source": "https://openrouter.ai/openai/gpt-4o" }, "openrouter/openai/gpt-4o-2024-05-13": { "input_cost_per_token": 5e-06, @@ -38631,30 +38649,36 @@ "openrouter/openai/o3-mini": { "input_cost_per_token": 1.1e-06, "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "cache_read_input_token_cost": 5.5e-07, + "supports_prompt_caching": true, + "source": "https://openrouter.ai/openai/o3-mini" }, "openrouter/openai/o3-mini-high": { "input_cost_per_token": 1.1e-06, "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "cache_read_input_token_cost": 5.5e-07, + "supports_prompt_caching": true, + "source": "https://openrouter.ai/openai/o3-mini-high" }, "openrouter/qwen/qwen-2.5-coder-32b-instruct": { "input_cost_per_token": 1.8e-07, @@ -39672,21 +39696,33 @@ "mode": "responses", "supports_web_search": true, "supports_reasoning": true, - "supports_function_calling": true + "supports_function_calling": true, + "input_cost_per_token": 1.75e-06, + "output_cost_per_token": 1.4e-05, + "cache_read_input_token_cost": 1.75e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/openai/gpt-5.1": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/openai/gpt-5-mini": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 2.5e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/anthropic/claude-opus-4-6": { "supports_adaptive_thinking": true, @@ -39696,7 +39732,11 @@ "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, - "supports_output_config": true + "supports_output_config": true, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "cache_read_input_token_cost": 5e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/anthropic/claude-opus-4-7": { "supports_adaptive_thinking": true, @@ -39705,7 +39745,11 @@ "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, - "supports_output_config": true + "supports_output_config": true, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "cache_read_input_token_cost": 5e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/anthropic/claude-opus-4-5": { "litellm_provider": "perplexity", @@ -39713,21 +39757,33 @@ "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, - "supports_output_config": true + "supports_output_config": true, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "cache_read_input_token_cost": 5e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/anthropic/claude-sonnet-4-5": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/anthropic/claude-haiku-4-5": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 5e-06, + "cache_read_input_token_cost": 1e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/google/gemini-3-pro-preview": { "litellm_provider": "perplexity", @@ -39741,7 +39797,11 @@ "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 5e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/google/gemini-2.5-pro": { "litellm_provider": "perplexity", @@ -39770,7 +39830,11 @@ "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 6.25e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/perplexity/deepseek-v4-flash-0731": { "cache_read_input_token_cost": 2.8e-08, @@ -59010,5 +59074,892 @@ "supported_endpoints": [ "/v1/audio/transcriptions" ] + }, + "gemini/lyria-3.5-clip-preview": { + "input_cost_per_token": 0, + "litellm_provider": "gemini", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_image": 0.04, + "output_cost_per_token": 0, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_prompt_caching": false, + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_web_search": false + }, + "gemini/lyria-3.5-pro-preview": { + "input_cost_per_token": 0, + "litellm_provider": "gemini", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_prompt_caching": false, + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_web_search": false, + "output_cost_per_image": 0.08 + }, + "perplexity/anthropic/claude-fable-5": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "cache_read_input_token_cost": 1e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/anthropic/claude-opus-5": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "cache_read_input_token_cost": 5e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/anthropic/claude-opus-4-8": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "cache_read_input_token_cost": 5e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/anthropic/claude-sonnet-5": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 2e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/anthropic/claude-sonnet-4-6": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/openai/gpt-5.6-sol": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 3e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token_above_272k_tokens": 1e-05, + "output_cost_per_token_above_272k_tokens": 4.5e-05, + "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/openai/gpt-5.6-terra": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token_above_272k_tokens": 4e-06, + "output_cost_per_token_above_272k_tokens": 1.8e-05, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/openai/gpt-5.6-luna": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token_above_272k_tokens": 4e-07, + "output_cost_per_token_above_272k_tokens": 1.8e-06, + "cache_read_input_token_cost_above_272k_tokens": 4e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/openai/gpt-5.5": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 3e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token_above_272k_tokens": 1e-05, + "output_cost_per_token_above_272k_tokens": 4.5e-05, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/openai/gpt-5.4": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token_above_272k_tokens": 5e-06, + "output_cost_per_token_above_272k_tokens": 2.25e-05, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/openai/gpt-5.4-mini": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 4.5e-06, + "cache_read_input_token_cost": 7.5e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/openai/gpt-5.4-nano": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.25e-06, + "cache_read_input_token_cost": 2e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/openai/gpt-5": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/google/gemini-3.1-pro-preview": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token_above_200k_tokens": 4e-06, + "output_cost_per_token_above_200k_tokens": 1.8e-05, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/google/gemini-3.1-flash-lite": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.5e-06, + "cache_read_input_token_cost": 2.5e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/google/gemini-3.5-flash": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 9e-06, + "cache_read_input_token_cost": 1.5e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/google/gemini-3.5-flash-lite": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 3e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/google/gemini-3.6-flash": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 7.5e-06, + "cache_read_input_token_cost": 1.5e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/google/gemini-3.7-flash": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 3.75e-06, + "cache_read_input_token_cost": 7.5e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/xai/grok-4.6": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token_above_200k_tokens": 4e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/xai/grok-4.5": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token_above_200k_tokens": 4e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/xai/grok-4.3": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/xai/grok-4.20-reasoning": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/xai/grok-4.20-non-reasoning": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/xai/grok-4.20-multi-agent": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/perplexity/glm-5.3": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "cache_read_input_token_cost": 2.6e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/perplexity/glm-5.3-flash": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 3e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/perplexity/nemotron-3.5-lightning-30b-a3b": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1.15e-08, + "output_cost_per_token": 1.7e-07, + "cache_read_input_token_cost": 1.15e-09, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/perplexity/nemotron-3-ultra-550b-a55b": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2.5e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "openrouter/anthropic/claude-fable-5": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/anthropic/claude-fable-5", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 1e-06, + "supports_prompt_caching": true, + "cache_creation_input_token_cost": 1.25e-05 + }, + "openrouter/anthropic/claude-fable-5.1": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/anthropic/claude-fable-5.1", + "supports_function_calling": true, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2.5e-07, + "supports_prompt_caching": true, + "cache_creation_input_token_cost": 1.25e-05 + }, + "openrouter/anthropic/claude-opus-4.8": { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/anthropic/claude-opus-4.8", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 5e-07, + "supports_prompt_caching": true, + "cache_creation_input_token_cost": 6.25e-06 + }, + "openrouter/anthropic/claude-sonnet-5": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/anthropic/claude-sonnet-5", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true, + "cache_creation_input_token_cost": 2.5e-06 + }, + "openrouter/google/gemini-2.5-flash-lite": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_tokens": 65535, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-2.5-flash-lite", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "cache_read_input_token_cost": 1e-08, + "supports_prompt_caching": true + }, + "openrouter/google/gemini-3.5-flash": { + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 9e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.5-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "cache_read_input_token_cost": 1.5e-07, + "supports_prompt_caching": true + }, + "openrouter/google/gemini-3.5-flash-lite": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2.5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.5-flash-lite", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "cache_read_input_token_cost": 3e-08, + "supports_prompt_caching": true + }, + "openrouter/google/gemini-3.6-flash": { + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 3.75e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.6-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "cache_read_input_token_cost": 7.5e-08, + "supports_prompt_caching": true + }, + "openrouter/google/gemini-3.7-flash": { + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 3.75e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.7-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "cache_read_input_token_cost": 7.5e-08, + "supports_prompt_caching": true + }, + "openrouter/google/gemini-3.8-flash": { + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 3.75e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.8-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "cache_read_input_token_cost": 7.5e-08, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-4o-mini": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-4o-mini", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 7.5e-08, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.1": { + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.1", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 1.25e-07, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.3-codex": { + "input_cost_per_token": 1.75e-06, + "output_cost_per_token": 1.4e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.3-codex", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 1.75e-07, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.4": { + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1.5e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.4", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2.5e-07, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.4-mini": { + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 4.5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.4-mini", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 7.5e-08, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.4-nano": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.25e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.4-nano", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2e-08, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.5": { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 3e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.5", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 5e-07, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.6-luna": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.6-luna", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2e-08, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.6-terra": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.6-terra", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true + }, + "openrouter/openai/o3": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "source": "https://openrouter.ai/openai/o3", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 5e-07, + "supports_prompt_caching": true + }, + "openrouter/openai/o4-mini": { + "input_cost_per_token": 1.1e-06, + "output_cost_per_token": 4.4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "source": "https://openrouter.ai/openai/o4-mini", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2.75e-07, + "supports_prompt_caching": true + }, + "openrouter/x-ai/grok-4.20": { + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 2.5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 2000000, + "max_output_tokens": 1800000, + "max_tokens": 1800000, + "mode": "chat", + "source": "https://openrouter.ai/x-ai/grok-4.20", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true + }, + "openrouter/x-ai/grok-4.20-multi-agent": { + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 2.5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 2000000, + "max_output_tokens": 1800000, + "max_tokens": 1800000, + "mode": "chat", + "source": "https://openrouter.ai/x-ai/grok-4.20-multi-agent", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true + }, + "openrouter/x-ai/grok-4.3": { + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 2.5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 900000, + "max_tokens": 900000, + "mode": "chat", + "source": "https://openrouter.ai/x-ai/grok-4.3", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true + }, + "openrouter/x-ai/grok-4.5": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 500000, + "max_output_tokens": 450000, + "max_tokens": 450000, + "mode": "chat", + "source": "https://openrouter.ai/x-ai/grok-4.5", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 3e-07, + "supports_prompt_caching": true + }, + "openrouter/x-ai/grok-4.6": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 500000, + "max_output_tokens": 450000, + "max_tokens": 450000, + "mode": "chat", + "source": "https://openrouter.ai/x-ai/grok-4.6", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 5e-07, + "supports_prompt_caching": true + }, + "openrouter/x-ai/grok-build-0.1": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 256000, + "max_output_tokens": 230400, + "max_tokens": 230400, + "mode": "chat", + "source": "https://openrouter.ai/x-ai/grok-build-0.1", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": false, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true } } From 5acb81888d6f62108194b57beccbb83c3e302ee6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:52:58 -0700 Subject: [PATCH 31/88] fix(proxy): settle rate-limit reservations at a failed stream's partial usage --- .../hooks/parallel_request_limiter_v3.py | 58 +++--- .../hooks/test_parallel_request_limiter_v3.py | 167 ++++++++++++++++++ 2 files changed, 204 insertions(+), 21 deletions(-) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 63129602082..31437af7770 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -4518,12 +4518,25 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): statuses=statuses, ) + def _recovered_partial_usage_tokens(self, source: Mapping[str, object]) -> tuple[int, int, int]: + usage: Final = source.get("combined_usage_object") + if not isinstance(usage, Usage) or (usage.completion_tokens or 0) <= 0: + return 0, 0, 0 + billable_input, completion_tokens, _ = self._resolve_io_token_reconcile_usage(usage) + return ( + self._get_total_tokens_from_usage(usage=usage, rate_limit_type=self.get_rate_limit_type()), + billable_input, + completion_tokens, + ) + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): """ On failure: decrement max_parallel_requests and refund the upfront TPM reservation only against the scopes the reservation actually charged. Unreserved scopes were never incremented at pre-call, so - refunding them would drive their counter negative. + refunding them would drive their counter negative. A failed stream + whose partial usage was recovered settles the reservation at that + usage instead of refunding it. """ from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, @@ -4552,31 +4565,31 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if stash is None or stash.reservation_released else (stash.reserved_tokens, stash.itpm_reserved_tokens, stash.otpm_reserved_tokens) ) + tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(kwargs) if stash is not None and reserved_tokens > 0: - verbose_proxy_logger.debug("Releasing reserved TPM tokens on failure: %s", reserved_tokens) - # Refund only against the scopes the reservation actually - # charged. _build_reservation_aware_tpm_ops with - # actual_tokens=0 emits -reserved on reserved scopes and 0 - # on unreserved (skipped), so unreserved scopes can't drift - # negative. + verbose_proxy_logger.debug( + "Settling reserved TPM tokens on failure: reserved=%s actual=%s", reserved_tokens, tpm_actual + ) + # Settle only against the scopes the reservation actually + # charged: unreserved scopes were never incremented, so a + # refund there would drive their counter negative. pipeline_operations.extend( self._build_reservation_aware_tpm_ops( targets=list(stash.reserved_scopes), reserved_scopes=stash.reserved_scopes, - actual_tokens=0, + actual_tokens=tpm_actual, reserved_tokens=reserved_tokens, ) ) - # Refund project ITPM/OTPM reservations the same way -- full - # refund, since a failed call has no billable usage to reconcile - # against. + # Settle project ITPM/OTPM reservations the same way: at the + # recovered partial usage, or a full refund when there is none. itpm_operations: Final = ( self._build_project_reservation_ops( targets=tuple(stash.itpm_reserved_scopes), reserved_scopes=stash.itpm_reserved_scopes, - actual_tokens=0, + actual_tokens=itpm_actual, reserved_tokens=itpm_reserved, reservation_window_identities=stash.itpm_reserved_window_identities, ) @@ -4584,7 +4597,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): else self._build_reservation_aware_tpm_ops( targets=tuple(stash.itpm_reserved_scopes), reserved_scopes=stash.itpm_reserved_scopes, - actual_tokens=0, + actual_tokens=itpm_actual, reserved_tokens=itpm_reserved, ) if stash is not None and itpm_reserved > 0 @@ -4595,7 +4608,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): self._build_project_reservation_ops( targets=tuple(stash.otpm_reserved_scopes), reserved_scopes=stash.otpm_reserved_scopes, - actual_tokens=0, + actual_tokens=otpm_actual, reserved_tokens=otpm_reserved, reservation_window_identities=stash.otpm_reserved_window_identities, ) @@ -4603,7 +4616,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): else self._build_reservation_aware_tpm_ops( targets=tuple(stash.otpm_reserved_scopes), reserved_scopes=stash.otpm_reserved_scopes, - actual_tokens=0, + actual_tokens=otpm_actual, reserved_tokens=otpm_reserved, ) if stash is not None and otpm_reserved > 0 @@ -4742,7 +4755,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): removal is a no-op ZREM on a second run), and the TPM/ITPM/OTPM refund is guarded by the stash's ``reservation_released`` flag — if both this hook and async_log_failure_event end up running in the same - flow, only the first release/refund applies. + flow, only the first release/refund applies. A mid-stream failure + relayed here with recovered partial usage settles the reservation at + that usage instead of refunding it. """ try: stash: Final = get_request_stash() @@ -4769,12 +4784,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): otpm_reserved: Final = stash.otpm_reserved_tokens if reserved_tokens <= 0 and itpm_reserved <= 0 and otpm_reserved <= 0: return + tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(request_data) combined_ops: Final = ( self._build_reservation_aware_tpm_ops( targets=tuple(stash.reserved_scopes), reserved_scopes=stash.reserved_scopes, - actual_tokens=0, + actual_tokens=tpm_actual, reserved_tokens=reserved_tokens, ) if reserved_tokens > 0 @@ -4784,7 +4800,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): self._build_project_reservation_ops( targets=tuple(stash.itpm_reserved_scopes), reserved_scopes=stash.itpm_reserved_scopes, - actual_tokens=0, + actual_tokens=itpm_actual, reserved_tokens=itpm_reserved, reservation_window_identities=stash.itpm_reserved_window_identities, ) @@ -4792,7 +4808,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): else self._build_reservation_aware_tpm_ops( targets=tuple(stash.itpm_reserved_scopes), reserved_scopes=stash.itpm_reserved_scopes, - actual_tokens=0, + actual_tokens=itpm_actual, reserved_tokens=itpm_reserved, ) if itpm_reserved > 0 @@ -4802,7 +4818,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): self._build_project_reservation_ops( targets=tuple(stash.otpm_reserved_scopes), reserved_scopes=stash.otpm_reserved_scopes, - actual_tokens=0, + actual_tokens=otpm_actual, reserved_tokens=otpm_reserved, reservation_window_identities=stash.otpm_reserved_window_identities, ) @@ -4810,7 +4826,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): else self._build_reservation_aware_tpm_ops( targets=tuple(stash.otpm_reserved_scopes), reserved_scopes=stash.otpm_reserved_scopes, - actual_tokens=0, + actual_tokens=otpm_actual, reserved_tokens=otpm_reserved, ) if otpm_reserved > 0 diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index fc0088b28d7..4003286d887 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -3647,6 +3647,173 @@ async def test_stash_applies_when_owner_or_callback_call_id_missing(): assert claimed.reservation_released is True +async def _reserve_tpm_for_owner_call(handler, local_cache, api_key: str, call_id: str) -> int: + await handler.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(api_key=api_key, tpm_limit=10_000), + cache=local_cache, + data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + "litellm_call_id": call_id, + }, + call_type="completion", + ) + stash = get_request_stash() + assert stash is not None and stash.reserved_tokens > 0 + return stash.reserved_tokens + + +@pytest.mark.asyncio +async def test_failure_event_settles_tpm_reservation_at_recovered_partial_usage_v3(): + """ + A stream that fails mid-way after the model already produced tokens is + logged as a failure carrying the recovered partial usage. Those tokens + were consumed, so the TPM window must settle at them instead of refunding + the whole reservation (which would let repeated timeouts burn output + tokens for free). + """ + _api_key = hash_token("sk-partial-stream-failure") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + tokens_key = handler.create_rate_limit_keys(key="api_key", value=_api_key, rate_limit_type="tokens") + await _reserve_tpm_for_owner_call(handler, local_cache, _api_key, "partial-call") + + await handler.async_log_failure_event( + kwargs={ + "litellm_call_id": "partial-call", + "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, + "combined_usage_object": Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27), + }, + response_obj=None, + start_time=None, + end_time=None, + ) + + assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == 27 + stash = get_request_stash() + assert stash is not None and stash.reservation_released is True + + +@pytest.mark.asyncio +async def test_failure_event_refunds_reservation_for_input_only_estimate_v3(): + """ + A failure with no recovered output carries only the input-token estimate + the proxy lifts onto every failure; that is not consumed usage, so the + reservation is still refunded in full. + """ + _api_key = hash_token("sk-estimated-failure") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + tokens_key = handler.create_rate_limit_keys(key="api_key", value=_api_key, rate_limit_type="tokens") + await _reserve_tpm_for_owner_call(handler, local_cache, _api_key, "estimate-call") + + await handler.async_log_failure_event( + kwargs={ + "litellm_call_id": "estimate-call", + "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, + "combined_usage_object": Usage(prompt_tokens=20, completion_tokens=0, total_tokens=20), + }, + response_obj=None, + start_time=None, + end_time=None, + ) + + assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == 0 + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_settles_reservation_at_recovered_partial_usage_v3(): + """ + Pass-through streams report a mid-stream failure through the proxy-level + failure hook first, with the recovered usage lifted onto request_data. + That hook must settle at the partial usage too, and the later failure + callback must not double-apply it. + """ + _api_key = hash_token("sk-partial-post-call") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + user_api_key_dict = UserAPIKeyAuth(api_key=_api_key, tpm_limit=10_000) + tokens_key = handler.create_rate_limit_keys(key="api_key", value=_api_key, rate_limit_type="tokens") + await _reserve_tpm_for_owner_call(handler, local_cache, _api_key, "post-call") + + await handler.async_post_call_failure_hook( + request_data={ + "model": "gpt-4o-mini", + "litellm_call_id": "post-call", + "combined_usage_object": Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27), + }, + original_exception=Exception("upstream dropped the stream"), + user_api_key_dict=user_api_key_dict, + ) + assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == 27 + + await handler.async_log_failure_event( + kwargs={ + "litellm_call_id": "post-call", + "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, + "combined_usage_object": Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27), + }, + response_obj=None, + start_time=None, + end_time=None, + ) + assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == 27 + + +@pytest.mark.asyncio +async def test_failure_event_settles_project_itpm_otpm_at_recovered_partial_usage_v3(): + """ + Project ITPM/OTPM reservations settle the same way: input at the billable + prompt tokens and output at the completion tokens the failed stream + actually produced. + """ + _api_key = hash_token("sk-partial-project-io") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, + project_id="proj-partial", + project_metadata={ + "model_itpm_limit": {"gpt-4o-mini": 10_000}, + "model_otpm_limit": {"gpt-4o-mini": 10_000}, + }, + ) + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + "litellm_call_id": "project-call", + }, + call_type="completion", + ) + stash = get_request_stash() + assert stash is not None and stash.itpm_reserved_tokens > 0 and stash.otpm_reserved_tokens > 0 + itpm_key = handler.create_rate_limit_keys( + key="model_per_project_itpm", value="proj-partial:gpt-4o-mini", rate_limit_type="tokens" + ) + otpm_key = handler.create_rate_limit_keys( + key="model_per_project_otpm", value="proj-partial:gpt-4o-mini", rate_limit_type="tokens" + ) + + await handler.async_log_failure_event( + kwargs={ + "litellm_call_id": "project-call", + "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, + "combined_usage_object": Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27), + }, + response_obj=None, + start_time=None, + end_time=None, + ) + + assert int(await local_cache.async_get_cache(key=itpm_key) or 0) == 20 + assert int(await local_cache.async_get_cache(key=otpm_key) or 0) == 7 + + # ----------------------- Per-MCP-server rate limiting (v3) ----------------------- From 2c4eb693ed92a1fe933794b3c8cf3cc0e6b905d9 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 3 Sep 2026 19:53:58 +0000 Subject: [PATCH 32/88] fix(model_prices): absorb Baseten GLM-5.3 and OpenRouter live prices, fix Bedrock Qwen3 Coder 480B input price and Gemini Live image price Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 253 ++++++++++-------- model_prices_and_context_window.json | 253 ++++++++++-------- .../test_get_model_cost_map.py | 52 +++- .../test_baseten_glm_5_3_model_metadata.py | 154 +++++++++++ tests/test_litellm/test_utils.py | 8 +- 5 files changed, 487 insertions(+), 233 deletions(-) create mode 100644 tests/test_litellm/test_baseten_glm_5_3_model_metadata.py diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 724929a48f5..a6f3095e974 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -22968,7 +22968,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "gemini_native_audio": true + "gemini_native_audio": true, + "input_cost_per_image_token": 3e-06 }, "gemini-live-2.5-flash-preview-native-audio-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -37637,7 +37638,7 @@ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "litellm_provider": "openrouter", - "max_input_tokens": 1000000, + "max_input_tokens": 200000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", @@ -37733,24 +37734,24 @@ "supports_tool_choice": true }, "openrouter/deepseek/deepseek-chat": { - "input_cost_per_token": 1.4e-07, + "input_cost_per_token": 3.2e-07, "litellm_provider": "openrouter", "max_input_tokens": 65536, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 2.8e-07, + "output_cost_per_token": 8.9e-07, "supports_prompt_caching": true, "supports_tool_choice": true }, "openrouter/deepseek/deepseek-chat-v3-0324": { - "input_cost_per_token": 1.4e-07, + "input_cost_per_token": 2.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 65536, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 2.8e-07, + "output_cost_per_token": 1e-06, "supports_prompt_caching": true, "supports_tool_choice": true }, @@ -37770,7 +37771,7 @@ "supports_tool_choice": true }, "openrouter/deepseek/deepseek-v3.2": { - "input_cost_per_token": 2.8e-07, + "input_cost_per_token": 2.69e-07, "input_cost_per_token_cache_hit": 2.8e-08, "litellm_provider": "openrouter", "max_input_tokens": 163840, @@ -37785,14 +37786,14 @@ "supports_tool_choice": true }, "openrouter/deepseek/deepseek-v3.2-exp": { - "input_cost_per_token": 2e-07, + "input_cost_per_token": 2.7e-07, "input_cost_per_token_cache_hit": 2e-08, "litellm_provider": "openrouter", "max_input_tokens": 163840, "max_output_tokens": 163840, "max_tokens": 163840, "mode": "chat", - "output_cost_per_token": 4e-07, + "output_cost_per_token": 4.1e-07, "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -37800,14 +37801,14 @@ "supports_tool_choice": true }, "openrouter/deepseek/deepseek-r1": { - "input_cost_per_token": 5.5e-07, + "input_cost_per_token": 7e-07, "input_cost_per_token_cache_hit": 1.4e-07, "litellm_provider": "openrouter", "max_input_tokens": 65336, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 2.19e-06, + "output_cost_per_token": 2.5e-06, "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -37903,8 +37904,8 @@ "input_cost_per_token": 1.25e-06, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_output_tokens": 65535, + "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 1e-05, "supports_audio_output": true, @@ -38119,19 +38120,19 @@ "supports_vision": true }, "openrouter/gryphe/mythomax-l2-13b": { - "input_cost_per_token": 1.875e-06, + "input_cost_per_token": 6e-08, "litellm_provider": "openrouter", "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.875e-06, + "output_cost_per_token": 6e-08, "supports_tool_choice": true }, "openrouter/mancer/weaver": { - "input_cost_per_token": 5.625e-06, + "input_cost_per_token": 4e-07, "litellm_provider": "openrouter", "max_tokens": 2000, "mode": "chat", - "output_cost_per_token": 5.625e-06, + "output_cost_per_token": 7.5e-07, "supports_tool_choice": true, "max_input_tokens": 8000, "max_output_tokens": 2000 @@ -38161,13 +38162,13 @@ }, "openrouter/mistralai/devstral-2512": { "input_cost_per_image": 0, - "input_cost_per_token": 1.5e-07, + "input_cost_per_token": 4e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 6e-07, + "output_cost_per_token": 2e-06, "supports_function_calling": true, "supports_prompt_caching": false, "supports_tool_choice": true, @@ -38240,54 +38241,54 @@ "max_output_tokens": 8191 }, "openrouter/mistralai/mistral-large": { - "input_cost_per_token": 8e-06, + "input_cost_per_token": 2e-06, "litellm_provider": "openrouter", "max_tokens": 8191, "mode": "chat", - "output_cost_per_token": 2.4e-05, + "output_cost_per_token": 6e-06, "supports_tool_choice": true, "max_input_tokens": 128000, "max_output_tokens": 8191 }, "openrouter/mistralai/mistral-small-3.1-24b-instruct": { - "input_cost_per_token": 1e-07, + "input_cost_per_token": 3.51e-07, "litellm_provider": "openrouter", "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 3e-07, + "output_cost_per_token": 5.55e-07, "supports_tool_choice": true, "max_input_tokens": 131072, "max_output_tokens": 131072 }, "openrouter/mistralai/mistral-small-3.2-24b-instruct": { - "input_cost_per_token": 1e-07, + "input_cost_per_token": 7.5e-08, "litellm_provider": "openrouter", "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3e-07, + "output_cost_per_token": 2e-07, "supports_tool_choice": true, "max_input_tokens": 128000, "max_output_tokens": 128000 }, "openrouter/mistralai/mixtral-8x22b-instruct": { - "input_cost_per_token": 6.5e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "openrouter", "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 6.5e-07, + "output_cost_per_token": 6e-06, "supports_tool_choice": true, "max_input_tokens": 65536, "max_output_tokens": 65536 }, "openrouter/moonshotai/kimi-k2.5": { - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 6e-07, + "cache_read_input_token_cost": 7e-08, + "input_cost_per_token": 4.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 262144, "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 3e-06, + "output_cost_per_token": 2.25e-06, "source": "https://openrouter.ai/moonshotai/kimi-k2.5", "supports_function_calling": true, "supports_tool_choice": true, @@ -38295,7 +38296,7 @@ "supports_vision": true }, "openrouter/nvidia/nemotron-3.5-lightning": { - "input_cost_per_token": 5e-08, + "input_cost_per_token": 8e-08, "litellm_provider": "openrouter", "max_input_tokens": 262144, "mode": "chat", @@ -38600,13 +38601,13 @@ "supports_vision": true }, "openrouter/openai/gpt-oss-120b": { - "input_cost_per_token": 1.8e-07, + "input_cost_per_token": 3.7e-08, "litellm_provider": "openrouter", "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 8e-07, + "output_cost_per_token": 1.7e-07, "source": "https://openrouter.ai/openai/gpt-oss-120b", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -38615,13 +38616,13 @@ "supports_tool_choice": true }, "openrouter/openai/gpt-oss-20b": { - "input_cost_per_token": 2e-08, + "input_cost_per_token": 3e-08, "litellm_provider": "openrouter", "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 1e-07, + "output_cost_per_token": 1.3e-07, "source": "https://openrouter.ai/openai/gpt-oss-20b", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -38681,13 +38682,13 @@ "source": "https://openrouter.ai/openai/o3-mini-high" }, "openrouter/qwen/qwen-2.5-coder-32b-instruct": { - "input_cost_per_token": 1.8e-07, + "input_cost_per_token": 6.6e-07, "litellm_provider": "openrouter", "max_input_tokens": 33792, "max_output_tokens": 33792, "max_tokens": 33792, "mode": "chat", - "output_cost_per_token": 1.8e-07, + "output_cost_per_token": 1e-06, "supports_tool_choice": true }, "openrouter/qwen/qwen-vl-plus": { @@ -38702,50 +38703,50 @@ "supports_vision": true }, "openrouter/qwen/qwen3-coder": { - "input_cost_per_token": 2.2e-07, + "input_cost_per_token": 3e-07, "litellm_provider": "openrouter", "max_input_tokens": 262100, "max_output_tokens": 262100, "max_tokens": 262100, "mode": "chat", - "output_cost_per_token": 9.5e-07, + "output_cost_per_token": 1e-06, "source": "https://openrouter.ai/qwen/qwen3-coder", "supports_tool_choice": true, "supports_function_calling": true }, "openrouter/qwen/qwen3-coder-plus": { - "input_cost_per_token": 1e-06, + "input_cost_per_token": 6.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 997952, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 5e-06, + "output_cost_per_token": 3.25e-06, "source": "https://openrouter.ai/qwen/qwen3-coder-plus", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true }, "openrouter/qwen/qwen3-235b-a22b-2507": { - "input_cost_per_token": 7.1e-08, + "input_cost_per_token": 8.75e-08, "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 262144, "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 1e-07, + "output_cost_per_token": 3.5e-07, "source": "https://openrouter.ai/qwen/qwen3-235b-a22b-2507", "supports_function_calling": true, "supports_tool_choice": true }, "openrouter/qwen/qwen3-235b-a22b-thinking-2507": { - "input_cost_per_token": 1.1e-07, + "input_cost_per_token": 2.3e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 262144, "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 6e-07, + "output_cost_per_token": 2.3e-06, "source": "https://openrouter.ai/qwen/qwen3-235b-a22b-thinking-2507", "supports_function_calling": true, "supports_reasoning": true, @@ -38772,7 +38773,7 @@ "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 2e-06, + "output_cost_per_token": 1.25e-06, "source": "https://openrouter.ai/qwen/qwen3.5-35b-a3b", "supports_function_calling": true, "supports_reasoning": true, @@ -38780,13 +38781,13 @@ "supports_vision": true }, "openrouter/qwen/qwen3.5-27b": { - "input_cost_per_token": 3e-07, + "input_cost_per_token": 1.95e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 2.4e-06, + "output_cost_per_token": 1.56e-06, "source": "https://openrouter.ai/qwen/qwen3.5-27b", "supports_function_calling": true, "supports_reasoning": true, @@ -38794,13 +38795,13 @@ "supports_vision": true }, "openrouter/qwen/qwen3.5-122b-a10b": { - "input_cost_per_token": 4e-07, + "input_cost_per_token": 2.9e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 2e-06, + "output_cost_per_token": 2.4e-06, "source": "https://openrouter.ai/qwen/qwen3.5-122b-a10b", "supports_function_calling": true, "supports_reasoning": true, @@ -38808,13 +38809,13 @@ "supports_vision": true }, "openrouter/qwen/qwen3.5-flash-02-23": { - "input_cost_per_token": 1e-07, + "input_cost_per_token": 6.5e-08, "litellm_provider": "openrouter", "max_input_tokens": 1000000, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 4e-07, + "output_cost_per_token": 2.6e-07, "source": "https://openrouter.ai/qwen/qwen3.5-flash-02-23", "supports_function_calling": true, "supports_reasoning": true, @@ -38822,14 +38823,14 @@ "supports_vision": true }, "openrouter/qwen/qwen3.5-plus-02-15": { - "input_cost_per_token": 4e-07, + "input_cost_per_token": 2.6e-07, "input_cost_per_token_above_256k_tokens": 5e-07, "litellm_provider": "openrouter", "max_input_tokens": 1000000, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 2.4e-06, + "output_cost_per_token": 1.56e-06, "output_cost_per_token_above_256k_tokens": 3e-06, "source": "https://openrouter.ai/qwen/qwen3.5-plus-02-15", "supports_function_calling": true, @@ -38838,13 +38839,13 @@ "supports_vision": true }, "openrouter/qwen/qwen3.5-397b-a17b": { - "input_cost_per_token": 6e-07, + "input_cost_per_token": 5.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 3.6e-06, + "output_cost_per_token": 3.5e-06, "source": "https://openrouter.ai/qwen/qwen3.5-397b-a17b", "supports_function_calling": true, "supports_reasoning": true, @@ -38863,11 +38864,11 @@ "supports_tool_choice": true }, "openrouter/undi95/remm-slerp-l2-13b": { - "input_cost_per_token": 1.875e-06, + "input_cost_per_token": 4.5e-07, "litellm_provider": "openrouter", "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 1.875e-06, + "output_cost_per_token": 6.5e-07, "supports_tool_choice": true, "max_input_tokens": 6144, "max_output_tokens": 4096 @@ -38887,13 +38888,13 @@ "supports_web_search": true }, "openrouter/z-ai/glm-4.6": { - "input_cost_per_token": 4e-07, + "input_cost_per_token": 5.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 202800, "max_output_tokens": 131000, "max_tokens": 131000, "mode": "chat", - "output_cost_per_token": 1.75e-06, + "output_cost_per_token": 2.2e-06, "source": "https://openrouter.ai/z-ai/glm-4.6", "supports_function_calling": true, "supports_prompt_caching": true, @@ -38931,10 +38932,10 @@ "supports_prompt_caching": true }, "openrouter/xiaomi/mimo-v2.5-pro": { - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3e-06, + "input_cost_per_token": 4.35e-07, + "output_cost_per_token": 8.7e-07, "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost": 3.6e-09, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 16384, @@ -38948,10 +38949,10 @@ "supports_prompt_caching": true }, "openrouter/xiaomi/mimo-v2.5": { - "input_cost_per_token": 4e-07, - "output_cost_per_token": 2e-06, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 8e-08, + "cache_read_input_token_cost": 2.8e-09, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 131072, @@ -38968,9 +38969,9 @@ }, "openrouter/z-ai/glm-4.7": { "input_cost_per_token": 4e-07, - "output_cost_per_token": 1.5e-06, + "output_cost_per_token": 1.75e-06, "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 0.0, + "cache_read_input_token_cost": 8e-08, "litellm_provider": "openrouter", "max_input_tokens": 202752, "max_output_tokens": 64000, @@ -38984,10 +38985,10 @@ "supports_assistant_prefill": true }, "openrouter/z-ai/glm-4.7-flash": { - "input_cost_per_token": 7e-08, + "input_cost_per_token": 6e-08, "output_cost_per_token": 4e-07, "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 0.0, + "cache_read_input_token_cost": 1e-08, "litellm_provider": "openrouter", "max_input_tokens": 200000, "max_output_tokens": 32000, @@ -39000,22 +39001,22 @@ "supports_prompt_caching": false }, "openrouter/z-ai/glm-5": { - "input_cost_per_token": 8e-07, + "input_cost_per_token": 6e-07, "litellm_provider": "openrouter", "max_input_tokens": 202752, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 2.56e-06, + "output_cost_per_token": 1.92e-06, "source": "https://openrouter.ai/z-ai/glm-5", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true }, "openrouter/z-ai/glm-5.1": { - "input_cost_per_token": 1.05e-06, - "output_cost_per_token": 3.5e-06, - "cache_read_input_token_cost": 5.25e-07, + "input_cost_per_token": 9.66e-07, + "output_cost_per_token": 3.036e-06, + "cache_read_input_token_cost": 1.794e-07, "cache_creation_input_token_cost": 0.0, "litellm_provider": "openrouter", "max_input_tokens": 202752, @@ -39029,10 +39030,10 @@ "supports_tool_choice": true }, "openrouter/minimax/minimax-m2.1": { - "input_cost_per_token": 2.7e-07, + "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 0.0, + "cache_read_input_token_cost": 3e-08, "litellm_provider": "openrouter", "max_input_tokens": 204000, "max_output_tokens": 64000, @@ -39046,9 +39047,9 @@ "supports_computer_use": false }, "openrouter/minimax/minimax-m2.5": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.1e-06, - "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 1.08e-06, + "cache_read_input_token_cost": 2.7e-08, "litellm_provider": "openrouter", "max_input_tokens": 196608, "max_output_tokens": 65536, @@ -39947,7 +39948,7 @@ "supports_reasoning": true }, "qwen.qwen3-coder-480b-a35b-v1:0": { - "input_cost_per_token": 2.2e-07, + "input_cost_per_token": 4.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 262000, "max_output_tokens": 65536, @@ -39957,7 +39958,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json" }, "qwen.qwen3-235b-a22b-2507-v1:0": { "input_cost_per_token": 2.2e-07, @@ -52852,7 +52854,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -52878,7 +52880,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -52904,7 +52906,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -52932,7 +52934,7 @@ "max_input_tokens": 131072, "max_output_tokens": 65536, "max_tokens": 65536, - "mode": "chat", + "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -52963,7 +52965,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -52991,7 +52993,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -53019,7 +53021,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -53049,7 +53051,7 @@ "max_input_tokens": 131072, "max_output_tokens": 65536, "max_tokens": 65536, - "mode": "chat", + "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -59463,7 +59465,8 @@ "supports_audio_input": false, "cache_read_input_token_cost": 1e-06, "supports_prompt_caching": true, - "cache_creation_input_token_cost": 1.25e-05 + "cache_creation_input_token_cost": 1.25e-05, + "prompt_cache_min_tokens": 512 }, "openrouter/anthropic/claude-fable-5.1": { "input_cost_per_token": 1e-05, @@ -59483,7 +59486,8 @@ "supports_audio_input": false, "cache_read_input_token_cost": 2.5e-07, "supports_prompt_caching": true, - "cache_creation_input_token_cost": 1.25e-05 + "cache_creation_input_token_cost": 1.25e-05, + "prompt_cache_min_tokens": 512 }, "openrouter/anthropic/claude-opus-4.8": { "input_cost_per_token": 5e-06, @@ -59549,8 +59553,8 @@ "output_cost_per_token": 9e-06, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_output_tokens": 65535, + "max_tokens": 65535, "mode": "chat", "source": "https://openrouter.ai/google/gemini-3.5-flash", "supports_function_calling": true, @@ -59662,7 +59666,7 @@ "input_cost_per_token": 1.25e-06, "output_cost_per_token": 1e-05, "litellm_provider": "openrouter", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -59681,7 +59685,7 @@ "input_cost_per_token": 1.75e-06, "output_cost_per_token": 1.4e-05, "litellm_provider": "openrouter", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -59719,7 +59723,7 @@ "input_cost_per_token": 7.5e-07, "output_cost_per_token": 4.5e-06, "litellm_provider": "openrouter", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -59738,7 +59742,7 @@ "input_cost_per_token": 2e-07, "output_cost_per_token": 1.25e-06, "litellm_provider": "openrouter", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -59776,7 +59780,7 @@ "input_cost_per_token": 2e-07, "output_cost_per_token": 1.2e-06, "litellm_provider": "openrouter", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -59795,7 +59799,7 @@ "input_cost_per_token": 2e-06, "output_cost_per_token": 1.2e-05, "litellm_provider": "openrouter", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -59852,9 +59856,9 @@ "input_cost_per_token": 1.25e-06, "output_cost_per_token": 2.5e-06, "litellm_provider": "openrouter", - "max_input_tokens": 2000000, - "max_output_tokens": 1800000, - "max_tokens": 1800000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", "source": "https://openrouter.ai/x-ai/grok-4.20", "supports_function_calling": true, @@ -59871,9 +59875,9 @@ "input_cost_per_token": 1.25e-06, "output_cost_per_token": 2.5e-06, "litellm_provider": "openrouter", - "max_input_tokens": 2000000, - "max_output_tokens": 1800000, - "max_tokens": 1800000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", "source": "https://openrouter.ai/x-ai/grok-4.20-multi-agent", "supports_function_calling": false, @@ -59891,8 +59895,8 @@ "output_cost_per_token": 2.5e-06, "litellm_provider": "openrouter", "max_input_tokens": 1000000, - "max_output_tokens": 900000, - "max_tokens": 900000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", "source": "https://openrouter.ai/x-ai/grok-4.3", "supports_function_calling": true, @@ -59910,8 +59914,8 @@ "output_cost_per_token": 6e-06, "litellm_provider": "openrouter", "max_input_tokens": 500000, - "max_output_tokens": 450000, - "max_tokens": 450000, + "max_output_tokens": 500000, + "max_tokens": 500000, "mode": "chat", "source": "https://openrouter.ai/x-ai/grok-4.5", "supports_function_calling": true, @@ -59929,8 +59933,8 @@ "output_cost_per_token": 6e-06, "litellm_provider": "openrouter", "max_input_tokens": 500000, - "max_output_tokens": 450000, - "max_tokens": 450000, + "max_output_tokens": 500000, + "max_tokens": 500000, "mode": "chat", "source": "https://openrouter.ai/x-ai/grok-4.6", "supports_function_calling": true, @@ -59948,8 +59952,8 @@ "output_cost_per_token": 2e-06, "litellm_provider": "openrouter", "max_input_tokens": 256000, - "max_output_tokens": 230400, - "max_tokens": 230400, + "max_output_tokens": 256000, + "max_tokens": 256000, "mode": "chat", "source": "https://openrouter.ai/x-ai/grok-build-0.1", "supports_function_calling": true, @@ -59961,5 +59965,26 @@ "supports_audio_input": false, "cache_read_input_token_cost": 2e-07, "supports_prompt_caching": true + }, + "baseten/zai-org/GLM-5.3": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://www.baseten.co/pricing/", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 724929a48f5..a6f3095e974 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -22968,7 +22968,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "gemini_native_audio": true + "gemini_native_audio": true, + "input_cost_per_image_token": 3e-06 }, "gemini-live-2.5-flash-preview-native-audio-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -37637,7 +37638,7 @@ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "litellm_provider": "openrouter", - "max_input_tokens": 1000000, + "max_input_tokens": 200000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", @@ -37733,24 +37734,24 @@ "supports_tool_choice": true }, "openrouter/deepseek/deepseek-chat": { - "input_cost_per_token": 1.4e-07, + "input_cost_per_token": 3.2e-07, "litellm_provider": "openrouter", "max_input_tokens": 65536, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 2.8e-07, + "output_cost_per_token": 8.9e-07, "supports_prompt_caching": true, "supports_tool_choice": true }, "openrouter/deepseek/deepseek-chat-v3-0324": { - "input_cost_per_token": 1.4e-07, + "input_cost_per_token": 2.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 65536, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 2.8e-07, + "output_cost_per_token": 1e-06, "supports_prompt_caching": true, "supports_tool_choice": true }, @@ -37770,7 +37771,7 @@ "supports_tool_choice": true }, "openrouter/deepseek/deepseek-v3.2": { - "input_cost_per_token": 2.8e-07, + "input_cost_per_token": 2.69e-07, "input_cost_per_token_cache_hit": 2.8e-08, "litellm_provider": "openrouter", "max_input_tokens": 163840, @@ -37785,14 +37786,14 @@ "supports_tool_choice": true }, "openrouter/deepseek/deepseek-v3.2-exp": { - "input_cost_per_token": 2e-07, + "input_cost_per_token": 2.7e-07, "input_cost_per_token_cache_hit": 2e-08, "litellm_provider": "openrouter", "max_input_tokens": 163840, "max_output_tokens": 163840, "max_tokens": 163840, "mode": "chat", - "output_cost_per_token": 4e-07, + "output_cost_per_token": 4.1e-07, "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -37800,14 +37801,14 @@ "supports_tool_choice": true }, "openrouter/deepseek/deepseek-r1": { - "input_cost_per_token": 5.5e-07, + "input_cost_per_token": 7e-07, "input_cost_per_token_cache_hit": 1.4e-07, "litellm_provider": "openrouter", "max_input_tokens": 65336, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 2.19e-06, + "output_cost_per_token": 2.5e-06, "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -37903,8 +37904,8 @@ "input_cost_per_token": 1.25e-06, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_output_tokens": 65535, + "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 1e-05, "supports_audio_output": true, @@ -38119,19 +38120,19 @@ "supports_vision": true }, "openrouter/gryphe/mythomax-l2-13b": { - "input_cost_per_token": 1.875e-06, + "input_cost_per_token": 6e-08, "litellm_provider": "openrouter", "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.875e-06, + "output_cost_per_token": 6e-08, "supports_tool_choice": true }, "openrouter/mancer/weaver": { - "input_cost_per_token": 5.625e-06, + "input_cost_per_token": 4e-07, "litellm_provider": "openrouter", "max_tokens": 2000, "mode": "chat", - "output_cost_per_token": 5.625e-06, + "output_cost_per_token": 7.5e-07, "supports_tool_choice": true, "max_input_tokens": 8000, "max_output_tokens": 2000 @@ -38161,13 +38162,13 @@ }, "openrouter/mistralai/devstral-2512": { "input_cost_per_image": 0, - "input_cost_per_token": 1.5e-07, + "input_cost_per_token": 4e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 6e-07, + "output_cost_per_token": 2e-06, "supports_function_calling": true, "supports_prompt_caching": false, "supports_tool_choice": true, @@ -38240,54 +38241,54 @@ "max_output_tokens": 8191 }, "openrouter/mistralai/mistral-large": { - "input_cost_per_token": 8e-06, + "input_cost_per_token": 2e-06, "litellm_provider": "openrouter", "max_tokens": 8191, "mode": "chat", - "output_cost_per_token": 2.4e-05, + "output_cost_per_token": 6e-06, "supports_tool_choice": true, "max_input_tokens": 128000, "max_output_tokens": 8191 }, "openrouter/mistralai/mistral-small-3.1-24b-instruct": { - "input_cost_per_token": 1e-07, + "input_cost_per_token": 3.51e-07, "litellm_provider": "openrouter", "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 3e-07, + "output_cost_per_token": 5.55e-07, "supports_tool_choice": true, "max_input_tokens": 131072, "max_output_tokens": 131072 }, "openrouter/mistralai/mistral-small-3.2-24b-instruct": { - "input_cost_per_token": 1e-07, + "input_cost_per_token": 7.5e-08, "litellm_provider": "openrouter", "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3e-07, + "output_cost_per_token": 2e-07, "supports_tool_choice": true, "max_input_tokens": 128000, "max_output_tokens": 128000 }, "openrouter/mistralai/mixtral-8x22b-instruct": { - "input_cost_per_token": 6.5e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "openrouter", "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 6.5e-07, + "output_cost_per_token": 6e-06, "supports_tool_choice": true, "max_input_tokens": 65536, "max_output_tokens": 65536 }, "openrouter/moonshotai/kimi-k2.5": { - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 6e-07, + "cache_read_input_token_cost": 7e-08, + "input_cost_per_token": 4.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 262144, "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 3e-06, + "output_cost_per_token": 2.25e-06, "source": "https://openrouter.ai/moonshotai/kimi-k2.5", "supports_function_calling": true, "supports_tool_choice": true, @@ -38295,7 +38296,7 @@ "supports_vision": true }, "openrouter/nvidia/nemotron-3.5-lightning": { - "input_cost_per_token": 5e-08, + "input_cost_per_token": 8e-08, "litellm_provider": "openrouter", "max_input_tokens": 262144, "mode": "chat", @@ -38600,13 +38601,13 @@ "supports_vision": true }, "openrouter/openai/gpt-oss-120b": { - "input_cost_per_token": 1.8e-07, + "input_cost_per_token": 3.7e-08, "litellm_provider": "openrouter", "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 8e-07, + "output_cost_per_token": 1.7e-07, "source": "https://openrouter.ai/openai/gpt-oss-120b", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -38615,13 +38616,13 @@ "supports_tool_choice": true }, "openrouter/openai/gpt-oss-20b": { - "input_cost_per_token": 2e-08, + "input_cost_per_token": 3e-08, "litellm_provider": "openrouter", "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 1e-07, + "output_cost_per_token": 1.3e-07, "source": "https://openrouter.ai/openai/gpt-oss-20b", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -38681,13 +38682,13 @@ "source": "https://openrouter.ai/openai/o3-mini-high" }, "openrouter/qwen/qwen-2.5-coder-32b-instruct": { - "input_cost_per_token": 1.8e-07, + "input_cost_per_token": 6.6e-07, "litellm_provider": "openrouter", "max_input_tokens": 33792, "max_output_tokens": 33792, "max_tokens": 33792, "mode": "chat", - "output_cost_per_token": 1.8e-07, + "output_cost_per_token": 1e-06, "supports_tool_choice": true }, "openrouter/qwen/qwen-vl-plus": { @@ -38702,50 +38703,50 @@ "supports_vision": true }, "openrouter/qwen/qwen3-coder": { - "input_cost_per_token": 2.2e-07, + "input_cost_per_token": 3e-07, "litellm_provider": "openrouter", "max_input_tokens": 262100, "max_output_tokens": 262100, "max_tokens": 262100, "mode": "chat", - "output_cost_per_token": 9.5e-07, + "output_cost_per_token": 1e-06, "source": "https://openrouter.ai/qwen/qwen3-coder", "supports_tool_choice": true, "supports_function_calling": true }, "openrouter/qwen/qwen3-coder-plus": { - "input_cost_per_token": 1e-06, + "input_cost_per_token": 6.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 997952, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 5e-06, + "output_cost_per_token": 3.25e-06, "source": "https://openrouter.ai/qwen/qwen3-coder-plus", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true }, "openrouter/qwen/qwen3-235b-a22b-2507": { - "input_cost_per_token": 7.1e-08, + "input_cost_per_token": 8.75e-08, "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 262144, "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 1e-07, + "output_cost_per_token": 3.5e-07, "source": "https://openrouter.ai/qwen/qwen3-235b-a22b-2507", "supports_function_calling": true, "supports_tool_choice": true }, "openrouter/qwen/qwen3-235b-a22b-thinking-2507": { - "input_cost_per_token": 1.1e-07, + "input_cost_per_token": 2.3e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 262144, "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 6e-07, + "output_cost_per_token": 2.3e-06, "source": "https://openrouter.ai/qwen/qwen3-235b-a22b-thinking-2507", "supports_function_calling": true, "supports_reasoning": true, @@ -38772,7 +38773,7 @@ "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 2e-06, + "output_cost_per_token": 1.25e-06, "source": "https://openrouter.ai/qwen/qwen3.5-35b-a3b", "supports_function_calling": true, "supports_reasoning": true, @@ -38780,13 +38781,13 @@ "supports_vision": true }, "openrouter/qwen/qwen3.5-27b": { - "input_cost_per_token": 3e-07, + "input_cost_per_token": 1.95e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 2.4e-06, + "output_cost_per_token": 1.56e-06, "source": "https://openrouter.ai/qwen/qwen3.5-27b", "supports_function_calling": true, "supports_reasoning": true, @@ -38794,13 +38795,13 @@ "supports_vision": true }, "openrouter/qwen/qwen3.5-122b-a10b": { - "input_cost_per_token": 4e-07, + "input_cost_per_token": 2.9e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 2e-06, + "output_cost_per_token": 2.4e-06, "source": "https://openrouter.ai/qwen/qwen3.5-122b-a10b", "supports_function_calling": true, "supports_reasoning": true, @@ -38808,13 +38809,13 @@ "supports_vision": true }, "openrouter/qwen/qwen3.5-flash-02-23": { - "input_cost_per_token": 1e-07, + "input_cost_per_token": 6.5e-08, "litellm_provider": "openrouter", "max_input_tokens": 1000000, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 4e-07, + "output_cost_per_token": 2.6e-07, "source": "https://openrouter.ai/qwen/qwen3.5-flash-02-23", "supports_function_calling": true, "supports_reasoning": true, @@ -38822,14 +38823,14 @@ "supports_vision": true }, "openrouter/qwen/qwen3.5-plus-02-15": { - "input_cost_per_token": 4e-07, + "input_cost_per_token": 2.6e-07, "input_cost_per_token_above_256k_tokens": 5e-07, "litellm_provider": "openrouter", "max_input_tokens": 1000000, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 2.4e-06, + "output_cost_per_token": 1.56e-06, "output_cost_per_token_above_256k_tokens": 3e-06, "source": "https://openrouter.ai/qwen/qwen3.5-plus-02-15", "supports_function_calling": true, @@ -38838,13 +38839,13 @@ "supports_vision": true }, "openrouter/qwen/qwen3.5-397b-a17b": { - "input_cost_per_token": 6e-07, + "input_cost_per_token": 5.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 3.6e-06, + "output_cost_per_token": 3.5e-06, "source": "https://openrouter.ai/qwen/qwen3.5-397b-a17b", "supports_function_calling": true, "supports_reasoning": true, @@ -38863,11 +38864,11 @@ "supports_tool_choice": true }, "openrouter/undi95/remm-slerp-l2-13b": { - "input_cost_per_token": 1.875e-06, + "input_cost_per_token": 4.5e-07, "litellm_provider": "openrouter", "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 1.875e-06, + "output_cost_per_token": 6.5e-07, "supports_tool_choice": true, "max_input_tokens": 6144, "max_output_tokens": 4096 @@ -38887,13 +38888,13 @@ "supports_web_search": true }, "openrouter/z-ai/glm-4.6": { - "input_cost_per_token": 4e-07, + "input_cost_per_token": 5.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 202800, "max_output_tokens": 131000, "max_tokens": 131000, "mode": "chat", - "output_cost_per_token": 1.75e-06, + "output_cost_per_token": 2.2e-06, "source": "https://openrouter.ai/z-ai/glm-4.6", "supports_function_calling": true, "supports_prompt_caching": true, @@ -38931,10 +38932,10 @@ "supports_prompt_caching": true }, "openrouter/xiaomi/mimo-v2.5-pro": { - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3e-06, + "input_cost_per_token": 4.35e-07, + "output_cost_per_token": 8.7e-07, "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost": 3.6e-09, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 16384, @@ -38948,10 +38949,10 @@ "supports_prompt_caching": true }, "openrouter/xiaomi/mimo-v2.5": { - "input_cost_per_token": 4e-07, - "output_cost_per_token": 2e-06, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 8e-08, + "cache_read_input_token_cost": 2.8e-09, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 131072, @@ -38968,9 +38969,9 @@ }, "openrouter/z-ai/glm-4.7": { "input_cost_per_token": 4e-07, - "output_cost_per_token": 1.5e-06, + "output_cost_per_token": 1.75e-06, "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 0.0, + "cache_read_input_token_cost": 8e-08, "litellm_provider": "openrouter", "max_input_tokens": 202752, "max_output_tokens": 64000, @@ -38984,10 +38985,10 @@ "supports_assistant_prefill": true }, "openrouter/z-ai/glm-4.7-flash": { - "input_cost_per_token": 7e-08, + "input_cost_per_token": 6e-08, "output_cost_per_token": 4e-07, "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 0.0, + "cache_read_input_token_cost": 1e-08, "litellm_provider": "openrouter", "max_input_tokens": 200000, "max_output_tokens": 32000, @@ -39000,22 +39001,22 @@ "supports_prompt_caching": false }, "openrouter/z-ai/glm-5": { - "input_cost_per_token": 8e-07, + "input_cost_per_token": 6e-07, "litellm_provider": "openrouter", "max_input_tokens": 202752, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 2.56e-06, + "output_cost_per_token": 1.92e-06, "source": "https://openrouter.ai/z-ai/glm-5", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true }, "openrouter/z-ai/glm-5.1": { - "input_cost_per_token": 1.05e-06, - "output_cost_per_token": 3.5e-06, - "cache_read_input_token_cost": 5.25e-07, + "input_cost_per_token": 9.66e-07, + "output_cost_per_token": 3.036e-06, + "cache_read_input_token_cost": 1.794e-07, "cache_creation_input_token_cost": 0.0, "litellm_provider": "openrouter", "max_input_tokens": 202752, @@ -39029,10 +39030,10 @@ "supports_tool_choice": true }, "openrouter/minimax/minimax-m2.1": { - "input_cost_per_token": 2.7e-07, + "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 0.0, + "cache_read_input_token_cost": 3e-08, "litellm_provider": "openrouter", "max_input_tokens": 204000, "max_output_tokens": 64000, @@ -39046,9 +39047,9 @@ "supports_computer_use": false }, "openrouter/minimax/minimax-m2.5": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.1e-06, - "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 1.08e-06, + "cache_read_input_token_cost": 2.7e-08, "litellm_provider": "openrouter", "max_input_tokens": 196608, "max_output_tokens": 65536, @@ -39947,7 +39948,7 @@ "supports_reasoning": true }, "qwen.qwen3-coder-480b-a35b-v1:0": { - "input_cost_per_token": 2.2e-07, + "input_cost_per_token": 4.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 262000, "max_output_tokens": 65536, @@ -39957,7 +39958,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json" }, "qwen.qwen3-235b-a22b-2507-v1:0": { "input_cost_per_token": 2.2e-07, @@ -52852,7 +52854,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -52878,7 +52880,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -52904,7 +52906,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -52932,7 +52934,7 @@ "max_input_tokens": 131072, "max_output_tokens": 65536, "max_tokens": 65536, - "mode": "chat", + "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -52963,7 +52965,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -52991,7 +52993,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -53019,7 +53021,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -53049,7 +53051,7 @@ "max_input_tokens": 131072, "max_output_tokens": 65536, "max_tokens": 65536, - "mode": "chat", + "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -59463,7 +59465,8 @@ "supports_audio_input": false, "cache_read_input_token_cost": 1e-06, "supports_prompt_caching": true, - "cache_creation_input_token_cost": 1.25e-05 + "cache_creation_input_token_cost": 1.25e-05, + "prompt_cache_min_tokens": 512 }, "openrouter/anthropic/claude-fable-5.1": { "input_cost_per_token": 1e-05, @@ -59483,7 +59486,8 @@ "supports_audio_input": false, "cache_read_input_token_cost": 2.5e-07, "supports_prompt_caching": true, - "cache_creation_input_token_cost": 1.25e-05 + "cache_creation_input_token_cost": 1.25e-05, + "prompt_cache_min_tokens": 512 }, "openrouter/anthropic/claude-opus-4.8": { "input_cost_per_token": 5e-06, @@ -59549,8 +59553,8 @@ "output_cost_per_token": 9e-06, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_output_tokens": 65535, + "max_tokens": 65535, "mode": "chat", "source": "https://openrouter.ai/google/gemini-3.5-flash", "supports_function_calling": true, @@ -59662,7 +59666,7 @@ "input_cost_per_token": 1.25e-06, "output_cost_per_token": 1e-05, "litellm_provider": "openrouter", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -59681,7 +59685,7 @@ "input_cost_per_token": 1.75e-06, "output_cost_per_token": 1.4e-05, "litellm_provider": "openrouter", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -59719,7 +59723,7 @@ "input_cost_per_token": 7.5e-07, "output_cost_per_token": 4.5e-06, "litellm_provider": "openrouter", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -59738,7 +59742,7 @@ "input_cost_per_token": 2e-07, "output_cost_per_token": 1.25e-06, "litellm_provider": "openrouter", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -59776,7 +59780,7 @@ "input_cost_per_token": 2e-07, "output_cost_per_token": 1.2e-06, "litellm_provider": "openrouter", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -59795,7 +59799,7 @@ "input_cost_per_token": 2e-06, "output_cost_per_token": 1.2e-05, "litellm_provider": "openrouter", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -59852,9 +59856,9 @@ "input_cost_per_token": 1.25e-06, "output_cost_per_token": 2.5e-06, "litellm_provider": "openrouter", - "max_input_tokens": 2000000, - "max_output_tokens": 1800000, - "max_tokens": 1800000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", "source": "https://openrouter.ai/x-ai/grok-4.20", "supports_function_calling": true, @@ -59871,9 +59875,9 @@ "input_cost_per_token": 1.25e-06, "output_cost_per_token": 2.5e-06, "litellm_provider": "openrouter", - "max_input_tokens": 2000000, - "max_output_tokens": 1800000, - "max_tokens": 1800000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", "source": "https://openrouter.ai/x-ai/grok-4.20-multi-agent", "supports_function_calling": false, @@ -59891,8 +59895,8 @@ "output_cost_per_token": 2.5e-06, "litellm_provider": "openrouter", "max_input_tokens": 1000000, - "max_output_tokens": 900000, - "max_tokens": 900000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", "source": "https://openrouter.ai/x-ai/grok-4.3", "supports_function_calling": true, @@ -59910,8 +59914,8 @@ "output_cost_per_token": 6e-06, "litellm_provider": "openrouter", "max_input_tokens": 500000, - "max_output_tokens": 450000, - "max_tokens": 450000, + "max_output_tokens": 500000, + "max_tokens": 500000, "mode": "chat", "source": "https://openrouter.ai/x-ai/grok-4.5", "supports_function_calling": true, @@ -59929,8 +59933,8 @@ "output_cost_per_token": 6e-06, "litellm_provider": "openrouter", "max_input_tokens": 500000, - "max_output_tokens": 450000, - "max_tokens": 450000, + "max_output_tokens": 500000, + "max_tokens": 500000, "mode": "chat", "source": "https://openrouter.ai/x-ai/grok-4.6", "supports_function_calling": true, @@ -59948,8 +59952,8 @@ "output_cost_per_token": 2e-06, "litellm_provider": "openrouter", "max_input_tokens": 256000, - "max_output_tokens": 230400, - "max_tokens": 230400, + "max_output_tokens": 256000, + "max_tokens": 256000, "mode": "chat", "source": "https://openrouter.ai/x-ai/grok-build-0.1", "supports_function_calling": true, @@ -59961,5 +59965,26 @@ "supports_audio_input": false, "cache_read_input_token_cost": 2e-07, "supports_prompt_caching": true + }, + "baseten/zai-org/GLM-5.3": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://www.baseten.co/pricing/", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true } } diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index a374e03d1c7..18185126775 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -9,7 +9,6 @@ import os import pytest - from litellm.litellm_core_utils.fallback_generalizations import ( get_fallback_generalization_rules, match_capability_generalizations, @@ -248,6 +247,57 @@ def test_azure_ai_claude_1m_context_entries(cost_map: dict): assert cost_map[model]["max_input_tokens"] == 200000, model +# OpenRouter headline rates from GET https://openrouter.ai/api/v1/models. +# These were the catalog values that disagreed with that API (and, for the +# two spotlight models, the public model pages that their source fields cite). +_OPENROUTER_LIVE_COSTS = { + "openrouter/qwen/qwen3.5-plus-02-15": (2.6e-07, 1.56e-06, None), + "openrouter/openai/gpt-oss-120b": (3.7e-08, 1.7e-07, None), + "openrouter/qwen/qwen3-coder-plus": (6.5e-07, 3.25e-06, None), + "openrouter/qwen/qwen3.5-flash-02-23": (6.5e-08, 2.6e-07, None), + "openrouter/qwen/qwen3.5-27b": (1.95e-07, 1.56e-06, None), + "openrouter/gryphe/mythomax-l2-13b": (6e-08, 6e-08, None), + "openrouter/mancer/weaver": (4e-07, 7.5e-07, None), + "openrouter/xiaomi/mimo-v2.5-pro": (4.35e-07, 8.7e-07, 3.6e-09), + "openrouter/moonshotai/kimi-k2.5": (4.5e-07, 2.25e-06, 7e-08), + "openrouter/z-ai/glm-5": (6e-07, 1.92e-06, None), +} + +_OPENROUTER_STALE_COSTS = { + "openrouter/qwen/qwen3.5-plus-02-15": (4e-07, 2.4e-06), + "openrouter/openai/gpt-oss-120b": (1.8e-07, 8e-07), + "openrouter/gryphe/mythomax-l2-13b": (1.875e-06, 1.875e-06), +} + + +@pytest.mark.parametrize( + "cost_map", + [_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()], + ids=["root", "bundled_backup"], +) +def test_openrouter_catalog_costs_match_live_headline_rates(cost_map: dict): + """openrouter/* spend tracking reads these catalog fields. The values must + stay aligned with OpenRouter's published headline rate, not the stale + figures that over/under-counted by up to 30x. Both maps are checked so + the root file and bundled backup cannot drift apart.""" + control = cost_map["openrouter/anthropic/claude-opus-5"] + assert control["input_cost_per_token"] == 5e-06 + assert control["output_cost_per_token"] == 2.5e-05 + assert control["cache_read_input_token_cost"] == 5e-07 + + for model, (inp, out, cache) in _OPENROUTER_LIVE_COSTS.items(): + entry = cost_map[model] + assert entry["input_cost_per_token"] == inp, model + assert entry["output_cost_per_token"] == out, model + if cache is not None: + assert entry["cache_read_input_token_cost"] == cache, model + + for model, (stale_in, stale_out) in _OPENROUTER_STALE_COSTS.items(): + entry = cost_map[model] + assert entry["input_cost_per_token"] != stale_in, model + assert entry["output_cost_per_token"] != stale_out, model + + def test_get_model_cost_map_stamps_loaded_at(monkeypatch): """The load time feeds each pod's reload-due decision; a load that does not stamp it would make manual reload requests race the proxy's startup""" diff --git a/tests/test_litellm/test_baseten_glm_5_3_model_metadata.py b/tests/test_litellm/test_baseten_glm_5_3_model_metadata.py new file mode 100644 index 00000000000..98a8cf026ec --- /dev/null +++ b/tests/test_litellm/test_baseten_glm_5_3_model_metadata.py @@ -0,0 +1,154 @@ +import json +from pathlib import Path + +import pytest + +import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.types.utils import PromptTokensDetailsWrapper, Usage +from litellm.utils import supports_function_calling, supports_prompt_caching + +REPO_ROOT = Path(__file__).parents[2] +MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" + +MODEL = "baseten/zai-org/GLM-5.3" + +INPUT_COST = 1.4e-06 +CACHED_INPUT_COST = 1.4e-07 +OUTPUT_COST = 4.4e-06 + + +def _load(path): + with open(path) as f: + return json.load(f) + + +@pytest.fixture +def local_model_cost_map(monkeypatch): + """Force get_model_info to resolve against the in-repo cost map instead of the + remote one fetched at import time, which still carries the pre-merge registry.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +def test_baseten_glm_5_3_specs(): + info = _load(MAIN_PATH).get(MODEL) + assert info is not None, f"{MODEL} missing from model_prices_and_context_window.json" + + assert info["litellm_provider"] == "baseten" + assert info["mode"] == "chat" + + assert info["input_cost_per_token"] == INPUT_COST + assert info["output_cost_per_token"] == OUTPUT_COST + assert info["cache_read_input_token_cost"] == CACHED_INPUT_COST + + assert info["max_input_tokens"] == 1048576 + assert info["max_output_tokens"] == 262144 + assert info["max_tokens"] == 262144 + + assert info["supports_function_calling"] is True + assert info["supports_prompt_caching"] is True + assert info["supports_response_schema"] is True + assert info["supports_tool_choice"] is True + assert info["supported_modalities"] == ["text"] + assert info["supported_output_modalities"] == ["text"] + + routed_model, provider, _, _ = get_llm_provider(model=MODEL) + assert routed_model == "zai-org/GLM-5.3" + assert provider == "baseten" + + +def test_baseten_glm_5_3_capabilities_are_visible_to_callers(local_model_cost_map): + """The entry advertises prompt caching and tool calling, so the helpers every + caller checks before sending a request must say so too.""" + assert supports_prompt_caching(model=MODEL) is True + assert supports_function_calling(model=MODEL) is True + + info = litellm.get_model_info(model="zai-org/GLM-5.3", custom_llm_provider="baseten") + assert info["max_input_tokens"] == 1048576 + assert info["max_output_tokens"] == 262144 + + +def test_cached_prompt_tokens_bill_at_the_cached_rate(local_model_cost_map): + """A cache hit reports its reused tokens under prompt_tokens_details, and those + tokens cost a tenth of the input rate, not the full rate and not nothing.""" + usage = Usage( + prompt_tokens=21010, + completion_tokens=100, + total_tokens=21110, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=20992), + ) + + prompt_cost, completion_cost = litellm.cost_per_token( + model=MODEL, usage_object=usage, custom_llm_provider="baseten" + ) + + assert prompt_cost == pytest.approx(18 * INPUT_COST + 20992 * CACHED_INPUT_COST) + assert completion_cost == pytest.approx(100 * OUTPUT_COST) + + +def test_backup_matches_main(): + """Ensure the bundled (backup) cost map stays in sync with the canonical file. + + Both keys are asserted present first: comparing two ``.get`` results alone passes + just as happily when neither file has the entry at all, which is the exact state + this test exists to catch. + """ + main_cost = _load(MAIN_PATH) + backup_cost = _load(BACKUP_PATH) + + assert MODEL in main_cost, f"{MODEL} missing from model_prices_and_context_window.json" + assert MODEL in backup_cost, f"{MODEL} missing from model_prices_and_context_window_backup.json" + assert backup_cost[MODEL] == main_cost[MODEL], f"{MODEL} differs between main and backup model cost maps" + + +def test_entry_advertises_only_what_the_baseten_path_accepts(local_model_cost_map): + """The entry must not claim a capability whose request parameter BasetenConfig + refuses. + + ``BasetenConfig.get_supported_openai_params`` returns one hardcoded list for every + Baseten model, and it carries neither ``parallel_tool_calls`` nor + ``reasoning_effort``. Baseten's own Model API does take ``reasoning_effort``, but + litellm's Baseten path drops it (``drop_params=True``) or raises + ``UnsupportedParamsError`` (``drop_params=False``), so declaring + ``supports_parallel_function_calling``, ``supports_reasoning`` or + ``reasoning_effort_levels`` here would advertise a level the gateway then refuses to + send. Wiring those params through the Baseten config is separate work; until it + lands, the registry stays honest. + """ + supported = litellm.get_supported_openai_params(model="zai-org/GLM-5.3", custom_llm_provider="baseten") + assert supported is not None + + entry = _load(MAIN_PATH)[MODEL] + + capability_to_param = { + "supports_function_calling": "tools", + "supports_tool_choice": "tool_choice", + "supports_response_schema": "response_format", + "supports_parallel_function_calling": "parallel_tool_calls", + "supports_reasoning": "reasoning_effort", + } + for capability, param in capability_to_param.items(): + if entry.get(capability): + assert param in supported, f"{MODEL} advertises {capability} but baseten drops/rejects {param}" + + assert "reasoning_effort_levels" not in entry, ( + "reasoning_effort_levels advertises accepted reasoning_effort values, which the Baseten path does not accept" + ) + assert "thinking_always_on" not in entry, ( + "thinking_always_on is only read by AnthropicModelInfo._is_always_on_thinking_model, " + "which no Baseten route reaches" + ) + + with pytest.raises(litellm.UnsupportedParamsError): + litellm.utils.get_optional_params( + model="zai-org/GLM-5.3", + custom_llm_provider="baseten", + parallel_tool_calls=True, + reasoning_effort="high", + drop_params=False, + ) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 99c69d6bc31..5575c4328a2 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -2779,7 +2779,7 @@ def test_model_info_for_openrouter_kimi_k2_5(): Model properties from OpenRouter API: - context_length: 262144 - - pricing: prompt=$0.0000006, completion=$0.000003, input_cache_read=$0.0000001 + - pricing: prompt=$0.00000045, completion=$0.00000225, input_cache_read=$0.00000007 - modality: text+image->text (supports vision) - supports: tool_choice, tools (function calling) """ @@ -2804,9 +2804,9 @@ def test_model_info_for_openrouter_kimi_k2_5(): assert model_info["max_tokens"] == 262144 # Verify pricing - assert model_info["input_cost_per_token"] == 6e-07 - assert model_info["output_cost_per_token"] == 3e-06 - assert model_info["cache_read_input_token_cost"] == 1e-07 + assert model_info["input_cost_per_token"] == 4.5e-07 + assert model_info["output_cost_per_token"] == 2.25e-06 + assert model_info["cache_read_input_token_cost"] == 7e-08 # Verify capabilities assert model_info["supports_vision"] is True From 080e364d5ec1248a0d045af13732024c1654d642 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 3 Sep 2026 20:28:48 +0000 Subject: [PATCH 33/88] fix(registry): carry Anthropic thinking/sampling flags on new Perplexity and OpenRouter Claude entries Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../model_prices_and_context_window_backup.json | 16 ++++++++++++++++ model_prices_and_context_window.json | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a6f3095e974..260a9c3a34e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -59130,6 +59130,8 @@ "perplexity/anthropic/claude-fable-5": { "litellm_provider": "perplexity", "mode": "responses", + "supports_adaptive_thinking": true, + "thinking_always_on": true, "supports_web_search": true, "supports_function_calling": true, "input_cost_per_token": 1e-05, @@ -59140,6 +59142,8 @@ "perplexity/anthropic/claude-opus-5": { "litellm_provider": "perplexity", "mode": "responses", + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 512, "supports_web_search": true, "supports_function_calling": true, "input_cost_per_token": 5e-06, @@ -59150,6 +59154,7 @@ "perplexity/anthropic/claude-opus-4-8": { "litellm_provider": "perplexity", "mode": "responses", + "supports_adaptive_thinking": true, "supports_web_search": true, "supports_function_calling": true, "input_cost_per_token": 5e-06, @@ -59160,6 +59165,7 @@ "perplexity/anthropic/claude-sonnet-5": { "litellm_provider": "perplexity", "mode": "responses", + "supports_adaptive_thinking": true, "supports_web_search": true, "supports_function_calling": true, "input_cost_per_token": 2e-06, @@ -59455,6 +59461,9 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "supports_sampling_params": false, + "supports_adaptive_thinking": true, + "thinking_always_on": true, "source": "https://openrouter.ai/anthropic/claude-fable-5", "supports_function_calling": true, "supports_tool_choice": true, @@ -59476,6 +59485,9 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "supports_sampling_params": false, + "supports_adaptive_thinking": true, + "thinking_always_on": true, "source": "https://openrouter.ai/anthropic/claude-fable-5.1", "supports_function_calling": true, "supports_tool_choice": false, @@ -59497,6 +59509,8 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "supports_sampling_params": false, + "supports_adaptive_thinking": true, "source": "https://openrouter.ai/anthropic/claude-opus-4.8", "supports_function_calling": true, "supports_tool_choice": true, @@ -59517,6 +59531,8 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "supports_sampling_params": false, + "supports_adaptive_thinking": true, "source": "https://openrouter.ai/anthropic/claude-sonnet-5", "supports_function_calling": true, "supports_tool_choice": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a6f3095e974..260a9c3a34e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -59130,6 +59130,8 @@ "perplexity/anthropic/claude-fable-5": { "litellm_provider": "perplexity", "mode": "responses", + "supports_adaptive_thinking": true, + "thinking_always_on": true, "supports_web_search": true, "supports_function_calling": true, "input_cost_per_token": 1e-05, @@ -59140,6 +59142,8 @@ "perplexity/anthropic/claude-opus-5": { "litellm_provider": "perplexity", "mode": "responses", + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 512, "supports_web_search": true, "supports_function_calling": true, "input_cost_per_token": 5e-06, @@ -59150,6 +59154,7 @@ "perplexity/anthropic/claude-opus-4-8": { "litellm_provider": "perplexity", "mode": "responses", + "supports_adaptive_thinking": true, "supports_web_search": true, "supports_function_calling": true, "input_cost_per_token": 5e-06, @@ -59160,6 +59165,7 @@ "perplexity/anthropic/claude-sonnet-5": { "litellm_provider": "perplexity", "mode": "responses", + "supports_adaptive_thinking": true, "supports_web_search": true, "supports_function_calling": true, "input_cost_per_token": 2e-06, @@ -59455,6 +59461,9 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "supports_sampling_params": false, + "supports_adaptive_thinking": true, + "thinking_always_on": true, "source": "https://openrouter.ai/anthropic/claude-fable-5", "supports_function_calling": true, "supports_tool_choice": true, @@ -59476,6 +59485,9 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "supports_sampling_params": false, + "supports_adaptive_thinking": true, + "thinking_always_on": true, "source": "https://openrouter.ai/anthropic/claude-fable-5.1", "supports_function_calling": true, "supports_tool_choice": false, @@ -59497,6 +59509,8 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "supports_sampling_params": false, + "supports_adaptive_thinking": true, "source": "https://openrouter.ai/anthropic/claude-opus-4.8", "supports_function_calling": true, "supports_tool_choice": true, @@ -59517,6 +59531,8 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "supports_sampling_params": false, + "supports_adaptive_thinking": true, "source": "https://openrouter.ai/anthropic/claude-sonnet-5", "supports_function_calling": true, "supports_tool_choice": true, From d0ac49414432522192b80faac1eaffd6c9b49197 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:31:56 -0700 Subject: [PATCH 34/88] fix(cost): honor off_peak_pricing in the fireworks_ai and perplexity cost calculators --- .../litellm_core_utils/llm_cost_calc/utils.py | 4 +- litellm/llms/fireworks_ai/cost_calculator.py | 50 +++++----- litellm/llms/perplexity/cost_calculator.py | 17 +++- .../test_fireworks_ai_cost_calculator.py | 75 +++++++++++++++ .../test_perplexity_cost_calculator.py | 92 +++++++++++++++++++ 5 files changed, 209 insertions(+), 29 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index b34c416cd40..21587af73aa 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -428,7 +428,7 @@ def _coerce_off_peak_rate(value: object, default: float) -> float: return default -def _apply_off_peak_pricing( +def apply_off_peak_pricing( model_info: ModelInfo, current_time: datetime | None, prompt_base_cost: float, @@ -462,7 +462,7 @@ def _apply_off_peak_to_base_costs( has no field for them. """ prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs - off_peak_prompt, off_peak_completion, off_peak_cache_read = _apply_off_peak_pricing( + off_peak_prompt, off_peak_completion, off_peak_cache_read = apply_off_peak_pricing( model_info, current_time, prompt, completion, cache_read ) return (off_peak_prompt, off_peak_completion, cache_creation, cache_creation_above_1hr, off_peak_cache_read) diff --git a/litellm/llms/fireworks_ai/cost_calculator.py b/litellm/llms/fireworks_ai/cost_calculator.py index 08e6f009010..df47d3546ca 100644 --- a/litellm/llms/fireworks_ai/cost_calculator.py +++ b/litellm/llms/fireworks_ai/cost_calculator.py @@ -2,6 +2,7 @@ For calculating cost of fireworks ai serverless inference models. """ +from datetime import datetime from typing import Final from litellm.constants import ( @@ -10,7 +11,8 @@ from litellm.constants import ( FIREWORKS_AI_56_B_MOE, FIREWORKS_AI_176_B_MOE, ) -from litellm.types.utils import Usage +from litellm.litellm_core_utils.llm_cost_calc.utils import apply_off_peak_pricing +from litellm.types.utils import ModelInfo, Usage from litellm.utils import get_model_info @@ -54,44 +56,46 @@ def get_base_model_for_pricing(model_name: str) -> str: return "fireworks-ai-default" -def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: +def _resolve_model_info(model: str) -> ModelInfo: + try: + return get_model_info(model=model, custom_llm_provider="fireworks_ai") + except Exception: + base_model: Final = get_base_model_for_pricing(model_name=model) + return get_model_info(model=base_model, custom_llm_provider="fireworks_ai") + + +def cost_per_token(model: str, usage: Usage, current_time: datetime | None = None) -> tuple[float, float]: """ - Calculates the cost per token for a given model, prompt tokens, and completion tokens. + Calculates the cost per token for a given model, prompt tokens, and completion tokens, + swapping in the model's off_peak_pricing rates while one of its windows is open. Input: - model: str, the model name without provider prefix - usage: LiteLLM Usage block, containing anthropic caching information + - current_time: the moment the request is billed at; defaults to now, UTC Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - ## check if model mapped, else use default pricing - try: - model_info = get_model_info(model=model, custom_llm_provider="fireworks_ai") - except Exception: - base_model: Final = get_base_model_for_pricing(model_name=model) + model_info: Final = _resolve_model_info(model) + standard_input_rate: Final[float] = model_info["input_cost_per_token"] or 0.0 + standard_cache_read_rate: Final = model_info.get("cache_read_input_token_cost") + input_rate, output_rate, cache_read_rate = apply_off_peak_pricing( + model_info, + current_time, + standard_input_rate, + model_info["output_cost_per_token"] or 0.0, + standard_cache_read_rate if standard_cache_read_rate is not None else standard_input_rate, + ) - ## GET MODEL INFO - model_info = get_model_info(model=base_model, custom_llm_provider="fireworks_ai") - - ## CALCULATE INPUT COST prompt_tokens_details: Final = usage.prompt_tokens_details cached_tokens: Final[int] = ( prompt_tokens_details.cached_tokens if prompt_tokens_details is not None and prompt_tokens_details.cached_tokens is not None else 0 ) - input_cost_per_token: Final[float] = model_info["input_cost_per_token"] or 0.0 - cache_read_input_token_cost: Final = model_info.get("cache_read_input_token_cost") - cache_read_cost_per_token: Final[float] = ( - cache_read_input_token_cost if cache_read_input_token_cost is not None else input_cost_per_token - ) non_cached_prompt_tokens: Final[int] = max(usage.prompt_tokens - cached_tokens, 0) - - prompt_cost: float = non_cached_prompt_tokens * input_cost_per_token + cached_tokens * cache_read_cost_per_token - - ## CALCULATE OUTPUT COST - output_cost_per_token: Final[float] = model_info["output_cost_per_token"] or 0.0 - completion_cost: Final[float] = usage.completion_tokens * output_cost_per_token + prompt_cost: Final[float] = non_cached_prompt_tokens * input_rate + cached_tokens * cache_read_rate + completion_cost: Final[float] = usage.completion_tokens * output_rate return prompt_cost, completion_cost diff --git a/litellm/llms/perplexity/cost_calculator.py b/litellm/llms/perplexity/cost_calculator.py index 27835ecbfe8..67949e850f0 100644 --- a/litellm/llms/perplexity/cost_calculator.py +++ b/litellm/llms/perplexity/cost_calculator.py @@ -3,19 +3,23 @@ Helper util for handling perplexity-specific cost calculation - e.g.: citation tokens, search queries """ +from datetime import datetime from typing import Final +from litellm.litellm_core_utils.llm_cost_calc.utils import apply_off_peak_pricing from litellm.types.utils import Usage from litellm.utils import get_model_info -def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: +def cost_per_token(model: str, usage: Usage, current_time: datetime | None = None) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. + The manual fallback swaps in the model's off_peak_pricing rates while one of its windows is open. Input: - model: str, the model name without provider prefix - usage: LiteLLM Usage block, containing perplexity-specific usage information + - current_time: the moment the request is billed at; defaults to now, UTC Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd @@ -48,8 +52,15 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: except (ValueError, TypeError): return default + input_cost_per_token, output_cost_per_token, _ = apply_off_peak_pricing( + model_info, + current_time, + _safe_float_cast(model_info.get("input_cost_per_token")), + _safe_float_cast(model_info.get("output_cost_per_token")), + 0.0, + ) + ## CALCULATE INPUT COST - input_cost_per_token: Final = _safe_float_cast(model_info.get("input_cost_per_token")) prompt_cost: float = (usage.prompt_tokens or 0) * input_cost_per_token ## ADD CITATION TOKENS COST (if present) @@ -60,8 +71,6 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: prompt_cost += citation_tokens * citation_cost_per_token ## CALCULATE OUTPUT COST - output_cost_per_token: Final = _safe_float_cast(model_info.get("output_cost_per_token")) - reasoning_tokens = getattr(usage, "reasoning_tokens", 0) or 0 if reasoning_tokens == 0 and hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details: reasoning_tokens = getattr(usage.completion_tokens_details, "reasoning_tokens", 0) or 0 diff --git a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py index f1664dabf48..21ee56a7873 100644 --- a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py +++ b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py @@ -1,4 +1,7 @@ +import math +from datetime import datetime, timezone + import pytest @@ -64,3 +67,75 @@ def test_no_cached_tokens_matches_full_input_rate(): assert prompt_cost == pytest.approx(prompt_tokens * INPUT_COST) assert completion_cost == pytest.approx(completion_tokens * OUTPUT_COST) + + +OFF_PEAK_MODEL = "accounts/fireworks/models/off-peak-test" +OFF_PEAK_WINDOW = "14:00-00:00" +INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc) +OUTSIDE_WINDOW = datetime(2026, 9, 3, 9, 0, tzinfo=timezone.utc) +STANDARD_INPUT_COST = 1.5e-07 +STANDARD_OUTPUT_COST = 6e-07 +STANDARD_CACHE_READ_COST = 1.5e-08 + + +def _register_off_peak_model(off_peak_pricing: dict) -> None: + litellm.model_cost[f"fireworks_ai/{OFF_PEAK_MODEL}"] = { + "litellm_provider": "fireworks_ai", + "mode": "chat", + "input_cost_per_token": STANDARD_INPUT_COST, + "output_cost_per_token": STANDARD_OUTPUT_COST, + "cache_read_input_token_cost": STANDARD_CACHE_READ_COST, + "off_peak_pricing": off_peak_pricing, + } + + +def test_off_peak_window_swaps_in_the_off_peak_rates(): + """ + Regression (LIT-6874): a deployment configured with off_peak_pricing kept billing the + standard fireworks_ai rates inside its window, while the same block on a deepseek + deployment billed the off-peak rates. + """ + _register_off_peak_model( + { + "hours_utc": OFF_PEAK_WINDOW, + "input_cost_per_token": 1e-08, + "output_cost_per_token": 2e-08, + "cache_read_input_token_cost": 1e-09, + } + ) + usage = _usage(prompt_tokens=1000, cached_tokens=300, completion_tokens=200) + + prompt_cost, completion_cost = cost_per_token(model=OFF_PEAK_MODEL, usage=usage, current_time=INSIDE_WINDOW) + + assert math.isclose(prompt_cost, (700 * 1e-08) + (300 * 1e-09), rel_tol=1e-10) + assert math.isclose(completion_cost, 200 * 2e-08, rel_tol=1e-10) + + peak_prompt_cost, peak_completion_cost = cost_per_token( + model=OFF_PEAK_MODEL, usage=usage, current_time=OUTSIDE_WINDOW + ) + + assert math.isclose(peak_prompt_cost, (700 * STANDARD_INPUT_COST) + (300 * STANDARD_CACHE_READ_COST), rel_tol=1e-10) + assert math.isclose(peak_completion_cost, 200 * STANDARD_OUTPUT_COST, rel_tol=1e-10) + + +def test_off_peak_rates_left_unset_keep_the_standard_rates(): + """A block that only overrides the input rate leaves output and cache reads on the standard rates.""" + _register_off_peak_model({"hours_utc": OFF_PEAK_WINDOW, "input_cost_per_token": 1e-08}) + usage = _usage(prompt_tokens=1000, cached_tokens=300, completion_tokens=200) + + prompt_cost, completion_cost = cost_per_token(model=OFF_PEAK_MODEL, usage=usage, current_time=INSIDE_WINDOW) + + assert math.isclose(prompt_cost, (700 * 1e-08) + (300 * STANDARD_CACHE_READ_COST), rel_tol=1e-10) + assert math.isclose(completion_cost, 200 * STANDARD_OUTPUT_COST, rel_tol=1e-10) + + +def test_off_peak_defaults_to_the_current_time(): + """The proxy's cost dispatch passes no clock, so an all-day window has to apply on the + default current time.""" + _register_off_peak_model({"hours_utc": "00:00-00:00", "input_cost_per_token": 1e-08, "output_cost_per_token": 2e-08}) + usage = _usage(prompt_tokens=1000, cached_tokens=0, completion_tokens=200) + + prompt_cost, completion_cost = cost_per_token(model=OFF_PEAK_MODEL, usage=usage) + + assert math.isclose(prompt_cost, 1000 * 1e-08, rel_tol=1e-10) + assert math.isclose(completion_cost, 200 * 2e-08, rel_tol=1e-10) diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py index 117379c331a..be338bd3dfa 100644 --- a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py @@ -8,6 +8,7 @@ search queries, and reasoning tokens. import json import math import os +from datetime import datetime, timezone from unittest.mock import patch import pytest @@ -523,3 +524,94 @@ class TestPerplexityCostCalculator: ) assert math.isclose(total_cost, 1000 * 1.4e-06 + 500 * 4.4e-06, rel_tol=1e-9) + + OFF_PEAK_MODEL = "sonar-off-peak-test" + OFF_PEAK_WINDOW = "14:00-00:00" + INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc) + OUTSIDE_WINDOW = datetime(2026, 9, 3, 9, 0, tzinfo=timezone.utc) + + def _register_off_peak_model(self, off_peak_pricing: dict) -> None: + litellm.model_cost[f"perplexity/{self.OFF_PEAK_MODEL}"] = { + "litellm_provider": "perplexity", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 1e-06, + "output_cost_per_reasoning_token": 3e-06, + "citation_cost_per_token": 2e-06, + "search_context_cost_per_query": {"search_context_size_low": 0.005}, + "off_peak_pricing": off_peak_pricing, + } + + def test_off_peak_window_swaps_in_the_off_peak_rates(self): + """ + Regression (LIT-6874): a deployment configured with off_peak_pricing kept billing the + standard perplexity rates inside its window, while the same block on a deepseek + deployment billed the off-peak rates. + """ + self._register_off_peak_model( + {"hours_utc": self.OFF_PEAK_WINDOW, "input_cost_per_token": 1e-07, "output_cost_per_token": 2e-07} + ) + usage = Usage(prompt_tokens=1000, completion_tokens=200, total_tokens=1200) + + prompt_cost, completion_cost = perplexity_cost_per_token( + model=self.OFF_PEAK_MODEL, usage=usage, current_time=self.INSIDE_WINDOW + ) + + assert math.isclose(prompt_cost, 1000 * 1e-07, rel_tol=1e-10) + assert math.isclose(completion_cost, 200 * 2e-07, rel_tol=1e-10) + + peak_prompt_cost, peak_completion_cost = perplexity_cost_per_token( + model=self.OFF_PEAK_MODEL, usage=usage, current_time=self.OUTSIDE_WINDOW + ) + + assert math.isclose(peak_prompt_cost, 1000 * 1e-06, rel_tol=1e-10) + assert math.isclose(peak_completion_cost, 200 * 1e-06, rel_tol=1e-10) + + def test_off_peak_rates_leave_citation_search_and_reasoning_fees_alone(self): + """Inside the window only the plain input and output rates change: citation tokens, the + per-request search fee, and a dedicated reasoning rate keep billing as published.""" + self._register_off_peak_model( + {"hours_utc": self.OFF_PEAK_WINDOW, "input_cost_per_token": 1e-07, "output_cost_per_token": 2e-07} + ) + usage = Usage( + prompt_tokens=1000, + completion_tokens=200, + total_tokens=1200, + prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=1), + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=50), + ) + usage.citation_tokens = 100 + + prompt_cost, completion_cost = perplexity_cost_per_token( + model=self.OFF_PEAK_MODEL, usage=usage, current_time=self.INSIDE_WINDOW + ) + + assert math.isclose(prompt_cost, (1000 * 1e-07) + (100 * 2e-06), rel_tol=1e-10) + assert math.isclose(completion_cost, (150 * 2e-07) + (50 * 3e-06) + 0.005, rel_tol=1e-10) + + def test_off_peak_defaults_to_the_current_time(self): + """The proxy's cost dispatch passes no clock, so an all-day window has to apply on the + default current time.""" + self._register_off_peak_model( + {"hours_utc": "00:00-00:00", "input_cost_per_token": 1e-07, "output_cost_per_token": 2e-07} + ) + usage = Usage(prompt_tokens=1000, completion_tokens=200, total_tokens=1200) + + prompt_cost, completion_cost = perplexity_cost_per_token(model=self.OFF_PEAK_MODEL, usage=usage) + + assert math.isclose(prompt_cost, 1000 * 1e-07, rel_tol=1e-10) + assert math.isclose(completion_cost, 200 * 2e-07, rel_tol=1e-10) + + def test_provider_stated_cost_still_wins_inside_an_off_peak_window(self): + """A response that carries Perplexity's own metered cost bills that cost whatever the + window says; the caller strips it when the deployment carries custom pricing.""" + self._register_off_peak_model( + {"hours_utc": "00:00-00:00", "input_cost_per_token": 1e-07, "output_cost_per_token": 2e-07} + ) + usage = Usage(prompt_tokens=1000, completion_tokens=200, total_tokens=1200) + usage.cost = {"total_cost": 0.00501} + + prompt_cost, completion_cost = perplexity_cost_per_token(model=self.OFF_PEAK_MODEL, usage=usage) + + assert prompt_cost == 0.0 + assert completion_cost == 0.00501 From e65e3d0e2b89becc8fb55268ff18a8141e9ddf4b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:45:17 -0700 Subject: [PATCH 35/88] fix(cost): bill fireworks cached tokens at the off-peak input rate when no cache-read rate exists --- litellm/llms/fireworks_ai/cost_calculator.py | 11 +++++---- .../test_fireworks_ai_cost_calculator.py | 23 +++++++++++++++++-- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/litellm/llms/fireworks_ai/cost_calculator.py b/litellm/llms/fireworks_ai/cost_calculator.py index df47d3546ca..3843bad6d8f 100644 --- a/litellm/llms/fireworks_ai/cost_calculator.py +++ b/litellm/llms/fireworks_ai/cost_calculator.py @@ -2,6 +2,7 @@ For calculating cost of fireworks ai serverless inference models. """ +import math from datetime import datetime from typing import Final @@ -15,6 +16,8 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import apply_off_peak_pricin from litellm.types.utils import ModelInfo, Usage from litellm.utils import get_model_info +NO_CACHE_READ_RATE: Final = float("nan") + # Extract the number of billion parameters from the model name # only used for together_computer LLMs @@ -78,15 +81,15 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ model_info: Final = _resolve_model_info(model) - standard_input_rate: Final[float] = model_info["input_cost_per_token"] or 0.0 standard_cache_read_rate: Final = model_info.get("cache_read_input_token_cost") - input_rate, output_rate, cache_read_rate = apply_off_peak_pricing( + input_rate, output_rate, cache_read_rate_or_unset = apply_off_peak_pricing( model_info, current_time, - standard_input_rate, + model_info["input_cost_per_token"] or 0.0, model_info["output_cost_per_token"] or 0.0, - standard_cache_read_rate if standard_cache_read_rate is not None else standard_input_rate, + standard_cache_read_rate if standard_cache_read_rate is not None else NO_CACHE_READ_RATE, ) + cache_read_rate: Final[float] = input_rate if math.isnan(cache_read_rate_or_unset) else cache_read_rate_or_unset prompt_tokens_details: Final = usage.prompt_tokens_details cached_tokens: Final[int] = ( diff --git a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py index 21ee56a7873..555fdf7e11d 100644 --- a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py +++ b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py @@ -78,14 +78,14 @@ STANDARD_OUTPUT_COST = 6e-07 STANDARD_CACHE_READ_COST = 1.5e-08 -def _register_off_peak_model(off_peak_pricing: dict) -> None: +def _register_off_peak_model(off_peak_pricing: dict, cache_read_cost: float | None = STANDARD_CACHE_READ_COST) -> None: litellm.model_cost[f"fireworks_ai/{OFF_PEAK_MODEL}"] = { "litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": STANDARD_INPUT_COST, "output_cost_per_token": STANDARD_OUTPUT_COST, - "cache_read_input_token_cost": STANDARD_CACHE_READ_COST, "off_peak_pricing": off_peak_pricing, + **({} if cache_read_cost is None else {"cache_read_input_token_cost": cache_read_cost}), } @@ -129,6 +129,25 @@ def test_off_peak_rates_left_unset_keep_the_standard_rates(): assert math.isclose(completion_cost, 200 * STANDARD_OUTPUT_COST, rel_tol=1e-10) +def test_off_peak_window_bills_cached_tokens_at_the_off_peak_input_rate_without_a_cache_read_rate(): + """Most fireworks_ai price-map entries carry no cache_read_input_token_cost, so cached tokens + fall back to the input rate, and inside the window that has to be the off-peak one.""" + _register_off_peak_model( + {"hours_utc": OFF_PEAK_WINDOW, "input_cost_per_token": 1e-08, "output_cost_per_token": 2e-08}, + cache_read_cost=None, + ) + usage = _usage(prompt_tokens=1000, cached_tokens=300, completion_tokens=200) + + prompt_cost, completion_cost = cost_per_token(model=OFF_PEAK_MODEL, usage=usage, current_time=INSIDE_WINDOW) + + assert math.isclose(prompt_cost, 1000 * 1e-08, rel_tol=1e-10) + assert math.isclose(completion_cost, 200 * 2e-08, rel_tol=1e-10) + + peak_prompt_cost, _ = cost_per_token(model=OFF_PEAK_MODEL, usage=usage, current_time=OUTSIDE_WINDOW) + + assert math.isclose(peak_prompt_cost, 1000 * STANDARD_INPUT_COST, rel_tol=1e-10) + + def test_off_peak_defaults_to_the_current_time(): """The proxy's cost dispatch passes no clock, so an all-day window has to apply on the default current time.""" From eae7b806e3b71a8adc36e6d3a5bf0b51edaa7f97 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 3 Sep 2026 20:49:44 +0000 Subject: [PATCH 36/88] fix(registry): point Bedrock Qwen3 Coder 480B source at the us-west-2 on-demand price list Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 2 +- model_prices_and_context_window.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 260a9c3a34e..3b6e7912253 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -39959,7 +39959,7 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_native_structured_output": true, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-west-2/index.json" }, "qwen.qwen3-235b-a22b-2507-v1:0": { "input_cost_per_token": 2.2e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 260a9c3a34e..3b6e7912253 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -39959,7 +39959,7 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_native_structured_output": true, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-west-2/index.json" }, "qwen.qwen3-235b-a22b-2507-v1:0": { "input_cost_per_token": 2.2e-07, From 8d82f28c85bf4dec5c1701290b1d8aa8ebc58376 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 3 Sep 2026 21:14:18 +0000 Subject: [PATCH 37/88] test(ocr): register the azure ocr4 mixed-rate cost test in the parity ledger Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json b/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json index 1ceb79b52bc..41381456993 100644 --- a/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json +++ b/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json @@ -146,6 +146,7 @@ {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_bills_annotation_only_response", "status": "unmapped", "reason": "cost-calc: annotation-only billing math is Python-only"}, {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_bills_annotation_pages_when_pages_processed_missing", "status": "unmapped", "reason": "cost-calc: fallback billing math is Python-only"}, {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_azure_doc_ai_annotation_pages_fall_back_to_ocr_rate", "status": "unmapped", "reason": "cost-calc: fallback billing math is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_azure_ocr4_bills_ocr_and_annotation_pages_at_their_own_rates", "status": "unmapped", "reason": "cost-calc: mixed-rate billing math is Python-only"}, {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_rust_ocr_serves_default_format", "status": "unmapped", "reason": "request-format gating decision is made in Python before the Rust bridge is ever invoked"}, {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_rust_ocr_skipped_for_native_format", "status": "unmapped", "reason": "request-format gating decision is made in Python before the Rust bridge is ever invoked"}, From 00bdfe797a385482bbfb66a167248eeee96ebf60 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 3 Sep 2026 21:16:55 +0000 Subject: [PATCH 38/88] fix(registry): mark gemini-3.5-live-translate-preview as realtime with official token limits Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 8 ++++++-- model_prices_and_context_window.json | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1c51385318b..682ad381268 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -55447,7 +55447,10 @@ "input_cost_per_audio_token": 3.5e-06, "input_cost_per_token": 3.5e-06, "litellm_provider": "gemini", - "mode": "chat", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "realtime", "output_cost_per_audio_token": 2.1e-05, "output_cost_per_token": 2.1e-05, "rpm": 10, @@ -55459,7 +55462,8 @@ "audio" ], "supported_output_modalities": [ - "audio" + "audio", + "text" ], "supports_audio_input": true, "supports_audio_output": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1c51385318b..682ad381268 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -55447,7 +55447,10 @@ "input_cost_per_audio_token": 3.5e-06, "input_cost_per_token": 3.5e-06, "litellm_provider": "gemini", - "mode": "chat", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "realtime", "output_cost_per_audio_token": 2.1e-05, "output_cost_per_token": 2.1e-05, "rpm": 10, @@ -55459,7 +55462,8 @@ "audio" ], "supported_output_modalities": [ - "audio" + "audio", + "text" ], "supports_audio_input": true, "supports_audio_output": true, From f29266760109252966e9b1739ceb6a9b97bac523 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 3 Sep 2026 21:41:12 +0000 Subject: [PATCH 39/88] fix(registry): mark gpt-daybreak-*-latest as responses mode to match their Responses-only endpoints Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 4 ++-- model_prices_and_context_window.json | 4 ++-- tests/test_litellm/test_daybreak_model_metadata.py | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 682ad381268..7f01d2322ff 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -29781,7 +29781,7 @@ "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "chat", + "mode": "responses", "output_cost_per_token": 7.5e-05, "output_cost_per_token_above_272k_tokens": 0.0001125, "supported_endpoints": [ @@ -29858,7 +29858,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "chat", + "mode": "responses", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "supported_endpoints": [ diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 682ad381268..7f01d2322ff 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -29781,7 +29781,7 @@ "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "chat", + "mode": "responses", "output_cost_per_token": 7.5e-05, "output_cost_per_token_above_272k_tokens": 0.0001125, "supported_endpoints": [ @@ -29858,7 +29858,7 @@ "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "chat", + "mode": "responses", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "supported_endpoints": [ diff --git a/tests/test_litellm/test_daybreak_model_metadata.py b/tests/test_litellm/test_daybreak_model_metadata.py index 068bc01e103..dbb7ecdffac 100644 --- a/tests/test_litellm/test_daybreak_model_metadata.py +++ b/tests/test_litellm/test_daybreak_model_metadata.py @@ -62,6 +62,7 @@ def test_official_alias_tracks_snapshot(alias, snapshot): snapshot_info = cost_map[snapshot] assert alias_info["supported_endpoints"] == ["/v1/responses"] + assert alias_info["mode"] == "responses" assert alias_info["source"] == f"https://developers.openai.com/api/docs/models/{alias}" assert {field: alias_info.get(field) for field in PRICE_FIELDS} == { field: snapshot_info.get(field) for field in PRICE_FIELDS From 0f759c56f002b511be497b7769041d03c791cee2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:00:08 -0700 Subject: [PATCH 40/88] fix(proxy): bill partial usage on failed Vertex and Gemini pass-through streams --- .../streaming_handler.py | 70 +++++++++++-- .../test_streaming_handler_interrupt.py | 97 +++++++++++++++++++ 2 files changed, 161 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index ba2717ef119..88c14c9348c 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -1,6 +1,7 @@ import traceback from collections.abc import Coroutine, Mapping, Sequence -from datetime import datetime +from dataclasses import dataclass +from datetime import datetime, timezone from typing import Final, Protocol import httpx @@ -13,7 +14,7 @@ from litellm.proxy._types import PassThroughEndpointLoggingResultValues from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType -from litellm.types.utils import StandardPassThroughResponseObject +from litellm.types.utils import StandardPassThroughResponseObject, Usage from .llm_provider_handlers.anthropic_passthrough_logging_handler import ( AnthropicPassthroughLoggingHandler, @@ -45,6 +46,13 @@ class RouteStreamingLogging(Protocol): ) -> Coroutine[None, None, None]: ... +@dataclass(frozen=True, slots=True) +class PassThroughStreamContext: + passthrough_success_handler_obj: PassThroughEndpointLogging + url_route: str + start_time: datetime + + class PassThroughStreamingHandler: @staticmethod def _stamp_first_chunk_if_needed(litellm_logging_obj: LiteLLMLoggingObj) -> None: @@ -58,11 +66,15 @@ class PassThroughStreamingHandler: request_body: Mapping[str, object], raw_bytes: Sequence[bytes], exception: Exception, + stream_context: PassThroughStreamContext | None = None, ) -> None: - if endpoint_type == EndpointType.ANTHROPIC: - AnthropicPassthroughLoggingHandler.record_partial_usage_for_failure( - litellm_logging_obj=litellm_logging_obj, request_body=request_body, all_chunks=raw_bytes - ) + PassThroughStreamingHandler._record_partial_usage_for_failure( + litellm_logging_obj=litellm_logging_obj, + endpoint_type=endpoint_type, + request_body=request_body, + raw_bytes=raw_bytes, + stream_context=stream_context, + ) try: GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( async_coroutine=litellm_logging_obj.dispatch_failure_handlers( @@ -72,6 +84,47 @@ class PassThroughStreamingHandler: except Exception as e: verbose_proxy_logger.error("Error scheduling stream failure logging: %s", e) + @staticmethod + def _record_partial_usage_for_failure( + litellm_logging_obj: LiteLLMLoggingObj, + endpoint_type: EndpointType, + request_body: Mapping[str, object], + raw_bytes: Sequence[bytes], + stream_context: PassThroughStreamContext | None, + ) -> None: + if endpoint_type == EndpointType.ANTHROPIC: + AnthropicPassthroughLoggingHandler.record_partial_usage_for_failure( + litellm_logging_obj=litellm_logging_obj, request_body=request_body, all_chunks=raw_bytes + ) + return + if stream_context is None or not raw_bytes: + return + try: + partial_response, kwargs = PassThroughStreamingHandler._build_passthrough_logging_result( + litellm_logging_obj=litellm_logging_obj, + passthrough_success_handler_obj=stream_context.passthrough_success_handler_obj, + url_route=stream_context.url_route, + request_body=dict(request_body), + endpoint_type=endpoint_type, + start_time=stream_context.start_time, + raw_bytes=list(raw_bytes), + end_time=datetime.now(timezone.utc), + model=None, + ) + except Exception as e: + verbose_proxy_logger.warning( + "Could not recover the partial usage of a failed %s pass-through stream: %s", endpoint_type.value, e + ) + return + usage: Final = getattr(partial_response, "usage", None) + if not isinstance(usage, Usage): + return + response_cost: Final = kwargs.get("response_cost") + litellm_logging_obj.record_partial_usage_for_failure( + usage=usage, + response_cost=float(response_cost) if isinstance(response_cost, (int, float)) else 0.0, + ) + @staticmethod async def chunk_processor( response: httpx.Response, @@ -174,6 +227,11 @@ class PassThroughStreamingHandler: request_body=request_body or {}, raw_bytes=raw_bytes, exception=e, + stream_context=PassThroughStreamContext( + passthrough_success_handler_obj=passthrough_success_handler_obj, + url_route=url_route, + start_time=start_time, + ), ) raise finally: diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py b/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py index c4ae0c81d6e..ea6adc35b9a 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py @@ -741,3 +741,100 @@ async def test_chunk_processor_logs_failure_not_success_on_mid_stream_exception( assert failure_payload["prompt_tokens"] == 52 assert failure_payload["response_cost"] > 0 assert isinstance(recorder.failure_kwargs[0]["exception"], httpx.ReadTimeout) + + +def _google_sse(prompt_tokens: int, completion_tokens: int, text: str) -> bytes: + payload = { + "candidates": [{"content": {"parts": [{"text": text}], "role": "model"}, "index": 0}], + "usageMetadata": { + "promptTokenCount": prompt_tokens, + "candidatesTokenCount": completion_tokens, + "totalTokenCount": prompt_tokens + completion_tokens, + }, + "modelVersion": "gemini-3.8-flash", + } + return f"data: {json.dumps(payload)}\r\n\r\n".encode() + + +def _google_stream_that_times_out_mid_stream(): + mock = MagicMock(spec=httpx.Response) + mock.status_code = 200 + + async def _aiter_bytes(): + yield _google_sse(9, 4, "The sea") + yield _google_sse(9, 12, " is wide and restless") + raise httpx.ReadTimeout("Timeout on reading data from socket") + + mock.aiter_bytes = _aiter_bytes + return mock + + +@pytest.mark.parametrize( + "endpoint_type, url_route", + [ + (EndpointType.GEMINI, "/gemini/v1beta/models/gemini-3.8-flash:streamGenerateContent?alt=sse"), + ( + EndpointType.VERTEX_AI, + "/vertex_ai/v1/projects/p/locations/us-central1/publishers/google/models/gemini-3.8-flash:streamGenerateContent?alt=sse", + ), + ], +) +@pytest.mark.asyncio +async def test_chunk_processor_bills_partial_google_usage_on_mid_stream_exception(endpoint_type, url_route): + """Google streams carry cumulative usage on every chunk, so a stream that + dies mid-way must log a failure billed at what was already delivered rather + than a failure at zero usage.""" + recorder = _EventRecorder() + logging_obj = LiteLLMLoggingObj( + model="gemini-3.8-flash", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id=f"test-google-mid-stream-timeout-{endpoint_type.value}", + function_id="test-google-mid-stream-timeout", + dynamic_async_success_callbacks=[recorder], + dynamic_async_failure_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model="gemini-3.8-flash", + user="unknown", + optional_params={}, + litellm_params={"metadata": {}}, + call_type="pass_through_endpoint", + ) + success_routes = [] + + async def _record_success_route(**kwargs): + success_routes.append(kwargs) + + async def _consume_stream(): + async for _ in PassThroughStreamingHandler.chunk_processor( + response=_google_stream_that_times_out_mid_stream(), + request_body={"contents": [{"role": "user", "parts": [{"text": "hi"}]}]}, + litellm_logging_obj=logging_obj, + endpoint_type=endpoint_type, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route=url_route, + route_streaming_logging=_record_success_route, + ): + pass + + with pytest.raises(httpx.ReadTimeout): + await _consume_stream() + + for _ in range(300): + if recorder.failure_kwargs: + break + await asyncio.sleep(0.01) + + assert success_routes == [] + assert recorder.success_kwargs == [] + assert len(recorder.failure_kwargs) == 1 + failure_payload = recorder.failure_kwargs[0]["standard_logging_object"] + assert failure_payload["status"] == "failure" + assert failure_payload["prompt_tokens"] == 9 + assert failure_payload["completion_tokens"] == 12 + assert failure_payload["response_cost"] > 12 * 3.75e-06 + assert isinstance(recorder.failure_kwargs[0]["exception"], httpx.ReadTimeout) From f9e41470d68fa2be215290300c4977890bab0d9b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:03:10 -0700 Subject: [PATCH 41/88] test(cost): type the off-peak fixture helpers with OffPeakPricing --- .../llms/fireworks_ai/test_fireworks_ai_cost_calculator.py | 4 ++-- .../llms/perplexity/test_perplexity_cost_calculator.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py index 555fdf7e11d..c2e42da1b4c 100644 --- a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py +++ b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py @@ -7,7 +7,7 @@ import pytest import litellm from litellm.llms.fireworks_ai.cost_calculator import cost_per_token -from litellm.types.utils import PromptTokensDetailsWrapper, Usage +from litellm.types.utils import OffPeakPricing, PromptTokensDetailsWrapper, Usage MODEL = "accounts/fireworks/models/glm-5p2" INPUT_COST = 1.4e-06 @@ -78,7 +78,7 @@ STANDARD_OUTPUT_COST = 6e-07 STANDARD_CACHE_READ_COST = 1.5e-08 -def _register_off_peak_model(off_peak_pricing: dict, cache_read_cost: float | None = STANDARD_CACHE_READ_COST) -> None: +def _register_off_peak_model(off_peak_pricing: OffPeakPricing, cache_read_cost: float | None = STANDARD_CACHE_READ_COST) -> None: litellm.model_cost[f"fireworks_ai/{OFF_PEAK_MODEL}"] = { "litellm_provider": "fireworks_ai", "mode": "chat", diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py index be338bd3dfa..6630039e92e 100644 --- a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py @@ -22,6 +22,7 @@ from litellm.llms.perplexity.cost_calculator import ( ) from litellm.types.utils import ( CompletionTokensDetailsWrapper, + OffPeakPricing, Usage, PromptTokensDetailsWrapper, ) @@ -530,7 +531,7 @@ class TestPerplexityCostCalculator: INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc) OUTSIDE_WINDOW = datetime(2026, 9, 3, 9, 0, tzinfo=timezone.utc) - def _register_off_peak_model(self, off_peak_pricing: dict) -> None: + def _register_off_peak_model(self, off_peak_pricing: OffPeakPricing) -> None: litellm.model_cost[f"perplexity/{self.OFF_PEAK_MODEL}"] = { "litellm_provider": "perplexity", "mode": "chat", From 04a2407244bc785d47f173c7efee1283122ad3fa Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 3 Sep 2026 23:03:14 +0000 Subject: [PATCH 42/88] fix(organization): clear org budget limits when PATCH /organization/update sends null A sent null for tpm_limit, rpm_limit, max_budget and the other budget fields was dropped by a 'v is not None' filter, so update_budget was never called and the request returned 200 without changing the budget row. Presence is now read from model_fields_set (merge-patch semantics, matching /v2/organization) and the nested litellm_budget_table payload no longer drops nulls either Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../organization_endpoints.py | 5 +- .../test_organization_endpoints.py | 58 +++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py index 5e38a016099..d32b842df58 100644 --- a/litellm/proxy/management_endpoints/organization_endpoints.py +++ b/litellm/proxy/management_endpoints/organization_endpoints.py @@ -312,7 +312,7 @@ def handle_nested_budget_structure_in_organization_update_request( # Extract valid budget fields and merge into top level budget_fields: Final = LiteLLM_BudgetTable.model_fields.keys() for key, value in budget_data.items(): - if key in budget_fields and value is not None: + if key in budget_fields: transformed_data[key] = value return transformed_data @@ -708,9 +708,8 @@ async def update_organization( existing_organization_row=existing_organization_row, ) - # Handle budget updates if budget fields are provided budget_fields: Final = { - k: v for k, v in data.model_dump().items() if k in LiteLLM_BudgetTable.model_fields and v is not None + k: v for k, v in data.model_dump().items() if k in _BUDGET_SETTABLE_FIELDS and k in data.model_fields_set } if budget_fields and existing_organization_row.budget_id: diff --git a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py index e2d89a660c2..5c2a0bdde3d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py @@ -963,6 +963,64 @@ async def test_v2_serializes_model_max_budget_on_budget_write(monkeypatch): assert json.loads(written) == {"gpt-4o": {"max_budget": 10}} +async def _run_legacy_update_organization(monkeypatch, *, body: dict, existing_budget_id: str): + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints import organization_endpoints + from litellm.proxy.management_endpoints.organization_endpoints import update_organization + from litellm.proxy.utils import jsonify_object + + mock_prisma_client = AsyncMock() + mock_prisma_client.jsonify_object = jsonify_object + + existing_org = MagicMock() + existing_org.budget_id = existing_budget_id + existing_org.metadata = {} + mock_prisma_client.db.litellm_organizationtable.find_unique = AsyncMock(return_value=existing_org) + mock_prisma_client.db.litellm_organizationtable.update = AsyncMock(return_value=MagicMock()) + mock_prisma_client.db.litellm_budgettable.update = AsyncMock() + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr(organization_endpoints, "_verify_org_access", AsyncMock()) + + request = MagicMock() + request.json = AsyncMock(return_value=body) + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin-1") + await update_organization(request=request, user_api_key_dict=auth) + return mock_prisma_client + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body", + [ + {"organization_id": "org-1", "tpm_limit": None}, + {"organization_id": "org-1", "litellm_budget_table": {"tpm_limit": None}}, + ], +) +async def test_legacy_update_clears_tpm_limit_when_sent_null(monkeypatch, body): + """PATCH /organization/update with tpm_limit: null writes None to the budget row instead of dropping it.""" + prisma = await _run_legacy_update_organization(monkeypatch, body=body, existing_budget_id="budget-1") + + budget_write = prisma.db.litellm_budgettable.update.await_args + assert budget_write.kwargs["where"] == {"budget_id": "budget-1"} + assert budget_write.kwargs["data"]["tpm_limit"] is None + assert "rpm_limit" not in budget_write.kwargs["data"] + assert "tpm_limit" not in prisma.db.litellm_organizationtable.update.await_args.kwargs["data"] + + +@pytest.mark.asyncio +async def test_legacy_update_without_budget_fields_skips_budget_write(monkeypatch): + """Omitted budget fields are left untouched: renaming the org must not write the budget row.""" + prisma = await _run_legacy_update_organization( + monkeypatch, + body={"organization_id": "org-1", "organization_alias": "renamed"}, + existing_budget_id="budget-1", + ) + + prisma.db.litellm_budgettable.update.assert_not_awaited() + assert prisma.db.litellm_organizationtable.update.await_args.kwargs["data"]["organization_alias"] == "renamed" + + def test_build_budget_write_data_recomputes_reset_at_on_duration(): """A sent budget_duration recomputes budget_reset_at so the reset window follows the new duration.""" from litellm.proxy.management_endpoints.organization_endpoints import build_budget_write_data From fde676dc386188f4acf8f991548cbfd61745d298 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:42:08 -0700 Subject: [PATCH 43/88] fix(anthropic): run the proxy failure hook when a detached /v1/messages stream fails --- litellm/litellm_core_utils/litellm_logging.py | 3 +- .../messages/streaming_iterator.py | 37 ++++-- litellm/proxy/common_request_processing.py | 33 ++++++ .../messages/test_streaming_iterator.py | 31 +++++ .../proxy/test_common_request_processing.py | 109 ++++++++++++++++++ 5 files changed, 204 insertions(+), 9 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 463cbf7cdbe..d6f2387ac71 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -10,7 +10,7 @@ import subprocess import sys import time import traceback -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from datetime import datetime as dt_object from functools import lru_cache from types import MappingProxyType, TracebackType @@ -576,6 +576,7 @@ class Logging(LiteLLMLoggingBaseClass): # enqueue closure here instead of firing it immediately. self._defer_async_logging: bool = False self._enqueue_deferred_logging: Callable[[], None] | None = None + self._on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None def set_response_timing_metrics(self, timing_metrics: Mapping[str, float]) -> None: """Keep ``_response_ms`` / ``litellm_overhead_time_ms`` for a result that has no ``_hidden_params``.""" diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py index 275608fcccc..7d01aee5d98 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py @@ -176,6 +176,11 @@ def _try_claim_detached_drain_slot() -> bool: return True +def _exception_left_unconsumed(queue: "asyncio.Queue[bytes | None | BaseException]", exc: BaseException) -> bool: + remaining: Final = tuple(queue.get_nowait() for _ in range(queue.qsize())) + return any(item is exc for item in remaining) + + def _sse_event(event_type: str, payload: Mapping[str, object]) -> bytes: return f"event: {event_type}\ndata: {json.dumps(payload)}\n\n".encode() @@ -663,18 +668,16 @@ class BaseAnthropicMessagesStreamingIterator: collected_chunks: Sequence[bytes], exc: Exception, ) -> None: - """Forward a provider error to a still-connected client and log the request as failed. + """Log the request as failed with its partial usage, then make sure the proxy's failure hook runs once. - The relay re-raises the forwarded exception so the proxy's failure hook - keeps the provider status; the logging object's failure handlers fire - here either way, carrying the partial usage the provider already - billed, so a client that left before consuming the exception still - gets a failure row rather than a success one. + A still-connected client gets the original exception through the queue, + the relay re-raises it, and the proxy's own failure handling records the + failed spend. When the client already left, or leaves before consuming + the queued exception, that handling never runs, so the detached-failure + hook the proxy armed on the logging object fires here instead. """ from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler - if not client_detached.is_set(): - await self._enqueue_for_client(queue, client_detached, exc) PassThroughStreamingHandler.schedule_stream_failure_logging( litellm_logging_obj=self.litellm_logging_obj, endpoint_type=EndpointType.ANTHROPIC, @@ -682,3 +685,21 @@ class BaseAnthropicMessagesStreamingIterator: raw_bytes=collected_chunks, exception=exc, ) + if not client_detached.is_set() and await self._enqueue_for_client(queue, client_detached, exc): + await client_detached.wait() + if not _exception_left_unconsumed(queue, exc): + return + await self._fire_detached_failure_hook(exc) + + async def _fire_detached_failure_hook(self, exc: Exception) -> None: + from litellm._logging import verbose_proxy_logger + + on_detached_failure: Final = getattr(self.litellm_logging_obj, "_on_detached_stream_failure", None) + if on_detached_failure is None: + return + try: + await on_detached_failure(exc) + except Exception as hook_failure: # noqa: BLE001 # a failing proxy hook must not crash the detached pump + verbose_proxy_logger.warning( + "async_sse_wrapper detached failure hook raised: %s(%s)", type(hook_failure).__name__, hook_failure + ) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 6542842f5e4..f25fa46197e 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2520,6 +2520,11 @@ class ProxyBaseLLMRequestProcessing: # This handles cases like websearch_interception agentic loop # which returns a non-streaming dict even for streaming requests if self._is_streaming_response(response): + self._arm_detached_stream_failure_hook( + logging_obj=logging_obj, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + ) selected_data_generator = ProxyBaseLLMRequestProcessing.async_sse_data_generator( response=response, user_api_key_dict=user_api_key_dict, @@ -2875,6 +2880,34 @@ class ProxyBaseLLMRequestProcessing: ), ) + def _arm_detached_stream_failure_hook( + self, + logging_obj: LiteLLMLoggingObj, + user_api_key_dict: "UserAPIKeyAuth", + proxy_logging_obj: ProxyLogging, + ) -> None: + """Let a stream that fails after the client left still reach ``post_call_failure_hook``. + + The client-facing generator reports a mid-stream failure itself, but once + the client disconnects that generator is gone and the detached upstream + drain is the only code that sees the provider error. It fires this closure + so the failed spend is still written and the budget reservation released; + a replacement error the hook raises has no client left to reach. + """ + request_data: Final = self.data + + async def _on_detached_stream_failure(exc: Exception) -> None: + try: + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=exc, + request_data=request_data, + ) + except HTTPException: + return + + logging_obj._on_detached_stream_failure = _on_detached_stream_failure + def _is_streaming_response(self, response: Any) -> bool: """ Check if the response object is actually a streaming response by inspecting its type. diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py index 3d41d0942e5..be33b2ee3b1 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py @@ -544,6 +544,25 @@ async def test_async_sse_wrapper_dispatches_deferred_logging_when_client_disconn await asyncio.wait_for(deferred_fired.wait(), timeout=5) +class _DetachedFailureRecorder: + """Stands in for the closure the proxy arms so a detached-stream failure still reaches its failure hook.""" + + def __init__(self): + self.exceptions = [] + + async def __call__(self, exc: Exception) -> None: + self.exceptions.append(exc) + + +async def _wait_for_detached_failure(recorder: _DetachedFailureRecorder) -> Exception: + for _ in range(200): + if recorder.exceptions: + await asyncio.sleep(0.02) + return recorder.exceptions[0] + await asyncio.sleep(0.01) + raise AssertionError("the detached failure hook never fired") + + class _ProviderStreamError(Exception): """Stand-in for a provider-specific streaming failure carrying a status code.""" @@ -573,6 +592,8 @@ async def test_async_sse_wrapper_reraises_upstream_error_to_connected_client(): litellm_logging_obj=_make_logging_obj("test_reraises_upstream_error", recorder), request_body={}, ) + detached_hook = _DetachedFailureRecorder() + iterator.litellm_logging_obj._on_detached_stream_failure = detached_hook received = [] @@ -591,6 +612,8 @@ async def test_async_sse_wrapper_reraises_upstream_error_to_connected_client(): assert iterator.logged_chunks == [] assert failure_kwargs["standard_logging_object"]["status"] == "failure" assert failure_kwargs["standard_logging_object"]["prompt_tokens"] == 52 + await asyncio.sleep(0.05) + assert detached_hook.exceptions == [], "the relay re-raised the error, so the proxy failure hook already ran" @pytest.mark.asyncio @@ -614,6 +637,8 @@ async def test_async_sse_wrapper_logs_failure_on_upstream_error_after_disconnect litellm_logging_obj=_make_logging_obj("test_failure_logged_on_late_error", recorder), request_body={}, ) + detached_hook = _DetachedFailureRecorder() + iterator.litellm_logging_obj._on_detached_stream_failure = detached_hook gen = iterator.async_sse_wrapper(_gated_failing_stream()) received = [await gen.__anext__(), await gen.__anext__()] @@ -627,6 +652,8 @@ async def test_async_sse_wrapper_logs_failure_on_upstream_error_after_disconnect assert failure_kwargs["standard_logging_object"]["status"] == "failure" assert failure_kwargs["standard_logging_object"]["prompt_tokens"] == 52 assert isinstance(failure_kwargs["exception"], _ProviderStreamError) + assert await _wait_for_detached_failure(detached_hook) is failure_kwargs["exception"] + assert len(detached_hook.exceptions) == 1 @pytest.mark.asyncio @@ -651,6 +678,8 @@ async def test_async_sse_wrapper_logs_failure_when_queued_error_is_never_consume litellm_logging_obj=_make_logging_obj("test_failure_logged_on_unconsumed_queued_error", recorder), request_body={}, ) + detached_hook = _DetachedFailureRecorder() + iterator.litellm_logging_obj._on_detached_stream_failure = detached_hook gen = iterator.async_sse_wrapper(_failing_stream()) received = [await gen.__anext__(), await gen.__anext__()] @@ -663,6 +692,8 @@ async def test_async_sse_wrapper_logs_failure_when_queued_error_is_never_consume assert iterator.logging_call_count == 0 assert failure_kwargs["standard_logging_object"]["status"] == "failure" assert failure_kwargs["standard_logging_object"]["prompt_tokens"] == 52 + assert await _wait_for_detached_failure(detached_hook) is failure_kwargs["exception"] + assert len(detached_hook.exceptions) == 1 @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index ea665b60b19..f7fe6ad9d39 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -7836,3 +7836,112 @@ def test_log_llm_api_exception_traceback_only_for_unexpected_errors(exc, expect_ records = [r for r in caplog.records if "_handle_llm_api_exception(): Exception occured" in r.getMessage()] assert len(records) == 1 assert (records[0].exc_info is not None) is expect_traceback + + +class _FailureHookRecorder: + """Stands in for ProxyLogging.post_call_failure_hook, recording what the detached-failure closure hands it.""" + + def __init__(self, raises: Optional[Exception] = None): + self.calls = [] + self._raises = raises + + async def post_call_failure_hook(self, **kwargs): + self.calls.append(kwargs) + if self._raises is not None: + raise self._raises + + +class TestDetachedStreamFailureHook: + """ + Regression for LIT-3798. A streaming /v1/messages request whose client disconnected + before the provider failed mid-stream never reached the proxy's failure hook: the + client-facing generator was gone, and the detached upstream drain only fired the + logging object's callbacks, so no failure spend row was written and the budget + reservation stayed held. base_process_llm_request now arms a closure on the logging + object that the detached drain awaits, and that closure runs post_call_failure_hook + with the request's key and data. + """ + + @staticmethod + def _logging_obj(): + logging_obj = MagicMock() + logging_obj.litellm_call_id = "call-lit3798" + logging_obj.model_call_details = {} + logging_obj._enqueue_deferred_logging = None + logging_obj._on_deferred_stream_complete = None + logging_obj._on_detached_stream_failure = None + return logging_obj + + @staticmethod + def _proxy_logging_obj(recorder: _FailureHookRecorder): + proxy_logging_obj = MagicMock(spec=ProxyLogging) + proxy_logging_obj.during_call_hook = AsyncMock(return_value=None) + proxy_logging_obj.update_request_status = AsyncMock(return_value=None) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + proxy_logging_obj.post_call_failure_hook = recorder.post_call_failure_hook + return proxy_logging_obj + + @pytest.mark.asyncio + async def test_streaming_messages_arms_the_detached_failure_hook(self, monkeypatch): + import litellm.proxy.common_request_processing as crp + from litellm.proxy._types import UserAPIKeyAuth as RealUserAPIKeyAuth + + async def _stream(): + yield b"event: message_start\n\n" + + async def fake_route_request(**kwargs): + async def _llm_call(): + return _stream() + + return _llm_call() + + monkeypatch.setattr(crp, "route_request", fake_route_request) + monkeypatch.setattr(litellm, "callbacks", []) + recorder = _FailureHookRecorder() + logging_obj = self._logging_obj() + user_api_key_dict = RealUserAPIKeyAuth(api_key="sk-test") + processing_obj = ProxyBaseLLMRequestProcessing( + data={"litellm_logging_obj": logging_obj, "model": "claude-sonnet-4-5"} + ) + + await processing_obj.base_process_llm_request( + request=MagicMock(spec=Request, headers={}), + fastapi_response=Response(), + user_api_key_dict=user_api_key_dict, + route_type="anthropic_messages", + proxy_logging_obj=self._proxy_logging_obj(recorder), + general_settings={}, + proxy_config=MagicMock(spec=ProxyConfig), + select_data_generator=None, + llm_router=None, + skip_pre_call_logic=True, + ) + + failure = RuntimeError("upstream died after the client left") + await logging_obj._on_detached_stream_failure(failure) + + assert recorder.calls == [ + { + "user_api_key_dict": user_api_key_dict, + "original_exception": failure, + "request_data": processing_obj.data, + } + ] + + @pytest.mark.asyncio + async def test_detached_failure_hook_drops_the_replacement_error_it_cannot_deliver(self): + from litellm.proxy._types import UserAPIKeyAuth as RealUserAPIKeyAuth + + recorder = _FailureHookRecorder(raises=HTTPException(status_code=429, detail="budget exceeded")) + logging_obj = self._logging_obj() + processing_obj = ProxyBaseLLMRequestProcessing(data={"litellm_logging_obj": logging_obj}) + processing_obj._arm_detached_stream_failure_hook( + logging_obj=logging_obj, + user_api_key_dict=RealUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=self._proxy_logging_obj(recorder), + ) + failure = RuntimeError("upstream died after the client left") + + await logging_obj._on_detached_stream_failure(failure) + + assert [call["original_exception"] for call in recorder.calls] == [failure] From ce95afe2bdfb47f4325c59fea89b8c8f8fb0a5a6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:58:17 -0700 Subject: [PATCH 44/88] fix(spend-tracking): reverse-hash dirty spend keys in Postgres instead of paging token tables --- litellm/integrations/cloudzero/database.py | 2 - litellm/integrations/focus/database.py | 2 - litellm/proxy/_types.py | 1 - .../spend_tracking/key_metadata_recovery.py | 232 ++++------------- .../spend_tracking/spend_tracking_utils.py | 1 - .../integrations/cloudzero/test_cloudzero.py | 2 +- .../test_common_daily_activity.py | 58 ++--- .../test_key_metadata_recovery.py | 234 ++++++++---------- .../test_spend_management_endpoints.py | 1 - .../test_spend_tracking_utils.py | 2 - 10 files changed, 189 insertions(+), 346 deletions(-) diff --git a/litellm/integrations/cloudzero/database.py b/litellm/integrations/cloudzero/database.py index e630bd85114..4adf725fd0f 100644 --- a/litellm/integrations/cloudzero/database.py +++ b/litellm/integrations/cloudzero/database.py @@ -105,8 +105,6 @@ class LiteLLMDatabase: if isinstance(db_response, list) else [] ) - # v1.99 double-hashed DailyUserSpend.api_key values miss the - # VerificationToken join above; recover alias/team for those rows. recovered_rows: Final = await fill_missing_api_key_aliases(client, usage_rows) return pl.DataFrame(tuple(recovered_rows), infer_schema_length=None) except Exception as e: diff --git a/litellm/integrations/focus/database.py b/litellm/integrations/focus/database.py index 02b1e9e944b..f214aa02b5b 100644 --- a/litellm/integrations/focus/database.py +++ b/litellm/integrations/focus/database.py @@ -107,8 +107,6 @@ class FocusLiteLLMDatabase: if isinstance(db_response, list) else [] ) - # v1.99 double-hashed DailyUserSpend.api_key values miss the - # VerificationToken join above; recover alias/team for those rows. recovered_rows: Final = await fill_missing_api_key_aliases(client, usage_rows) return pl.DataFrame(tuple(recovered_rows), infer_schema_length=None) except Exception as exc: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9b4a55d3510..5d5a25e7cd6 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3659,7 +3659,6 @@ class SpendLogsMetadata(TypedDict): user_api_key_project_alias: str | None user_api_key_org_id: str | None user_api_key_user_id: str | None - user_api_key_user_email: ReadOnly[str | None] user_api_key_team_alias: str | None spend_logs_metadata: dict | None # special param to log k,v pairs to spendlogs for a call requester_ip_address: str | None diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index 895f97e1a05..d524b158c9c 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -1,38 +1,30 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence from collections.abc import Set as AbstractSet from types import MappingProxyType -from typing import Final, Protocol, TypeVar +from typing import Final, TypeVar +from pydantic import BaseModel, TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash -from litellm.proxy.utils import PrismaClient, hash_token -from litellm.repositories.table_repositories import DeletedVerificationTokenRepository +from litellm.proxy.utils import PrismaClient from litellm.repositories.user_repository import UserRepository -from litellm.repositories.verification_token_repository import ( - VerificationTokenRepository, -) _T = TypeVar("_T") -_TOKEN_SCAN_PAGE: Final = 10_000 +_ACTIVE_TOKEN_DIGEST_SQL: Final = """ +SELECT encode(sha256(convert_to(token, 'UTF8')), 'hex') AS digest, key_alias, team_id, user_id +FROM "LiteLLM_VerificationToken" +WHERE encode(sha256(convert_to(token, 'UTF8')), 'hex') = ANY($1::text[]) +""" -_SPEND_LOGS_KEY_METADATA_SQL: Final = """ -SELECT DISTINCT ON (api_key) - api_key, - metadata->>'user_api_key_alias' AS key_alias, - metadata->>'user_api_key_team_id' AS team_id, - metadata->>'user_api_key_user_id' AS user_id, - metadata->>'user_api_key_user_email' AS user_email -FROM "LiteLLM_SpendLogs" -WHERE api_key = ANY($1::text[]) - AND ( - NULLIF(metadata->>'user_api_key_alias', '') IS NOT NULL - OR NULLIF(metadata->>'user_api_key_team_id', '') IS NOT NULL - OR NULLIF(metadata->>'user_api_key_user_email', '') IS NOT NULL - ) -ORDER BY api_key, "startTime" DESC NULLS LAST +_DELETED_TOKEN_DIGEST_SQL: Final = """ +SELECT DISTINCT ON (token) + encode(sha256(convert_to(token, 'UTF8')), 'hex') AS digest, key_alias, team_id, user_id +FROM "LiteLLM_DeletedVerificationToken" +WHERE encode(sha256(convert_to(token, 'UTF8')), 'hex') = ANY($1::text[]) +ORDER BY token, deleted_at DESC """ @@ -43,24 +35,18 @@ class KeyMetadataDict(TypedDict, total=False): user_email: ReadOnly[str | None] +class _TokenDigestRow(BaseModel): + digest: str + key_alias: str | None = None + team_id: str | None = None + user_id: str | None = None + + +_TOKEN_DIGEST_ROWS: Final = TypeAdapter(tuple[_TokenDigestRow, ...]) _EMPTY_KEY_METADATA: Final[Mapping[str, KeyMetadataDict]] = MappingProxyType({}) _EMPTY_EMAILS: Final[Mapping[str, str]] = MappingProxyType({}) -class _TokenAliasRecord(Protocol): - @property - def token(self) -> str: ... - - @property - def key_alias(self) -> str | None: ... - - @property - def team_id(self) -> str | None: ... - - @property - def user_id(self) -> str | None: ... - - async def _db_or_empty( load: Callable[[], Awaitable[_T]], warning: str, @@ -75,142 +61,25 @@ async def _db_or_empty( return None -def _record_metadata(record: _TokenAliasRecord) -> KeyMetadataDict: - meta: Final[KeyMetadataDict] = { - "key_alias": record.key_alias, - "team_id": record.team_id, - "user_id": getattr(record, "user_id", None), - } - return meta - - -def _spend_log_row_metadata(row: Mapping[str, object]) -> KeyMetadataDict: - meta: Final[KeyMetadataDict] = { - "key_alias": row.get("key_alias") if isinstance(row.get("key_alias"), str) else None, - "team_id": row.get("team_id") if isinstance(row.get("team_id"), str) else None, - "user_id": row.get("user_id") if isinstance(row.get("user_id"), str) else None, - "user_email": row.get("user_email") if isinstance(row.get("user_email"), str) else None, - } - return meta - - -def _token_digest_metadata( - records: Sequence[_TokenAliasRecord], - wanted: AbstractSet[str], -) -> Mapping[str, KeyMetadataDict]: - return MappingProxyType( - { - digested: _record_metadata(record) - for record in records - for digested in (hash_token(record.token),) - if digested in wanted - } - ) - - -async def _paginate_token_digest_metadata( - load_page: Callable[[int], Awaitable[Sequence[_TokenAliasRecord] | None]], - wanted: AbstractSet[str], - *, - page_size: int, - skip: int = 0, - accumulated: Mapping[str, KeyMetadataDict] = _EMPTY_KEY_METADATA, -) -> Mapping[str, KeyMetadataDict]: - if not wanted: - return accumulated - records: Final = await load_page(skip) - if records is None: - return accumulated - page_hits: Final = _token_digest_metadata(records, wanted) - combined: Final[Mapping[str, KeyMetadataDict]] = ( - MappingProxyType({**accumulated, **page_hits}) if page_hits else accumulated - ) - still_wanted: Final = wanted - frozenset(page_hits) - if not still_wanted or len(records) < page_size: - return combined - return await _paginate_token_digest_metadata( - load_page, - still_wanted, - page_size=page_size, - skip=skip + page_size, - accumulated=combined, - ) - - -async def _reverse_hash_active_key_metadata( - prisma_client: PrismaClient, - wanted: AbstractSet[str], - *, - page_size: int, -) -> Mapping[str, KeyMetadataDict]: - async def load_page(skip: int) -> Sequence[_TokenAliasRecord] | None: - return await _db_or_empty( - lambda: VerificationTokenRepository(prisma_client).table.find_many( - take=page_size, - skip=skip, - order={"token": "asc"}, # mutable-ok: Prisma find_many order= is a dict - ), - "Failed reverse-hash recovery against active keys for %d missing keys: %s", - len(wanted), - ) - - return await _paginate_token_digest_metadata(load_page, wanted, page_size=page_size) - - -async def _reverse_hash_deleted_key_metadata( - prisma_client: PrismaClient, - wanted: AbstractSet[str], - *, - page_size: int, -) -> Mapping[str, KeyMetadataDict]: - async def load_page(skip: int) -> Sequence[_TokenAliasRecord] | None: - return await _db_or_empty( - lambda: DeletedVerificationTokenRepository(prisma_client).table.find_many( - take=page_size, - skip=skip, - order=[{"deleted_at": "desc"}, {"id": "asc"}], # mutable-ok: Prisma find_many order= is a dict - ), - "Failed reverse-hash recovery against deleted keys for %d missing keys: %s", - len(wanted), - ) - - return await _paginate_token_digest_metadata(load_page, wanted, page_size=page_size) - - async def _reverse_hash_key_metadata( prisma_client: PrismaClient, + sql: str, wanted: AbstractSet[str], *, - page_size: int, + warning: str, ) -> Mapping[str, KeyMetadataDict]: - from_active: Final = await _reverse_hash_active_key_metadata(prisma_client, wanted, page_size=page_size) - still_wanted: Final = wanted - frozenset(from_active) - if not still_wanted: - return from_active - from_deleted: Final = await _reverse_hash_deleted_key_metadata(prisma_client, still_wanted, page_size=page_size) - return MappingProxyType({**from_active, **from_deleted}) - - -async def _spend_logs_key_metadata( - prisma_client: PrismaClient, - wanted: AbstractSet[str], -) -> Mapping[str, KeyMetadataDict]: - spend_log_rows: Final = await _db_or_empty( - lambda: prisma_client.db.query_raw( - _SPEND_LOGS_KEY_METADATA_SQL, - tuple(wanted), - ), - "Failed SpendLogs metadata recovery for %d missing keys: %s", + rows: Final = await _db_or_empty( + lambda: prisma_client.db.query_raw(sql, sorted(wanted)), + warning, len(wanted), ) - if not isinstance(spend_log_rows, list): + if rows is None: return _EMPTY_KEY_METADATA - return MappingProxyType( { - row["api_key"]: _spend_log_row_metadata(row) - for row in spend_log_rows - if isinstance(row, dict) and isinstance(row.get("api_key"), str) and row["api_key"] in wanted + row.digest: KeyMetadataDict(key_alias=row.key_alias, team_id=row.team_id, user_id=row.user_id) + for row in _TOKEN_DIGEST_ROWS.validate_python(rows) + if row.digest in wanted } ) @@ -223,7 +92,7 @@ async def _emails_for_user_ids( return _EMPTY_EMAILS users: Final = await _db_or_empty( lambda: UserRepository(prisma_client).table.find_many( - where={"user_id": {"in": tuple(user_ids)}}, # mutable-ok: Prisma find_many where= is a dict + where={"user_id": {"in": list(user_ids)}}, # mutable-ok: Prisma find_many where= is a dict ), "Failed user_email recovery for %d user ids: %s", len(user_ids), @@ -268,31 +137,35 @@ async def attach_user_emails( async def recover_double_hashed_key_metadata( prisma_client: PrismaClient, missing_keys: AbstractSet[str], - *, - token_scan_page_size: int = _TOKEN_SCAN_PAGE, ) -> Mapping[str, KeyMetadataDict]: """ - Recover key_alias/team_id/user_email for DailyUserSpend.api_key values that + Recover key_alias/team_id/user_id for DailyUserSpend.api_key values that were double-hashed by the v1.99 spend-log provenance gate. Those rows store hash(VerificationToken.token) instead of the token, so the - exact join misses. Page through active then deleted tokens until every - wanted digest is found or the table ends; fall back to SpendLogs metadata. - Emails come from SpendLogs when present, otherwise from UserTable via the - recovered key's user_id. + exact join misses. Postgres hashes the token column itself, one pass over + active keys and one over deleted keys, so no key row crosses the wire. """ sha_missing: Final = frozenset(key for key in missing_keys if is_valid_sha256_hash(key)) if not sha_missing: return _EMPTY_KEY_METADATA - from_tokens: Final = await _reverse_hash_key_metadata(prisma_client, sha_missing, page_size=token_scan_page_size) - still_missing: Final = sha_missing - frozenset(from_tokens) - recovered: Final = ( - from_tokens - if not still_missing - else MappingProxyType({**from_tokens, **(await _spend_logs_key_metadata(prisma_client, still_missing))}) + from_active: Final = await _reverse_hash_key_metadata( + prisma_client, + _ACTIVE_TOKEN_DIGEST_SQL, + sha_missing, + warning="Failed reverse-hash recovery against active keys for %d missing keys: %s", ) - return await attach_user_emails(prisma_client, recovered) + still_missing: Final = sha_missing - frozenset(from_active) + if not still_missing: + return from_active + from_deleted: Final = await _reverse_hash_key_metadata( + prisma_client, + _DELETED_TOKEN_DIGEST_SQL, + still_missing, + warning="Failed reverse-hash recovery against deleted keys for %d missing keys: %s", + ) + return MappingProxyType({**from_active, **from_deleted}) def _row_with_recovered_fields( @@ -345,7 +218,10 @@ async def fill_missing_api_key_aliases( if not missing_keys: return tuple(rows) - recovered: Final = await recover_double_hashed_key_metadata(prisma_client, missing_keys) + recovered: Final = await attach_user_emails( + prisma_client, + await recover_double_hashed_key_metadata(prisma_client, missing_keys), + ) if not recovered: return tuple(rows) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 4c7333143c5..a37c3ba4405 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -139,7 +139,6 @@ def _get_spend_logs_metadata( user_api_key_project_alias=None, user_api_key_org_id=None, user_api_key_user_id=None, - user_api_key_user_email=None, user_api_key_team_alias=None, spend_logs_metadata=None, requester_ip_address=None, diff --git a/tests/test_litellm/integrations/cloudzero/test_cloudzero.py b/tests/test_litellm/integrations/cloudzero/test_cloudzero.py index c543156eedd..1b6e8ca513c 100644 --- a/tests/test_litellm/integrations/cloudzero/test_cloudzero.py +++ b/tests/test_litellm/integrations/cloudzero/test_cloudzero.py @@ -74,7 +74,7 @@ class TestCloudZeroHourlyExport: fake_db = MagicMock() async def query_raw_mock(query: str, *params): - if "LiteLLM_SpendLogs" in query: + if "sha256(" in query: return [] start_time_utc = params[0] if len(params) > 0 else None end_time_utc = params[1] if len(params) > 1 else None diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index b9c0d953086..37a54c4901a 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -459,33 +459,23 @@ async def test_get_api_key_metadata_recovers_double_hashed_key_via_reverse_hash( """ v1.99 spend logging re-hashed already-hashed api_key values when provenance was missing. Usage joins DailyUserSpend.api_key to VerificationToken.token, so those - rows looked like key-hash-... with a null alias. Reverse-hash recovery must map - hash(token) back to the key's alias for historical dirty spend. + rows looked like key-hash-... with a null alias. Recovery asks Postgres for the + key whose hashed token matches the dirty value and maps it back to its alias. """ from litellm.proxy.utils import hash_token - token = "a" * 64 - double_hashed = hash_token(token) + double_hashed = hash_token("a" * 64) mock_prisma = MagicMock() - - mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock( - side_effect=[ - [], # exact join miss - [ - SimpleNamespace( - token=token, - key_alias="batch-worker", - team_id="team-1", - user_id="alice", - ) - ], - ] - ) + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) mock_prisma.db.litellm_usertable.find_many = AsyncMock( return_value=[SimpleNamespace(user_id="alice", user_email="alice@example.com")] ) - mock_prisma.db.query_raw = AsyncMock(return_value=[]) + mock_prisma.db.query_raw = AsyncMock( + return_value=[ + {"digest": double_hashed, "key_alias": "batch-worker", "team_id": "team-1", "user_id": "alice"} + ] + ) result = await get_api_key_metadata( prisma_client=mock_prisma, @@ -495,37 +485,37 @@ async def test_get_api_key_metadata_recovers_double_hashed_key_via_reverse_hash( assert result[double_hashed]["key_alias"] == "batch-worker" assert result[double_hashed]["team_id"] == "team-1" assert result[double_hashed]["user_email"] == "alice@example.com" - mock_prisma.db.query_raw.assert_not_called() + ((digest_sql, digests),) = [call.args for call in mock_prisma.db.query_raw.call_args_list] + assert '"LiteLLM_VerificationToken"' in digest_sql + assert digests == [double_hashed] @pytest.mark.asyncio -async def test_get_api_key_metadata_recovers_double_hashed_key_via_spend_logs(): - """When the token tables cannot reverse-hash the dirty key, use SpendLogs metadata.""" +async def test_get_api_key_metadata_permanent_miss_never_pages_tokens_or_reads_spend_logs(): + """A dirty key no table can explain costs two digest lookups, never a token page walk or a SpendLogs scan.""" from litellm.proxy.utils import hash_token double_hashed = hash_token("b" * 64) mock_prisma = MagicMock() mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) - mock_prisma.db.query_raw = AsyncMock( - return_value=[ - { - "api_key": double_hashed, - "key_alias": "from-spend-log", - "team_id": "team-spend", - } - ] - ) mock_prisma.db.litellm_usertable.find_many = AsyncMock(return_value=[]) + mock_prisma.db.query_raw = AsyncMock(return_value=[]) result = await get_api_key_metadata( prisma_client=mock_prisma, api_keys={double_hashed}, ) - assert result[double_hashed]["key_alias"] == "from-spend-log" - assert result[double_hashed]["team_id"] == "team-spend" - mock_prisma.db.query_raw.assert_called_once() + assert double_hashed not in result + issued_sql = [call.args[0] for call in mock_prisma.db.query_raw.call_args_list] + assert len(issued_sql) == 2 + assert not any("LiteLLM_SpendLogs" in sql for sql in issued_sql) + token_lookups = ( + mock_prisma.db.litellm_verificationtoken.find_many.call_args_list + + mock_prisma.db.litellm_deletedverificationtoken.find_many.call_args_list + ) + assert all("take" not in call.kwargs and "skip" not in call.kwargs for call in token_lookups) def test_key_metadata_includes_recovered_user_email(): diff --git a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py index 704de049fc8..43f7d20cf13 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py +++ b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py @@ -1,3 +1,4 @@ +from collections.abc import Sequence from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -11,55 +12,126 @@ from litellm.proxy.spend_tracking.key_metadata_recovery import ( from litellm.proxy.utils import hash_token +def _digest_row(digest: str, key_alias: str, team_id: str | None, user_id: str | None) -> dict[str, str | None]: + return {"digest": digest, "key_alias": key_alias, "team_id": team_id, "user_id": user_id} + + +def _query_raw_by_table( + active_rows: Sequence[dict[str, str | None]], + deleted_rows: Sequence[dict[str, str | None]], +) -> AsyncMock: + async def query_raw(sql: str, *params: object) -> list[dict[str, str | None]]: + if '"LiteLLM_VerificationToken"' in sql: + return list(active_rows) + if '"LiteLLM_DeletedVerificationToken"' in sql: + return list(deleted_rows) + raise AssertionError(f"unexpected query: {sql}") + + return AsyncMock(side_effect=query_raw) + + @pytest.mark.asyncio -async def test_recover_double_hashed_key_metadata_via_reverse_hash(): - token = "a" * 64 - double_hashed = hash_token(token) +async def test_recover_double_hashed_key_metadata_via_active_token_digest(): + double_hashed = hash_token("a" * 64) mock_prisma = MagicMock() - mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock( - return_value=[ - SimpleNamespace( - token=token, - key_alias="batch-worker", - team_id="team-1", - user_id="alice", - ) - ] + mock_prisma.db.query_raw = _query_raw_by_table( + active_rows=[_digest_row(double_hashed, "batch-worker", "team-1", "alice")], + deleted_rows=[], ) - mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) - mock_prisma.db.litellm_usertable.find_many = AsyncMock( - return_value=[SimpleNamespace(user_id="alice", user_email="alice@example.com")] - ) - mock_prisma.db.query_raw = AsyncMock(return_value=[]) result = await recover_double_hashed_key_metadata(mock_prisma, {double_hashed}) assert result[double_hashed]["key_alias"] == "batch-worker" assert result[double_hashed]["team_id"] == "team-1" - assert result[double_hashed]["user_email"] == "alice@example.com" + assert result[double_hashed]["user_id"] == "alice" + ((_, digests),) = [call.args for call in mock_prisma.db.query_raw.call_args_list] + assert digests == [double_hashed] + + +@pytest.mark.asyncio +async def test_recover_double_hashed_key_metadata_falls_back_to_deleted_tokens(): + double_hashed = hash_token("y" * 64) + mock_prisma = MagicMock() + mock_prisma.db.query_raw = _query_raw_by_table( + active_rows=[], + deleted_rows=[_digest_row(double_hashed, "deleted-key", "team-del", "erin")], + ) + + result = await recover_double_hashed_key_metadata(mock_prisma, {double_hashed}) + + assert result[double_hashed]["key_alias"] == "deleted-key" + assert result[double_hashed]["team_id"] == "team-del" + assert result[double_hashed]["user_id"] == "erin" + assert [call.args[1] for call in mock_prisma.db.query_raw.call_args_list] == [[double_hashed], [double_hashed]] + + +@pytest.mark.asyncio +async def test_recover_only_asks_deleted_tokens_for_digests_active_keys_missed(): + found_active = hash_token("1" * 64) + found_deleted = hash_token("2" * 64) + mock_prisma = MagicMock() + mock_prisma.db.query_raw = _query_raw_by_table( + active_rows=[_digest_row(found_active, "active-key", None, None)], + deleted_rows=[_digest_row(found_deleted, "deleted-key", None, None)], + ) + + result = await recover_double_hashed_key_metadata(mock_prisma, {found_active, found_deleted}) + + assert result[found_active]["key_alias"] == "active-key" + assert result[found_deleted]["key_alias"] == "deleted-key" + assert [call.args[1] for call in mock_prisma.db.query_raw.call_args_list] == [ + sorted((found_active, found_deleted)), + [found_deleted], + ] + + +@pytest.mark.asyncio +async def test_recover_permanent_miss_costs_two_digest_lookups_and_no_table_walk(): + double_hashed = hash_token("b" * 64) + mock_prisma = MagicMock() + mock_prisma.db.query_raw = _query_raw_by_table(active_rows=[], deleted_rows=[]) + + result = await recover_double_hashed_key_metadata(mock_prisma, {double_hashed}) + + assert result == {} + assert len(mock_prisma.db.query_raw.call_args_list) == 2 + mock_prisma.db.litellm_verificationtoken.find_many.assert_not_called() + mock_prisma.db.litellm_deletedverificationtoken.find_many.assert_not_called() + + +@pytest.mark.asyncio +async def test_recover_skips_keys_that_are_not_sha256_digests(): + mock_prisma = MagicMock() + mock_prisma.db.query_raw = AsyncMock(return_value=[]) + + result = await recover_double_hashed_key_metadata(mock_prisma, {"sk-plain-key", "key-hash-short"}) + + assert result == {} mock_prisma.db.query_raw.assert_not_called() @pytest.mark.asyncio -async def test_fill_missing_api_key_aliases_updates_null_alias_and_email_rows(): - token = "c" * 64 - double_hashed = hash_token(token) +async def test_recover_returns_empty_when_digest_lookup_raises_prisma_error(): + double_hashed = hash_token("c" * 64) mock_prisma = MagicMock() - mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock( - return_value=[ - SimpleNamespace( - token=token, - key_alias="recovered-alias", - team_id="team-9", - user_id="bob", - ) - ] + mock_prisma.db.query_raw = AsyncMock(side_effect=PrismaError("db down")) + + result = await recover_double_hashed_key_metadata(mock_prisma, {double_hashed}) + + assert result == {} + + +@pytest.mark.asyncio +async def test_fill_missing_api_key_aliases_updates_null_alias_and_email_rows(): + double_hashed = hash_token("d" * 64) + mock_prisma = MagicMock() + mock_prisma.db.query_raw = _query_raw_by_table( + active_rows=[_digest_row(double_hashed, "recovered-alias", "team-9", "bob")], + deleted_rows=[], ) - mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) mock_prisma.db.litellm_usertable.find_many = AsyncMock( return_value=[SimpleNamespace(user_id="bob", user_email="bob@example.com")] ) - mock_prisma.db.query_raw = AsyncMock(return_value=[]) rows = ( { @@ -83,104 +155,18 @@ async def test_fill_missing_api_key_aliases_updates_null_alias_and_email_rows(): assert filled[0]["api_key_alias"] == "recovered-alias" assert filled[0]["team_id"] == "team-9" assert filled[0]["user_email"] == "bob@example.com" + assert filled[0]["spend"] == 12.5 assert filled[1]["api_key_alias"] == "named-key" + assert mock_prisma.db.litellm_usertable.find_many.call_args.kwargs["where"] == {"user_id": {"in": ["bob"]}} @pytest.mark.asyncio -async def test_recover_falls_back_to_spend_logs_when_token_scan_raises_prisma_error(): - token = "b" * 64 - double_hashed = hash_token(token) +async def test_fill_missing_api_key_aliases_leaves_rows_untouched_when_nothing_is_missing(): mock_prisma = MagicMock() - mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(side_effect=PrismaError("db down")) - mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(side_effect=PrismaError("db down")) - mock_prisma.db.litellm_usertable.find_many = AsyncMock(return_value=[]) - mock_prisma.db.query_raw = AsyncMock( - return_value=[ - { - "api_key": double_hashed, - "key_alias": "from-spend-logs", - "team_id": "team-sl", - "user_id": "carol", - "user_email": "carol@example.com", - } - ] - ) - - result = await recover_double_hashed_key_metadata(mock_prisma, {double_hashed}) - - assert result[double_hashed]["key_alias"] == "from-spend-logs" - assert result[double_hashed]["team_id"] == "team-sl" - assert result[double_hashed]["user_email"] == "carol@example.com" - - -@pytest.mark.asyncio -async def test_recover_double_hashed_key_metadata_scans_past_first_page(): - token = "z" * 64 - double_hashed = hash_token(token) - decoys = ( - SimpleNamespace(token="1" * 64, key_alias="decoy-1", team_id=None, user_id=None), - SimpleNamespace(token="2" * 64, key_alias="decoy-2", team_id=None, user_id=None), - ) - match = SimpleNamespace(token=token, key_alias="late-key", team_id="team-late", user_id="dana") - - async def find_many(*, take: int | None = None, skip: int | None = None, order: object = None): - if skip == 0: - return list(decoys) - if skip == 2: - return [match] - return [] - - mock_prisma = MagicMock() - mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(side_effect=find_many) - mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) - mock_prisma.db.litellm_usertable.find_many = AsyncMock( - return_value=[SimpleNamespace(user_id="dana", user_email="dana@example.com")] - ) mock_prisma.db.query_raw = AsyncMock(return_value=[]) + rows = ({"api_key": hash_token("e" * 64), "api_key_alias": "named", "user_email": "x@example.com"},) - result = await recover_double_hashed_key_metadata(mock_prisma, {double_hashed}, token_scan_page_size=2) + filled = await fill_missing_api_key_aliases(mock_prisma, rows) - assert result[double_hashed]["key_alias"] == "late-key" - assert result[double_hashed]["team_id"] == "team-late" - assert result[double_hashed]["user_email"] == "dana@example.com" - assert [call.kwargs["skip"] for call in mock_prisma.db.litellm_verificationtoken.find_many.call_args_list] == [0, 2] - mock_prisma.db.litellm_deletedverificationtoken.find_many.assert_not_called() - mock_prisma.db.query_raw.assert_not_called() - - -@pytest.mark.asyncio -async def test_recover_double_hashed_key_metadata_pages_deleted_tokens(): - token = "y" * 64 - double_hashed = hash_token(token) - decoys = ( - SimpleNamespace(token="3" * 64, key_alias="deleted-decoy-1", team_id=None, user_id=None), - SimpleNamespace(token="4" * 64, key_alias="deleted-decoy-2", team_id=None, user_id=None), - ) - match = SimpleNamespace(token=token, key_alias="deleted-late-key", team_id="team-del", user_id="erin") - - async def find_deleted(*, take: int | None = None, skip: int | None = None, order: object = None): - if skip == 0: - return list(decoys) - if skip == 2: - return [match] - return [] - - mock_prisma = MagicMock() - mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) - mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(side_effect=find_deleted) - mock_prisma.db.litellm_usertable.find_many = AsyncMock( - return_value=[SimpleNamespace(user_id="erin", user_email="erin@example.com")] - ) - mock_prisma.db.query_raw = AsyncMock(return_value=[]) - - result = await recover_double_hashed_key_metadata(mock_prisma, {double_hashed}, token_scan_page_size=2) - - assert result[double_hashed]["key_alias"] == "deleted-late-key" - assert result[double_hashed]["user_email"] == "erin@example.com" - assert [ - call.kwargs["skip"] for call in mock_prisma.db.litellm_deletedverificationtoken.find_many.call_args_list - ] == [ - 0, - 2, - ] + assert filled == rows mock_prisma.db.query_raw.assert_not_called() diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index dfdad545003..73a29afd9b9 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -511,7 +511,6 @@ ignored_keys = [ "metadata.user_api_key_project_alias", "metadata.user_api_key_org_id", "metadata.user_api_key_user_id", - "metadata.user_api_key_user_email", "metadata.user_api_key_team_alias", "metadata.spend_logs_metadata", "metadata.requester_ip_address", diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 3cfc16eedec..fb0cdc175b1 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -2729,7 +2729,6 @@ def test_get_logging_payload_batch_attribution_keeps_verification_token_hash(): "user_api_key_hash": token_hash, "user_api_key_alias": "batch-creator", "user_api_key_user_id": "alice", - "user_api_key_user_email": "alice@example.com", "user_api_key_team_id": "team-1", } }, @@ -2746,7 +2745,6 @@ def test_get_logging_payload_batch_attribution_keeps_verification_token_hash(): parsed_meta = json.loads(payload["metadata"]) assert parsed_meta["user_api_key"] == token_hash assert parsed_meta["user_api_key_alias"] == "batch-creator" - assert parsed_meta["user_api_key_user_email"] == "alice@example.com" def test_get_spend_logs_metadata_provenance_bypass_requires_hash_match(): From 33815682bc20ec5b32d9b363995608fc29ee357a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 4 Sep 2026 01:38:26 +0000 Subject: [PATCH 45/88] fix(spend): keep CloudZero and Focus spend-user email when filling alias Export rows already join user_email from DailyUserSpend.user_id. Recovered key-owner email must not replace that when only the alias join missed. Co-authored-by: Mateo Wang --- .../spend_tracking/key_metadata_recovery.py | 2 +- .../test_key_metadata_recovery.py | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index d524b158c9c..2940c9a8337 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -186,7 +186,7 @@ def _row_with_recovered_fields( **row, alias_field: meta.get("key_alias") or row.get(alias_field), team_id_field: meta.get("team_id") or row.get(team_id_field), - user_email_field: meta.get("user_email") or row.get(user_email_field), + user_email_field: row.get(user_email_field) or meta.get("user_email"), } ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py index 43f7d20cf13..d76d355ec0e 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py +++ b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py @@ -170,3 +170,32 @@ async def test_fill_missing_api_key_aliases_leaves_rows_untouched_when_nothing_i assert filled == rows mock_prisma.db.query_raw.assert_not_called() + + +@pytest.mark.asyncio +async def test_fill_missing_api_key_aliases_keeps_spend_user_email_when_alias_is_missing(): + double_hashed = hash_token("f" * 64) + mock_prisma = MagicMock() + mock_prisma.db.query_raw = _query_raw_by_table( + active_rows=[_digest_row(double_hashed, "team-key", "team-9", "key-owner")], + deleted_rows=[], + ) + mock_prisma.db.litellm_usertable.find_many = AsyncMock( + return_value=[SimpleNamespace(user_id="key-owner", user_email="owner@example.com")] + ) + + rows = ( + { + "api_key": double_hashed, + "api_key_alias": None, + "team_id": None, + "user_email": "spender@example.com", + "spend": 4.0, + }, + ) + + filled = await fill_missing_api_key_aliases(mock_prisma, rows) + + assert filled[0]["api_key_alias"] == "team-key" + assert filled[0]["team_id"] == "team-9" + assert filled[0]["user_email"] == "spender@example.com" From 048499cdf55959753c04aae7d4be995cc3606a2c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 4 Sep 2026 01:44:18 +0000 Subject: [PATCH 46/88] fix(spend): only reverse-hash export rows whose key alias join missed Team and service keys often have no user_email after a successful token join. Treating empty email as a miss hashed every verification token on routine CloudZero and Focus exports. Co-authored-by: Mateo Wang --- .../spend_tracking/key_metadata_recovery.py | 4 +--- .../test_key_metadata_recovery.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index 2940c9a8337..7de18521edd 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -211,9 +211,7 @@ async def fill_missing_api_key_aliases( key for row in rows for key in (row.get(api_key_field),) - if isinstance(key, str) - and key - and (row.get(alias_field) in (None, "") or row.get(user_email_field) in (None, "")) + if isinstance(key, str) and key and row.get(alias_field) in (None, "") ) if not missing_keys: return tuple(rows) diff --git a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py index d76d355ec0e..7a80319239d 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py +++ b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py @@ -199,3 +199,22 @@ async def test_fill_missing_api_key_aliases_keeps_spend_user_email_when_alias_is assert filled[0]["api_key_alias"] == "team-key" assert filled[0]["team_id"] == "team-9" assert filled[0]["user_email"] == "spender@example.com" + + +@pytest.mark.asyncio +async def test_fill_missing_api_key_aliases_skips_named_keys_that_have_no_email(): + mock_prisma = MagicMock() + mock_prisma.db.query_raw = AsyncMock(return_value=[]) + rows = ( + { + "api_key": hash_token("g" * 64), + "api_key_alias": "service-key", + "team_id": "team-svc", + "user_email": None, + }, + ) + + filled = await fill_missing_api_key_aliases(mock_prisma, rows) + + assert filled == rows + mock_prisma.db.query_raw.assert_not_called() From 2b7e14872f7d4bb776ef60c8168bd7e78778a86b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 18:46:01 -0700 Subject: [PATCH 47/88] fix(spend-tracking): hand plain dict rows to polars in the CloudZero and Focus exports --- litellm/integrations/cloudzero/database.py | 2 +- litellm/integrations/focus/database.py | 2 +- .../integrations/cloudzero/test_cloudzero.py | 24 +++++++++++++++++++ .../integrations/focus/test_focus_database.py | 22 +++++++++++++++++ 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/cloudzero/database.py b/litellm/integrations/cloudzero/database.py index 4adf725fd0f..2fb10ad8a96 100644 --- a/litellm/integrations/cloudzero/database.py +++ b/litellm/integrations/cloudzero/database.py @@ -106,6 +106,6 @@ class LiteLLMDatabase: else [] ) recovered_rows: Final = await fill_missing_api_key_aliases(client, usage_rows) - return pl.DataFrame(tuple(recovered_rows), infer_schema_length=None) + return pl.DataFrame([dict(row) for row in recovered_rows], infer_schema_length=None) except Exception as e: raise Exception(f"Error retrieving usage data: {e}") diff --git a/litellm/integrations/focus/database.py b/litellm/integrations/focus/database.py index f214aa02b5b..657c7e0d264 100644 --- a/litellm/integrations/focus/database.py +++ b/litellm/integrations/focus/database.py @@ -108,7 +108,7 @@ class FocusLiteLLMDatabase: else [] ) recovered_rows: Final = await fill_missing_api_key_aliases(client, usage_rows) - return pl.DataFrame(tuple(recovered_rows), infer_schema_length=None) + return pl.DataFrame([dict(row) for row in recovered_rows], infer_schema_length=None) except Exception as exc: raise RuntimeError(f"Error retrieving usage data: {exc}") from exc diff --git a/tests/test_litellm/integrations/cloudzero/test_cloudzero.py b/tests/test_litellm/integrations/cloudzero/test_cloudzero.py index 1b6e8ca513c..6ddb8cbaa7c 100644 --- a/tests/test_litellm/integrations/cloudzero/test_cloudzero.py +++ b/tests/test_litellm/integrations/cloudzero/test_cloudzero.py @@ -1,3 +1,4 @@ +import hashlib from datetime import datetime from unittest.mock import AsyncMock, MagicMock, patch @@ -165,3 +166,26 @@ class TestCloudZeroHourlyExport: logger = CloudZeroLogger(api_key="test", connection_id="test") await logger._hourly_usage_data_export() + + +class TestLiteLLMDatabaseUsageData: + @pytest.mark.asyncio + async def test_builds_frame_from_rows_recovered_for_double_hashed_keys(self, monkeypatch: pytest.MonkeyPatch): + double_hashed = hashlib.sha256(b"sk-hashed-token").hexdigest() + joined_row = {"api_key": "sk-joined", "api_key_alias": "joined", "team_id": "team-0", "user_email": None, "spend": 0.1} + dirty_row = {"api_key": double_hashed, "api_key_alias": None, "team_id": None, "user_email": None, "spend": 0.5} + + async def query_raw(query: str, *params): + if "sha256(" in query: + return [{"digest": double_hashed, "key_alias": "batch-worker", "team_id": "team-1", "user_id": None}] + return [joined_row, dirty_row] + + fake_client = MagicMock() + fake_client.db.query_raw = AsyncMock(side_effect=query_raw) + db = LiteLLMDatabase() + monkeypatch.setattr(db, "_ensure_prisma_client", lambda: fake_client) + + result = await db.get_usage_data() + + assert result["api_key_alias"].to_list() == ["joined", "batch-worker"] + assert result["team_id"].to_list() == ["team-0", "team-1"] diff --git a/tests/test_litellm/integrations/focus/test_focus_database.py b/tests/test_litellm/integrations/focus/test_focus_database.py index 5c13665f1f1..06240eac387 100644 --- a/tests/test_litellm/integrations/focus/test_focus_database.py +++ b/tests/test_litellm/integrations/focus/test_focus_database.py @@ -1,5 +1,6 @@ """Tests for FocusLiteLLMDatabase query construction.""" +import hashlib from datetime import datetime, timezone from types import SimpleNamespace from unittest.mock import AsyncMock @@ -87,3 +88,24 @@ async def test_should_join_organization_table(monkeypatch: pytest.MonkeyPatch): ) assert "ot.organization_alias as organization_alias" in query_text assert 'LEFT JOIN "LiteLLM_OrganizationTable" ot' in query_text + + +@pytest.mark.asyncio +async def test_should_build_frame_from_rows_recovered_for_double_hashed_keys(monkeypatch: pytest.MonkeyPatch): + double_hashed = hashlib.sha256(b"sk-hashed-token").hexdigest() + joined_row = {"api_key": "sk-joined", "api_key_alias": "joined", "team_id": "team-0", "user_email": None, "spend": 0.1} + dirty_row = {"api_key": double_hashed, "api_key_alias": None, "team_id": None, "user_email": None, "spend": 0.5} + + async def query_raw(query: str, *params): + if "sha256(" in query: + return [{"digest": double_hashed, "key_alias": "batch-worker", "team_id": "team-1", "user_id": None}] + return [joined_row, dirty_row] + + mock_client = SimpleNamespace(db=SimpleNamespace(query_raw=AsyncMock(side_effect=query_raw))) + db = FocusLiteLLMDatabase() + monkeypatch.setattr(db, "_ensure_prisma_client", lambda: mock_client) + + result = await db.get_usage_data() + + assert result["api_key_alias"].to_list() == ["joined", "batch-worker"] + assert result["team_id"].to_list() == ["team-0", "team-1"] From ddc5d8dc37ad4b3a2c4f14c10c9839b3d94e092c Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 3 Sep 2026 19:10:53 -0700 Subject: [PATCH 48/88] fix(router): bound auto-router classifier latency --- .../complexity_router/complexity_router.py | 27 ++++--- .../router_strategy/test_complexity_router.py | 71 +++++++++++++++++++ 2 files changed, 88 insertions(+), 10 deletions(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 17e3d1256d0..9bdcb45a789 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1694,16 +1694,23 @@ class ComplexityRouter(CustomLogger): } } - response: Final[ModelResponse] = await self.litellm_router_instance.acompletion( - model=llm_config.model, - messages=messages_for_call, - response_format=response_format, - timeout=llm_config.timeout_ms / 1000, - metadata=metadata, - proxy_server_request=proxy_server_request, - turn_off_message_logging=turn_off_message_logging, - **classifier_call_params, - **_parent_session_kwargs(request_kwargs), + classifier_timeout_s: Final[float] = llm_config.timeout_ms / 1000 + response: Final[ModelResponse] = await asyncio.wait_for( + self.litellm_router_instance.acompletion( + model=llm_config.model, + messages=messages_for_call, + stream=False, + response_format=response_format, + timeout=classifier_timeout_s, + num_retries=0, + disable_fallbacks=True, + metadata=metadata, + proxy_server_request=proxy_server_request, + turn_off_message_logging=turn_off_message_logging, + **classifier_call_params, + **_parent_session_kwargs(request_kwargs), + ), + timeout=classifier_timeout_s, ) content: Final = response.choices[0].message.content if not content: diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index da3791da39a..17594f7d444 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1994,6 +1994,77 @@ class TestLLMClassifier: assert outcome.cause == "llm_classifier" assert outcome.classifier_cost == pytest.approx(1.35e-05) + @pytest.mark.asyncio + async def test_aclassify_timeout_does_not_inherit_router_retries_or_fallbacks( + self, llm_classifier_config + ): + real_router = Router( + model_list=[ + { + "model_name": "haiku-classifier", + "litellm_params": { + "model": "openai/mock-classifier", + "api_key": "mock-key", + "mock_timeout": True, + }, + }, + { + "model_name": "backup-classifier", + "litellm_params": { + "model": "openai/mock-backup-classifier", + "api_key": "mock-key", + "mock_response": '{"tier": "COMPLEX"}', + }, + }, + ], + num_retries=2, + fallbacks=[{"haiku-classifier": ["backup-classifier"]}], + ) + config = { + **llm_classifier_config, + "classifier_llm_config": {"model": "haiku-classifier", "timeout_ms": 10}, + } + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=real_router, + complexity_router_config=config, + ) + + outcome = await router.aclassify("hi") + + assert outcome.cause == "heuristic_scorer" + assert real_router.total_calls["openai/mock-classifier"] == 1 + assert real_router.total_calls["openai/mock-backup-classifier"] == 0 + + @pytest.mark.asyncio + async def test_aclassify_enforces_total_classifier_deadline( + self, mock_router_instance, llm_classifier_config + ): + cancelled = asyncio.Event() + + async def slow_classifier(**_kwargs: object) -> None: + try: + await asyncio.sleep(1) + except asyncio.CancelledError: + cancelled.set() + raise + + mock_router_instance.acompletion = AsyncMock(side_effect=slow_classifier) + config = { + **llm_classifier_config, + "classifier_llm_config": {"model": "haiku-classifier", "timeout_ms": 10}, + } + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=config, + ) + + outcome = await router.aclassify("hi") + + assert outcome.cause == "heuristic_scorer" + assert cancelled.is_set() + @pytest.mark.asyncio async def test_aclassify_classifier_cost_is_none_when_call_is_unpriced( self, llm_complexity_router, mock_router_instance From d671e0ea5de884a6ef17b7e925ae8d10e408cb8f Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 3 Sep 2026 19:20:47 -0700 Subject: [PATCH 49/88] fix(router): honor explicit retry opt-out --- litellm/router.py | 2 +- .../test_router_per_deployment_num_retries.py | 23 ++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 6da201725b6..6c7611c6236 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7513,7 +7513,7 @@ class Router: # Check retry policy FIRST, before should_retry_this_error # This allows retry policies to override the healthy deployments check _retry_policy_applies = False - if self.retry_policy is not None or model_group_retry_policy is not None: + if request_num_retries != 0 and (self.retry_policy is not None or model_group_retry_policy is not None): # get num_retries from retry policy # Use the model_group captured at the start of the function, or get it from metadata # kwargs.get("model") at this point is the deployment model, not the model_group diff --git a/tests/test_litellm/test_router_per_deployment_num_retries.py b/tests/test_litellm/test_router_per_deployment_num_retries.py index 1bf5781c2d0..44f0ca319be 100644 --- a/tests/test_litellm/test_router_per_deployment_num_retries.py +++ b/tests/test_litellm/test_router_per_deployment_num_retries.py @@ -490,7 +490,7 @@ class TestRequestNumRetriesBeatsGlobal: litellm.callbacks = prev_callbacks @staticmethod - def _router(global_num_retries): + def _router(global_num_retries, retry_policy=None): return Router( model_list=[ { @@ -503,6 +503,7 @@ class TestRequestNumRetriesBeatsGlobal: } ], num_retries=global_num_retries, + retry_policy=retry_policy, ) async def _count_attempts(self, *, global_num_retries, request_num_retries): @@ -530,6 +531,26 @@ class TestRequestNumRetriesBeatsGlobal: attempts = await self._count_attempts(global_num_retries=3, request_num_retries=0) assert attempts == 1 + @pytest.mark.asyncio + async def test_request_num_retries_zero_disables_retry_policy(self): + """An explicit zero remains a single attempt when a retry policy matches the error.""" + counter = _AttemptCounter() + litellm.callbacks = [counter] + router = self._router( + global_num_retries=3, + retry_policy=RetryPolicy(InternalServerErrorRetries=2), + ) + + with patch("asyncio.sleep", return_value=None): + with pytest.raises(litellm.InternalServerError): + await router.acompletion( + model="mock", + messages=[{"role": "user", "content": "hi"}], + num_retries=0, + ) + + assert counter.attempts == 1 + @pytest.mark.asyncio async def test_global_num_retries_applies_when_request_omits_it(self): """No request num_retries -> the global still applies: 1 initial + 3 retries = 4.""" From c0a401947a835ce88f7f4ffb91976d58add91c21 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 3 Sep 2026 19:29:54 -0700 Subject: [PATCH 50/88] test(router): cover retry policy opt-out --- .../test_router_per_deployment_num_retries.py | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/test_litellm/test_router_per_deployment_num_retries.py b/tests/test_litellm/test_router_per_deployment_num_retries.py index 44f0ca319be..d75e32a1821 100644 --- a/tests/test_litellm/test_router_per_deployment_num_retries.py +++ b/tests/test_litellm/test_router_per_deployment_num_retries.py @@ -490,7 +490,7 @@ class TestRequestNumRetriesBeatsGlobal: litellm.callbacks = prev_callbacks @staticmethod - def _router(global_num_retries, retry_policy=None): + def _router(global_num_retries): return Router( model_list=[ { @@ -503,7 +503,6 @@ class TestRequestNumRetriesBeatsGlobal: } ], num_retries=global_num_retries, - retry_policy=retry_policy, ) async def _count_attempts(self, *, global_num_retries, request_num_retries): @@ -534,22 +533,32 @@ class TestRequestNumRetriesBeatsGlobal: @pytest.mark.asyncio async def test_request_num_retries_zero_disables_retry_policy(self): """An explicit zero remains a single attempt when a retry policy matches the error.""" - counter = _AttemptCounter() - litellm.callbacks = [counter] - router = self._router( - global_num_retries=3, - retry_policy=RetryPolicy(InternalServerErrorRetries=2), + router = Router( + model_list=[ + { + "model_name": "mock", + "litellm_params": { + "model": "openai/mock-timeout", + "api_key": "sk-fake", + "mock_timeout": True, + }, + } + ], + num_retries=3, + retry_after=0, + retry_policy=RetryPolicy(TimeoutErrorRetries=2), ) with patch("asyncio.sleep", return_value=None): - with pytest.raises(litellm.InternalServerError): + with pytest.raises(litellm.Timeout): await router.acompletion( model="mock", messages=[{"role": "user", "content": "hi"}], + timeout=0.001, num_retries=0, ) - assert counter.attempts == 1 + assert router.total_calls["openai/mock-timeout"] == 1 @pytest.mark.asyncio async def test_global_num_retries_applies_when_request_omits_it(self): From 510424c86c80b24b418ca852304de6a27c5a396d Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 3 Sep 2026 19:59:14 -0700 Subject: [PATCH 51/88] feat(router): add classifier circuit breaker --- .../complexity_router/README.md | 9 ++ .../complexity_router/complexity_router.py | 97 ++++++++++++++++++- .../complexity_router/config.py | 17 ++++ .../router_strategy/test_complexity_router.py | 95 +++++++++++++++++- .../add_model/ClassificationMethodConfig.tsx | 5 + .../ClassifierCircuitBreakerConfig.tsx | 69 +++++++++++++ .../add_model/ComplexityRouterConfig.test.tsx | 32 ++++++ .../add_model/ComplexityRouterConfig.tsx | 2 + .../build_complexity_router_config.test.ts | 15 +++ .../build_complexity_router_config.ts | 19 +++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 12 +++ 11 files changed, 363 insertions(+), 9 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/add_model/ClassifierCircuitBreakerConfig.tsx diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index afa27719064..2d66d28a93e 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -275,6 +275,15 @@ model_list: keep the classifier deployment or provider default, or set a supported value such as `none` or `low` to override that call. +Classifier calls have a one-attempt hard deadline. After a timeout, the router opens a process-local +circuit for that classifier and sends every session through `classifier_fallback` for +`classifier_llm_config.circuit_breaker_cooldown_seconds` (30 seconds by default). When the cooldown +expires, one request probes the classifier while concurrent requests continue through the fallback. +A successful probe closes the circuit; a failed probe restarts the cooldown. The circuit breaker is +on by default; set `classifier_llm_config.circuit_breaker_enabled: false` to disable it. The default +fallback is the local heuristic scorer, so a classifier outage does not repeat its timeout across +every turn or session handled by the router process. + A request short-circuits, meaning it routes on the scorer's own tier with no classifier call, when two things hold: the scorer landed at or below `heuristic_first_max_tier`, and it produced at least one signal. Everything else goes to the classifier, which then decides as it normally would. diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 9bdcb45a789..9176b3da02a 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -18,8 +18,10 @@ from __future__ import annotations import asyncio import random import re -from collections.abc import Iterator, Mapping, Sequence +import time +from collections.abc import Callable, Iterator, Mapping, Sequence from itertools import accumulate, islice, takewhile +from threading import Lock from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast @@ -816,6 +818,61 @@ class ClassificationOutcome(NamedTuple): classifier_cost: float | None = None +class _ClassifierCircuitBreaker: + """Process-local timeout breaker for one complexity-router classifier. + + The router instance serves every session assigned to that auto-router deployment, so the + breaker prevents one unhealthy classifier from charging the same timeout to each session. + Exactly one request becomes the recovery probe after the cooldown; the lock makes that state + transition atomic even when several request tasks arrive together. + """ + + CLOSED: Final = "closed" + OPEN: Final = "open" + HALF_OPEN: Final = "half_open" + + def __init__(self, cooldown_seconds: float, clock: Callable[[], float] = time.monotonic) -> None: + self._cooldown_seconds = cooldown_seconds + self._clock = clock + self._state = self.CLOSED + self._opened_at: float | None = None + self._lock = Lock() + + def allow_request(self) -> bool: + """Allow ordinary calls while closed and exactly one probe after cooldown.""" + with self._lock: + if self._state == self.CLOSED: + return True + if self._state == self.HALF_OPEN: + return False + opened_at: Final = self._opened_at + if opened_at is not None and self._clock() - opened_at >= self._cooldown_seconds: + self._state = self.HALF_OPEN + return True + return False + + def record_success(self) -> None: + with self._lock: + self._state = self.CLOSED + self._opened_at = None + + def record_failure(self, *, is_timeout: bool) -> None: + """Open on a normal timeout, or reopen when the single recovery probe fails.""" + with self._lock: + if not is_timeout and self._state != self.HALF_OPEN: + return + self._state = self.OPEN + self._opened_at = self._clock() + + +def _is_classifier_timeout(exc: BaseException) -> bool: + if isinstance(exc, TimeoutError): + return True + from litellm.exceptions import Timeout as LiteLLMTimeout + + return isinstance(exc, LiteLLMTimeout) + + def _allowed(models: tuple[str, ...], fit_filter: frozenset[str] | None) -> tuple[str, ...]: return models if fit_filter is None else tuple(model for model in models if model in fit_filter) @@ -993,6 +1050,15 @@ class ComplexityRouter(CustomLogger): if llm_classifier_configured else None ) + self._classifier_circuit_breaker: _ClassifierCircuitBreaker | None = ( + _ClassifierCircuitBreaker(self.config.classifier_llm_config.circuit_breaker_cooldown_seconds) + if ( + llm_classifier_configured + and self.config.classifier_llm_config is not None + and self.config.classifier_llm_config.circuit_breaker_enabled + ) + else None + ) self._tier_success_predictor: TierSuccessPredictor | None = ( TierSuccessPredictor(resolve_tier_artifact(self.config.heuristic_v2_artifact)) if self.config.classifier_type == "heuristic_v2" @@ -1474,8 +1540,19 @@ class ComplexityRouter(CustomLogger): `scored` is the heuristic outcome the caller already computed, which only "heuristic_first" has. It is handed to the failure path so a classifier error does not re-run the scorer. """ + breaker: Final = self._classifier_circuit_breaker + if breaker is not None and not breaker.allow_request(): + return self._classifier_failure_outcome( + "LLM classifier circuit is open", + prompt, + system_prompt, + scored, + signal="classifier-circuit-open", + ) try: tier, classifier_cost = await self._classify_with_llm(prompt, system_prompt, request_kwargs, messages) + if breaker is not None: + breaker.record_success() return ClassificationOutcome( tier=tier, score=None, @@ -1484,6 +1561,8 @@ class ComplexityRouter(CustomLogger): classifier_cost=classifier_cost, ) except Exception as e: # noqa: BLE001 -- external LLM call can fail in many distinct ways (timeout, provider error, validation, parse error); any failure must fall back to the configured fallback path + if breaker is not None: + breaker.record_failure(is_timeout=_is_classifier_timeout(e)) return self._classifier_failure_outcome(f"LLM classifier failed ({e})", prompt, system_prompt, scored) def _classifier_failure_outcome( @@ -1492,6 +1571,7 @@ class ComplexityRouter(CustomLogger): prompt: str, system_prompt: str | None, scored: ClassificationOutcome | None = None, + signal: str | None = None, ) -> ClassificationOutcome: """The outcome when the LLM classifier or classifier plugin produced no usable tier: fallback_tier on a custom tier set, classifier_fallback otherwise. @@ -1501,21 +1581,28 @@ class ComplexityRouter(CustomLogger): fallback_tier: Final = self.config.fallback_tier if fallback_tier is not None: verbose_router_logger.warning("ComplexityRouter: %s, routing to fallback_tier %s", reason, fallback_tier) - return ClassificationOutcome( + outcome: Final = ClassificationOutcome( tier=fallback_tier, score=None, signals=(f"classifier-fallback:{fallback_tier}",), cause="classifier_fallback", ) + return outcome if signal is None else outcome._replace(signals=(*outcome.signals, signal)) verbose_router_logger.warning( "ComplexityRouter: %s, falling back to %s", reason, self.config.classifier_fallback ) if self.config.classifier_fallback == "default_model": - return self._default_model_fallback_outcome() + outcome = self._default_model_fallback_outcome() + return outcome if signal is None else outcome._replace(signals=(*outcome.signals, signal)) if scored is not None: - return scored + return scored if signal is None else scored._replace(signals=(*scored.signals, signal)) tier, score, signals, cause = self._score_and_classify(prompt, system_prompt) - return ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause) + return ClassificationOutcome( + tier=tier, + score=score, + signals=signals if signal is None else (*signals, signal), + cause=cause, + ) async def _classify_with_plugin( self, diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 70c1b281e31..fa086c57687 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -444,6 +444,23 @@ class ClassifierLLMConfig(BaseModel): default=3000, description="Timeout budget for the classification call, in milliseconds", ) + circuit_breaker_enabled: bool = Field( + default=True, + description=( + "Whether one classifier timeout temporarily sends requests through classifier_fallback. " + "Enabled by default so an unhealthy classifier cannot repeat its timeout across sessions." + ), + ) + circuit_breaker_cooldown_seconds: float = Field( + default=30.0, + gt=0.0, + description=( + "How long to skip this router's LLM classifier after a classification call times out. " + "Requests use classifier_fallback during the cooldown. When it expires, one request " + "probes the classifier while concurrent requests keep using the fallback; a successful " + "probe closes the circuit and a failed probe restarts the cooldown." + ), + ) classification_rubric: ClassificationRubric | None = Field( default=None, description=( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 17594f7d444..d55cabd6806 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -14,7 +14,6 @@ from pydantic import ValidationError import litellm from litellm import Router -from litellm.router_utils.auto_router_model_naming import count_heuristic_v2_routers from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY @@ -26,6 +25,7 @@ from litellm.router_strategy.complexity_router.complexity_router import ( DimensionScore, KeywordOverride, _built_in_prompt, + _ClassifierCircuitBreaker, _matched_plan_mode_sentinel, classification_system_prompt, ) @@ -43,6 +43,7 @@ from litellm.router_strategy.complexity_router.tier_predictor import ( TierGlobalStatistic, TrainedTierArtifact, ) +from litellm.router_utils.auto_router_model_naming import count_heuristic_v2_routers from litellm.types.router import ( Deployment, LiteLLM_Params, @@ -1718,6 +1719,13 @@ class TestLLMClassifierConfig: assert config.classifier_type == "heuristic" assert config.classifier_llm_config is None + def test_classifier_circuit_breaker_defaults_on_and_requires_positive_cooldown(self): + config = ClassifierLLMConfig(model="haiku-classifier") + assert config.circuit_breaker_enabled is True + assert config.circuit_breaker_cooldown_seconds == 30.0 + with pytest.raises(ValidationError): + ClassifierLLMConfig(model="haiku-classifier", circuit_breaker_cooldown_seconds=0) + @pytest.mark.parametrize("reasoning_effort", ["", "ultra"]) def test_classifier_reasoning_effort_rejects_unsupported_values(self, reasoning_effort): with pytest.raises(ValidationError): @@ -2031,8 +2039,11 @@ class TestLLMClassifier: ) outcome = await router.aclassify("hi") + next_outcome = await router.aclassify("hi again") assert outcome.cause == "heuristic_scorer" + assert next_outcome.cause == "heuristic_scorer" + assert "classifier-circuit-open" in next_outcome.signals assert real_router.total_calls["openai/mock-classifier"] == 1 assert real_router.total_calls["openai/mock-backup-classifier"] == 0 @@ -2065,6 +2076,79 @@ class TestLLMClassifier: assert outcome.cause == "heuristic_scorer" assert cancelled.is_set() + @pytest.mark.asyncio + async def test_timeout_opens_classifier_circuit_for_other_sessions( + self, mock_router_instance, llm_classifier_config + ): + """One classifier outage is deployment-wide, so a second session must not pay the timeout.""" + mock_router_instance.acompletion = AsyncMock(side_effect=TimeoutError("classifier timed out")) + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=llm_classifier_config, + ) + + first = await router.aclassify("first ask", request_kwargs={"metadata": {"session_id": "session-a"}}) + second = await router.aclassify("second ask", request_kwargs={"metadata": {"session_id": "session-b"}}) + + assert first.cause == "heuristic_scorer" + assert second.cause == "heuristic_scorer" + assert "classifier-circuit-open" in second.signals + mock_router_instance.acompletion.assert_awaited_once() + + def test_classifier_circuit_allows_one_probe_and_closes_on_success(self): + now = 100.0 + breaker = _ClassifierCircuitBreaker(30.0, clock=lambda: now) + + assert breaker.allow_request() is True + breaker.record_failure(is_timeout=True) + assert breaker.allow_request() is False + + now = 130.0 + assert breaker.allow_request() is True + assert breaker.allow_request() is False + + breaker.record_success() + assert breaker.allow_request() is True + + def test_failed_classifier_probe_restarts_cooldown(self): + now = 100.0 + breaker = _ClassifierCircuitBreaker(30.0, clock=lambda: now) + breaker.record_failure(is_timeout=True) + + now = 130.0 + assert breaker.allow_request() is True + breaker.record_failure(is_timeout=False) + assert breaker.allow_request() is False + + now = 160.0 + assert breaker.allow_request() is True + + @pytest.mark.asyncio + async def test_classifier_circuit_can_be_disabled(self, mock_router_instance, llm_classifier_config): + mock_router_instance.acompletion = AsyncMock(side_effect=TimeoutError("classifier timed out")) + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + **llm_classifier_config, + "classifier_llm_config": { + **llm_classifier_config["classifier_llm_config"], + "circuit_breaker_enabled": False, + }, + }, + ) + + await router.aclassify("first ask") + await router.aclassify("second ask") + + assert mock_router_instance.acompletion.await_count == 2 + + def test_non_timeout_failure_does_not_open_closed_classifier_circuit(self): + breaker = _ClassifierCircuitBreaker(30.0) + breaker.record_failure(is_timeout=False) + assert breaker.allow_request() is True + @pytest.mark.asyncio async def test_aclassify_classifier_cost_is_none_when_call_is_unpriced( self, llm_complexity_router, mock_router_instance @@ -8527,7 +8611,8 @@ class TestClassifierFallbackChoice: @pytest.mark.asyncio async def test_a_classifier_failure_does_not_pin_the_session_to_the_default_model(self, mock_router_instance): """One transient timeout must not hold a session on default_model for the whole affinity TTL: - that turn was never classified, so there is nothing worth pinning and the next turn retries.""" + that turn was never classified, so there is nothing worth pinning. The circuit breaker is + disabled here so the next turn isolates and verifies the affinity contract.""" router = ComplexityRouter( model_name="test-complexity-router", litellm_router_instance=mock_router_instance, @@ -8539,7 +8624,11 @@ class TestClassifierFallbackChoice: "REASONING": "o1-preview", }, "classifier_type": "llm", - "classifier_llm_config": {"model": "haiku-classifier", "timeout_ms": 400}, + "classifier_llm_config": { + "model": "haiku-classifier", + "timeout_ms": 400, + "circuit_breaker_enabled": False, + }, "classifier_fallback": "default_model", "default_model": "gpt-4o", "session_affinity": True, diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index 00ee7bd7d6e..e596c406799 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -14,6 +14,7 @@ import CustomTierPromptEditor from "./CustomTierPromptEditor"; import { RestrictedSection, restrictedBy } from "./TierRestrictions"; import HeuristicScoringConfig from "./HeuristicScoringConfig"; import ClassifierReasoningEffortSelect from "./ClassifierReasoningEffortSelect"; +import ClassifierCircuitBreakerConfig from "./ClassifierCircuitBreakerConfig"; import type { ReasoningEffort } from "./complexity_router_tiers"; import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults"; import { @@ -555,6 +556,10 @@ const ClassificationMethodConfig: React.FC = ({ How long the classifier call has before it fails and the fallback below takes over. + onChange({ ...value, classifier_llm_config })} + />
Classification Rubric diff --git a/ui/litellm-dashboard/src/components/add_model/ClassifierCircuitBreakerConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassifierCircuitBreakerConfig.tsx new file mode 100644 index 00000000000..40c6efca4af --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/ClassifierCircuitBreakerConfig.tsx @@ -0,0 +1,69 @@ +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import React from "react"; + +import type { ClassifierLLMConfig } from "./ComplexityRouterConfig"; + +export const DEFAULT_CLASSIFIER_CIRCUIT_BREAKER_ENABLED = true; +export const DEFAULT_CLASSIFIER_CIRCUIT_BREAKER_COOLDOWN_SECONDS = 30; + +const COOLDOWN_ID = "classifier-circuit-breaker-cooldown-seconds"; + +interface ClassifierCircuitBreakerConfigProps { + value: ClassifierLLMConfig; + onChange: (value: ClassifierLLMConfig) => void; +} + +const ClassifierCircuitBreakerConfig: React.FC = ({ value, onChange }) => { + const [draftCooldown, setDraftCooldown] = React.useState(null); + const enabled = value.circuit_breaker_enabled ?? DEFAULT_CLASSIFIER_CIRCUIT_BREAKER_ENABLED; + + const handleCooldownChange = (raw: string) => { + setDraftCooldown(raw); + const parsed = Number(raw); + if (raw.trim() === "" || !Number.isFinite(parsed)) return; + onChange({ + ...value, + circuit_breaker_cooldown_seconds: Math.max(1, Math.round(parsed)), + }); + }; + + return ( +
+
+ onChange({ ...value, circuit_breaker_enabled })} + aria-label="Classifier circuit breaker" + /> + Classifier circuit breaker +
+ + After one classifier timeout, use the fallback immediately for every session until a recovery probe succeeds. + Enabled by default. + + {enabled && ( +
+ + handleCooldownChange(event.target.value)} + onBlur={() => setDraftCooldown(null)} + className="w-full" + /> +
+ )} +
+ ); +}; + +export default ClassifierCircuitBreakerConfig; diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index bdca5205b2e..588f9e777c5 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -166,10 +166,31 @@ describe("ComplexityRouterConfig", () => { expect(screen.getByText("Classifier Model")).toBeInTheDocument(); expect(screen.getByLabelText("Timeout (ms)")).toHaveValue("750"); + expect(screen.getByRole("switch", { name: "Classifier circuit breaker" })).toBeChecked(); + expect(screen.getByLabelText("Circuit breaker cooldown (seconds)")).toHaveValue("30"); expect(screen.getByLabelText("Context Window Size")).toHaveValue("5"); expect(screen.queryByText("Context Per-Turn Character Limit")).not.toBeInTheDocument(); }); + it("should allow the default-on classifier circuit breaker to be disabled", () => { + const onChange = vi.fn(); + const llmValue: ComplexityRouterConfigValue = { + ...defaultValue, + classifier_type: "llm", + classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, + }; + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + + fireEvent.click(screen.getByRole("switch", { name: "Classifier circuit breaker" })); + + expect(onChange).toHaveBeenCalledWith( + expect.objectContaining({ + classifier_llm_config: expect.objectContaining({ circuit_breaker_enabled: false }), + }), + ); + }); + it("should default the context window and budget when llm is selected", () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, @@ -278,6 +299,17 @@ describe("ComplexityRouterConfig", () => { it.each([ ["Timeout (ms)", "7", { classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 7 } }], + [ + "Circuit breaker cooldown (seconds)", + "45", + { + classifier_llm_config: { + model: "gpt-3.5-turbo", + timeout_ms: 3000, + circuit_breaker_cooldown_seconds: 45, + }, + }, + ], ["Context Window Size", "0", { classifier_context_window_size: 0 }], ["Context Character Budget", "7", { classifier_context_budget_chars: 7 }], ])("keeps %s empty while it is being edited, then commits %s", (label, replacement, expected) => { diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 06363830d64..2a024ab7fdf 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -126,6 +126,8 @@ export const CLASSIFICATION_RUBRIC_KEYS = Object.keys(CLASSIFICATION_RUBRIC_DESC export interface ClassifierLLMConfig { model: string; timeout_ms: number; + circuit_breaker_enabled?: boolean; + circuit_breaker_cooldown_seconds?: number; reasoning_effort?: ReasoningEffort; classification_rubric?: ClassificationRubric; system_prompt?: string; diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 9ee555f5dd2..87e82ef3c4b 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -113,6 +113,21 @@ describe("buildComplexityRouterConfig", () => { expect(config.classifier_llm_config).toEqual({ model: "gpt-4o-mini", timeout_ms: 3000 }); }); + it("preserves explicit classifier circuit-breaker settings, including disabled", () => { + const classifierLlmConfig = { + model: "gpt-4o-mini", + timeout_ms: 3000, + circuit_breaker_enabled: false, + circuit_breaker_cooldown_seconds: 45, + }; + const config = buildComplexityRouterConfig({ + ...baseParams, + classifierType: "llm", + classifierLlmConfig, + }); + expect(config.classifier_llm_config).toEqual(classifierLlmConfig); + }); + it("omits classifier_llm_config when classifier_type is heuristic even if config lingers in state", () => { const config = buildComplexityRouterConfig({ ...baseParams, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 956e593a234..32633a809a2 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -56,15 +56,26 @@ import { export const normalizeClassifierLlmConfig = ({ model, timeout_ms, + circuit_breaker_enabled, + circuit_breaker_cooldown_seconds, reasoning_effort, classification_rubric, system_prompt, }: ClassifierLLMConfig): ClassifierLLMConfig => system_prompt?.trim() - ? { model, timeout_ms, ...(reasoning_effort && { reasoning_effort }), system_prompt } + ? { + model, + timeout_ms, + ...(circuit_breaker_enabled !== undefined && { circuit_breaker_enabled }), + ...(circuit_breaker_cooldown_seconds !== undefined && { circuit_breaker_cooldown_seconds }), + ...(reasoning_effort && { reasoning_effort }), + system_prompt, + } : { model, timeout_ms, + ...(circuit_breaker_enabled !== undefined && { circuit_breaker_enabled }), + ...(circuit_breaker_cooldown_seconds !== undefined && { circuit_breaker_cooldown_seconds }), ...(reasoning_effort && { reasoning_effort }), ...(classification_rubric && { classification_rubric }), }; @@ -325,6 +336,12 @@ export const customTierWireFields = ( classifier_llm_config: { model: classifierLlmConfig.model, timeout_ms: classifierLlmConfig.timeout_ms, + ...(classifierLlmConfig.circuit_breaker_enabled !== undefined && { + circuit_breaker_enabled: classifierLlmConfig.circuit_breaker_enabled, + }), + ...(classifierLlmConfig.circuit_breaker_cooldown_seconds !== undefined && { + circuit_breaker_cooldown_seconds: classifierLlmConfig.circuit_breaker_cooldown_seconds, + }), ...(classifierLlmConfig.reasoning_effort && { reasoning_effort: classifierLlmConfig.reasoning_effort }), }, }), diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 3dcfeb64866..c7c324b533c 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25206,6 +25206,18 @@ export interface components { * @description Configuration for the LLM-based complexity classifier. */ ClassifierLLMConfig: { + /** + * Circuit Breaker Cooldown Seconds + * @description How long to skip this router's LLM classifier after a classification call times out. Requests use classifier_fallback during the cooldown. When it expires, one request probes the classifier while concurrent requests keep using the fallback; a successful probe closes the circuit and a failed probe restarts the cooldown. + * @default 30 + */ + circuit_breaker_cooldown_seconds: number; + /** + * Circuit Breaker Enabled + * @description Whether one classifier timeout temporarily sends requests through classifier_fallback. Enabled by default so an unhealthy classifier cannot repeat its timeout across sessions. + * @default true + */ + circuit_breaker_enabled: boolean; /** @description Which calibration examples the built-in rubric carries. 'agentic' anchors routine installs, builds, multi-file edits, and standard debugging at MEDIUM, so ordinary engineering does not route to the most expensive tier; it suits agent, terminal, and coding-assistant traffic as well as mixed traffic. 'chat' omits those engineering anchors, for a deployment serving only conversational traffic. 'business' carries business/sales anchors and business-flavored tier criteria that keep routine drafting and summarizing off the expensive tiers and reserve the top tier for committing to decisions under tradeoffs; it suits sales, support, and go-to-market traffic. Every preset keeps the same four tiers, so this moves where the boundary sits without changing the taxonomy. Leave unset for 'legacy', the rubric as it shipped before calibration examples existed, so an existing router's tier decisions and spend do not move on upgrade. Mutually exclusive with system_prompt, which replaces the rubric this would select. Only applies when classifier_type is 'llm'. */ classification_rubric?: components["schemas"]["ClassificationRubric"] | null; /** From 5a2845d183d588fa892c7409b2fed077fd568c3d Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 3 Sep 2026 20:10:02 -0700 Subject: [PATCH 52/88] fix(router): preserve classifier breaker state under concurrency --- .../complexity_router/complexity_router.py | 47 ++++++++---- .../router_strategy/test_complexity_router.py | 75 +++++++++++++++---- 2 files changed, 94 insertions(+), 28 deletions(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 9176b3da02a..d7bbf85d4fa 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -836,31 +836,45 @@ class _ClassifierCircuitBreaker: self._clock = clock self._state = self.CLOSED self._opened_at: float | None = None + self._generation = 0 self._lock = Lock() - def allow_request(self) -> bool: - """Allow ordinary calls while closed and exactly one probe after cooldown.""" + def acquire_permit(self) -> int | None: + """Return a generation-scoped permit, or deny the call while the circuit is open. + + Calls admitted together while closed share a generation. The first timeout advances it, + making every other in-flight completion stale so it cannot erase the new cooldown. + """ with self._lock: if self._state == self.CLOSED: - return True + return self._generation if self._state == self.HALF_OPEN: - return False + return None opened_at: Final = self._opened_at if opened_at is not None and self._clock() - opened_at >= self._cooldown_seconds: self._state = self.HALF_OPEN - return True - return False + return self._generation + return None - def record_success(self) -> None: + def record_success(self, permit: int) -> None: + """Close only when the current half-open recovery probe succeeds.""" with self._lock: + if self._state != self.HALF_OPEN or permit != self._generation: + return self._state = self.CLOSED self._opened_at = None - def record_failure(self, *, is_timeout: bool) -> None: + def record_failure(self, permit: int, *, is_timeout: bool) -> None: """Open on a normal timeout, or reopen when the single recovery probe fails.""" with self._lock: - if not is_timeout and self._state != self.HALF_OPEN: + if permit != self._generation: return + if self._state == self.CLOSED: + if not is_timeout: + return + elif self._state != self.HALF_OPEN: + return + self._generation += 1 self._state = self.OPEN self._opened_at = self._clock() @@ -1541,7 +1555,8 @@ class ComplexityRouter(CustomLogger): has. It is handed to the failure path so a classifier error does not re-run the scorer. """ breaker: Final = self._classifier_circuit_breaker - if breaker is not None and not breaker.allow_request(): + permit: Final = breaker.acquire_permit() if breaker is not None else None + if breaker is not None and permit is None: return self._classifier_failure_outcome( "LLM classifier circuit is open", prompt, @@ -1551,8 +1566,8 @@ class ComplexityRouter(CustomLogger): ) try: tier, classifier_cost = await self._classify_with_llm(prompt, system_prompt, request_kwargs, messages) - if breaker is not None: - breaker.record_success() + if breaker is not None and permit is not None: + breaker.record_success(permit) return ClassificationOutcome( tier=tier, score=None, @@ -1560,9 +1575,13 @@ class ComplexityRouter(CustomLogger): cause="llm_classifier", classifier_cost=classifier_cost, ) + except asyncio.CancelledError: + if breaker is not None and permit is not None: + breaker.record_failure(permit, is_timeout=False) + raise except Exception as e: # noqa: BLE001 -- external LLM call can fail in many distinct ways (timeout, provider error, validation, parse error); any failure must fall back to the configured fallback path - if breaker is not None: - breaker.record_failure(is_timeout=_is_classifier_timeout(e)) + if breaker is not None and permit is not None: + breaker.record_failure(permit, is_timeout=_is_classifier_timeout(e)) return self._classifier_failure_outcome(f"LLM classifier failed ({e})", prompt, system_prompt, scored) def _classifier_failure_outcome( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index d55cabd6806..137fb128a57 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2100,29 +2100,74 @@ class TestLLMClassifier: now = 100.0 breaker = _ClassifierCircuitBreaker(30.0, clock=lambda: now) - assert breaker.allow_request() is True - breaker.record_failure(is_timeout=True) - assert breaker.allow_request() is False + initial_permit = breaker.acquire_permit() + assert initial_permit is not None + breaker.record_failure(initial_permit, is_timeout=True) + assert breaker.acquire_permit() is None now = 130.0 - assert breaker.allow_request() is True - assert breaker.allow_request() is False + probe_permit = breaker.acquire_permit() + assert probe_permit is not None + assert breaker.acquire_permit() is None - breaker.record_success() - assert breaker.allow_request() is True + breaker.record_success(probe_permit) + assert breaker.acquire_permit() is not None def test_failed_classifier_probe_restarts_cooldown(self): now = 100.0 breaker = _ClassifierCircuitBreaker(30.0, clock=lambda: now) - breaker.record_failure(is_timeout=True) + initial_permit = breaker.acquire_permit() + assert initial_permit is not None + breaker.record_failure(initial_permit, is_timeout=True) now = 130.0 - assert breaker.allow_request() is True - breaker.record_failure(is_timeout=False) - assert breaker.allow_request() is False + probe_permit = breaker.acquire_permit() + assert probe_permit is not None + breaker.record_failure(probe_permit, is_timeout=False) + assert breaker.acquire_permit() is None now = 160.0 - assert breaker.allow_request() is True + assert breaker.acquire_permit() is not None + + def test_stale_success_cannot_close_circuit_opened_by_overlapping_timeout(self): + breaker = _ClassifierCircuitBreaker(30.0) + timeout_permit = breaker.acquire_permit() + stale_success_permit = breaker.acquire_permit() + assert timeout_permit is not None + assert stale_success_permit is not None + + breaker.record_failure(timeout_permit, is_timeout=True) + breaker.record_success(stale_success_permit) + + assert breaker.acquire_permit() is None + + @pytest.mark.asyncio + async def test_cancelled_classifier_probe_restarts_cooldown(self, mock_router_instance, llm_classifier_config): + now = 100.0 + mock_router_instance.acompletion = AsyncMock( + side_effect=[ + TimeoutError("classifier timed out"), + asyncio.CancelledError(), + _llm_response('{"tier": "SIMPLE"}'), + ] + ) + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=llm_classifier_config, + ) + router._classifier_circuit_breaker = _ClassifierCircuitBreaker(30.0, clock=lambda: now) + + await router.aclassify("open the circuit") + now = 130.0 + with pytest.raises(asyncio.CancelledError): + await router.aclassify("cancel the recovery probe") + + outcome = await router.aclassify("stay in cooldown") + + assert outcome.cause == "heuristic_scorer" + assert "classifier-circuit-open" in outcome.signals + assert mock_router_instance.acompletion.await_count == 2 @pytest.mark.asyncio async def test_classifier_circuit_can_be_disabled(self, mock_router_instance, llm_classifier_config): @@ -2146,8 +2191,10 @@ class TestLLMClassifier: def test_non_timeout_failure_does_not_open_closed_classifier_circuit(self): breaker = _ClassifierCircuitBreaker(30.0) - breaker.record_failure(is_timeout=False) - assert breaker.allow_request() is True + permit = breaker.acquire_permit() + assert permit is not None + breaker.record_failure(permit, is_timeout=False) + assert breaker.acquire_permit() is not None @pytest.mark.asyncio async def test_aclassify_classifier_cost_is_none_when_call_is_unpriced( From 81dd911bdc20db3a5a8a3d60837ffcc56287ed44 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 3 Sep 2026 20:19:13 -0700 Subject: [PATCH 53/88] fix(router): recognize asyncio classifier timeouts --- .../router_strategy/complexity_router/complexity_router.py | 4 +++- tests/test_litellm/router_strategy/test_complexity_router.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index d7bbf85d4fa..2c1097b7af3 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -880,7 +880,9 @@ class _ClassifierCircuitBreaker: def _is_classifier_timeout(exc: BaseException) -> bool: - if isinstance(exc, TimeoutError): + # asyncio.TimeoutError became an alias of the built-in TimeoutError in Python 3.11. + # LiteLLM still supports 3.10, where they are distinct exception classes. + if isinstance(exc, (TimeoutError, asyncio.TimeoutError)): return True from litellm.exceptions import Timeout as LiteLLMTimeout diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 137fb128a57..130a6f5a488 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -26,6 +26,7 @@ from litellm.router_strategy.complexity_router.complexity_router import ( KeywordOverride, _built_in_prompt, _ClassifierCircuitBreaker, + _is_classifier_timeout, _matched_plan_mode_sentinel, classification_system_prompt, ) @@ -2196,6 +2197,9 @@ class TestLLMClassifier: breaker.record_failure(permit, is_timeout=False) assert breaker.acquire_permit() is not None + def test_asyncio_timeout_is_a_classifier_timeout_on_python_310(self): + assert _is_classifier_timeout(asyncio.TimeoutError()) is True + @pytest.mark.asyncio async def test_aclassify_classifier_cost_is_none_when_call_is_unpriced( self, llm_complexity_router, mock_router_instance From 8e83d6d63d3d5bed593de08a2aab9da7c9dab0aa Mon Sep 17 00:00:00 2001 From: mateo Date: Fri, 4 Sep 2026 14:06:54 +0000 Subject: [PATCH 54/88] fix(model_prices): add Databricks Sep-2026 catalog, Azure gpt-realtime-2.x, per-token realtime image pricing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 733 +++++++++++++++++- model_prices_and_context_window.json | 733 +++++++++++++++++- .../llm_cost_calc/test_llm_cost_calc_utils.py | 22 + .../test_databricks_cost_calculator.py | 24 + 4 files changed, 1458 insertions(+), 54 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 616d58970cf..7304ca04eb5 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -5311,7 +5311,7 @@ "cache_read_input_token_cost": 4e-06, "deprecation_date": "2027-03-02", "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image": 5e-06, + "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "azure", "max_input_tokens": 32000, @@ -5344,7 +5344,7 @@ "cache_read_input_token_cost": 4e-06, "deprecation_date": "2027-08-24", "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image": 5e-06, + "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "azure", "max_input_tokens": 32000, @@ -5372,11 +5372,115 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "azure/gpt-realtime-2": { + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "deprecation_date": "2026-08-31", + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image_token": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 2.4e-05, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/gpt-realtime-2.1": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "deprecation_date": "2027-06-25", + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image_token": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 2.4e-05, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/gpt-realtime-2.1-mini": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "deprecation_date": "2027-06-25", + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image_token": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "azure/gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, - "input_cost_per_image": 8e-07, + "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, "litellm_provider": "azure", "max_input_tokens": 32000, @@ -5408,7 +5512,7 @@ "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, - "input_cost_per_image": 8e-07, + "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, "litellm_provider": "azure", "max_input_tokens": 32000, @@ -16995,6 +17099,7 @@ "databricks/databricks-claude-3-7-sonnet": { "cache_creation_input_token_cost": 3.74997e-06, "cache_read_input_token_cost": 3.0002e-07, + "deprecation_date": "2026-04-12", "input_cost_per_token": 2.9999900000000002e-06, "input_dbu_cost_per_token": 4.2857e-05, "litellm_provider": "databricks", @@ -17042,6 +17147,35 @@ "supports_vision": false, "thinking_always_on": true }, + "databricks/databricks-claude-fable-5-1": { + "cache_creation_input_token_cost": 1.250004e-05, + "cache_read_input_token_cost": 2.5004e-07, + "input_cost_per_token": 1.000006e-05, + "input_dbu_cost_per_token": 0.000142858, + "litellm_provider": "databricks", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 5.000002e-05, + "output_dbu_cost_per_token": 0.000714286, + "prompt_cache_min_tokens": 512, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "thinking_always_on": true + }, "databricks/databricks-claude-haiku-4-5": { "cache_creation_input_token_cost": 1.24999e-06, "cache_read_input_token_cost": 1.0003e-07, @@ -17242,6 +17376,7 @@ "databricks/databricks-claude-sonnet-4": { "cache_creation_input_token_cost": 3.74997e-06, "cache_read_input_token_cost": 3.0002e-07, + "deprecation_date": "2026-10-09", "input_cost_per_token": 2.9999900000000002e-06, "input_dbu_cost_per_token": 4.2857e-05, "litellm_provider": "databricks", @@ -17254,13 +17389,13 @@ "mode": "chat", "output_cost_per_token": 1.5000020000000002e-05, "output_dbu_cost_per_token": 0.000214286, + "prompt_cache_min_tokens": 1024, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true, - "prompt_cache_min_tokens": 1024 + "supports_tool_choice": true }, "databricks/databricks-claude-sonnet-4-1": { "cache_creation_input_token_cost": 3.74997e-06, @@ -17417,6 +17552,7 @@ "databricks/databricks-gemini-2-5-flash": { "cache_creation_input_token_cost": 3.0002e-07, "cache_read_input_token_cost": 3.0002e-08, + "deprecation_date": "2026-10-02", "input_cost_per_token": 3.0001999999999996e-07, "input_dbu_cost_per_token": 4.285999999999999e-06, "litellm_provider": "databricks", @@ -17474,6 +17610,48 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "databricks/databricks-gemini-3-1-flash-image": { + "litellm_provider": "databricks", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "metadata": { + "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." + }, + "mode": "chat", + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_vision": true + }, + "databricks/databricks-gemini-3-pro-image": { + "litellm_provider": "databricks", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "metadata": { + "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." + }, + "mode": "chat", + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_vision": true + }, "databricks/databricks-gemini-3-1-pro": { "cache_creation_input_token_cost": 2.49998e-06, "cache_read_input_token_cost": 2.4997e-07, @@ -17534,6 +17712,148 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "databricks/databricks-gemini-3-8-flash": { + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." + }, + "mode": "chat", + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gemini-3-7-flash": { + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." + }, + "mode": "chat", + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gemini-3-6-flash": { + "cache_creation_input_token_cost": 1.87502e-06, + "cache_read_input_token_cost": 1.8753e-07, + "input_cost_per_token": 1.87502e-06, + "input_dbu_cost_per_token": 2.6786e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 9.37503e-06, + "output_dbu_cost_per_token": 0.000133929, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gemini-3-5-flash": { + "cache_creation_input_token_cost": 1.87502e-06, + "cache_read_input_token_cost": 1.8753e-07, + "input_cost_per_token": 1.87502e-06, + "input_dbu_cost_per_token": 2.6786e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 1.124998e-05, + "output_dbu_cost_per_token": 0.000160714, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gemini-3-5-flash-lite": { + "cache_creation_input_token_cost": 3.7499e-07, + "cache_read_input_token_cost": 3.752e-08, + "input_cost_per_token": 3.7499e-07, + "input_dbu_cost_per_token": 5.357e-06, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 3.12501e-06, + "output_dbu_cost_per_token": 4.4643e-05, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "databricks/databricks-gemma-3-12b": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, @@ -17579,16 +17899,53 @@ "supports_tool_choice": true, "supports_vision": false }, + "databricks/databricks-glm-5-3": { + "cache_creation_input_token_cost": 1.4e-06, + "cache_read_input_token_cost": 2.5998e-07, + "input_cost_per_token": 1.4e-06, + "input_dbu_cost_per_token": 2e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 4.39999e-06, + "output_dbu_cost_per_token": 6.2857e-05, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false, + "thinking_always_on": true + }, "databricks/databricks-glm-5-3-flash": { + "cache_creation_input_token_cost": 1.5001e-07, + "cache_read_input_token_cost": 3.003e-08, + "input_cost_per_token": 1.5001e-07, + "input_dbu_cost_per_token": 2.143e-06, "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 131072, "max_tokens": 131072, "metadata": { - "notes": "Databricks has not published pay-per-token DBU rates for this model yet (not on the foundation-model-serving pricing page as of 2026-08-27), so cost fields are omitted until rates are published." + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, "mode": "chat", - "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "output_cost_per_token": 5.0001e-07, + "output_dbu_cost_per_token": 7.143e-06, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supported_modalities": [ "text", "image" @@ -17600,7 +17957,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "thinking_always_on": true }, "databricks/databricks-gpt-5": { "cache_creation_input_token_cost": 1.24999e-06, @@ -17618,7 +17976,9 @@ "output_cost_per_token": 9.999990000000002e-06, "output_dbu_cost_per_token": 0.000142857, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true }, "databricks/databricks-gpt-5-1": { "cache_creation_input_token_cost": 1.24999e-06, @@ -17636,11 +17996,14 @@ "output_cost_per_token": 9.999990000000002e-06, "output_dbu_cost_per_token": 0.000142857, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true }, "databricks/databricks-gpt-5-1-codex-max": { "cache_creation_input_token_cost": 1.24999e-06, "cache_read_input_token_cost": 1.2502e-07, + "deprecation_date": "2026-07-16", "input_cost_per_token": 1.24999e-06, "input_dbu_cost_per_token": 1.7857e-05, "litellm_provider": "databricks", @@ -17659,6 +18022,7 @@ "databricks/databricks-gpt-5-1-codex-mini": { "cache_creation_input_token_cost": 2.4997e-07, "cache_read_input_token_cost": 2.499e-08, + "deprecation_date": "2026-07-16", "input_cost_per_token": 2.4997e-07, "input_dbu_cost_per_token": 3.571e-06, "litellm_provider": "databricks", @@ -17690,11 +18054,14 @@ "output_cost_per_token": 1.4e-05, "output_dbu_cost_per_token": 0.0002, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true }, "databricks/databricks-gpt-5-2-codex": { "cache_creation_input_token_cost": 1.75e-06, "cache_read_input_token_cost": 1.75e-07, + "deprecation_date": "2026-07-16", "input_cost_per_token": 1.75e-06, "input_dbu_cost_per_token": 2.5e-05, "litellm_provider": "databricks", @@ -17726,7 +18093,9 @@ "output_cost_per_token": 1.4e-05, "output_dbu_cost_per_token": 0.0002, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true }, "databricks/databricks-gpt-5-4": { "cache_creation_input_token_cost": 2.49998e-06, @@ -17734,7 +18103,7 @@ "input_cost_per_token": 2.49998e-06, "input_dbu_cost_per_token": 3.5714e-05, "litellm_provider": "databricks", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "metadata": { @@ -17744,7 +18113,18 @@ "output_cost_per_token": 1.5000020000000002e-05, "output_dbu_cost_per_token": 0.000214286, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true }, "databricks/databricks-gpt-5-4-mini": { "cache_creation_input_token_cost": 7.4998e-07, @@ -17762,7 +18142,18 @@ "output_cost_per_token": 4.50002e-06, "output_dbu_cost_per_token": 6.4286e-05, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true }, "databricks/databricks-gpt-5-4-nano": { "cache_creation_input_token_cost": 1.9999e-07, @@ -17780,7 +18171,163 @@ "output_cost_per_token": 1.24999e-06, "output_dbu_cost_per_token": 1.7857e-05, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gpt-5-6-sol": { + "cache_creation_input_token_cost": 5.00003e-06, + "cache_read_input_token_cost": 3.9998e-07, + "input_cost_per_token": 4.00001e-06, + "input_dbu_cost_per_token": 5.7143e-05, + "litellm_provider": "databricks", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields. Rates reflect OpenAI's promotional pricing in effect through November 21, 2026; afterwards input, cache and Batch rates are 25% higher and output rates 50% higher." + }, + "mode": "chat", + "output_cost_per_token": 1.999998e-05, + "output_dbu_cost_per_token": 0.000285714, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gpt-5-6-terra": { + "cache_creation_input_token_cost": 3.12501e-06, + "cache_read_input_token_cost": 2.4997e-07, + "input_cost_per_token": 2.49998e-06, + "input_dbu_cost_per_token": 3.5714e-05, + "litellm_provider": "databricks", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 1.500002e-05, + "output_dbu_cost_per_token": 0.000214286, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gpt-5-6-luna": { + "cache_creation_input_token_cost": 1.24999e-06, + "cache_read_input_token_cost": 1.0003e-07, + "input_cost_per_token": 1.00002e-06, + "input_dbu_cost_per_token": 1.4286e-05, + "litellm_provider": "databricks", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 5.99998e-06, + "output_dbu_cost_per_token": 8.5714e-05, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gpt-5-5": { + "cache_creation_input_token_cost": 5.00003e-06, + "cache_read_input_token_cost": 5.0001e-07, + "input_cost_per_token": 5.00003e-06, + "input_dbu_cost_per_token": 7.1429e-05, + "litellm_provider": "databricks", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "responses", + "output_cost_per_token": 2.999997e-05, + "output_dbu_cost_per_token": 0.000428571, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gpt-5-5-pro": { + "cache_creation_input_token_cost": 2.999997e-05, + "cache_read_input_token_cost": 2.999997e-05, + "input_cost_per_token": 2.999997e-05, + "input_dbu_cost_per_token": 0.000428571, + "litellm_provider": "databricks", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "responses", + "output_cost_per_token": 0.00018000003, + "output_dbu_cost_per_token": 0.002571429, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true }, "databricks/databricks-gpt-5-mini": { "cache_creation_input_token_cost": 2.4997e-07, @@ -17798,7 +18345,9 @@ "output_cost_per_token": 1.9999700000000004e-06, "output_dbu_cost_per_token": 2.8571e-05, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true }, "databricks/databricks-gpt-5-nano": { "cache_creation_input_token_cost": 4.998e-08, @@ -17816,7 +18365,9 @@ "output_cost_per_token": 3.9998000000000007e-07, "output_dbu_cost_per_token": 5.714000000000001e-06, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true }, "databricks/databricks-gpt-oss-120b": { "cache_creation_input_token_cost": 1.5001e-07, @@ -17852,6 +18403,32 @@ "output_dbu_cost_per_token": 4.285999999999999e-06, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, + "databricks/databricks-grok-4-6": { + "cache_creation_input_token_cost": 2.49998e-06, + "cache_read_input_token_cost": 6.2503e-07, + "input_cost_per_token": 2.49998e-06, + "input_dbu_cost_per_token": 3.5714e-05, + "litellm_provider": "databricks", + "max_input_tokens": 500000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 7.50001e-06, + "output_dbu_cost_per_token": 0.000107143, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, "databricks/databricks-gte-large-en": { "cache_creation_input_token_cost": 1.2999e-07, "cache_read_input_token_cost": 1.2999e-07, @@ -17869,6 +18446,34 @@ "output_vector_size": 1024, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, + "databricks/databricks-inkling": { + "cache_creation_input_token_cost": 1.00002e-06, + "cache_read_input_token_cost": 1.7003e-07, + "input_cost_per_token": 1.00002e-06, + "input_dbu_cost_per_token": 1.4286e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1000000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 4.04999e-06, + "output_dbu_cost_per_token": 5.7857e-05, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "thinking_always_on": true + }, "databricks/databricks-kimi-k3": { "cache_creation_input_token_cost": 2.99999e-06, "cache_read_input_token_cost": 3.0002e-07, @@ -17901,6 +18506,7 @@ "databricks/databricks-llama-2-70b-chat": { "cache_creation_input_token_cost": 5.0001e-07, "cache_read_input_token_cost": 5.0001e-07, + "deprecation_date": "2024-10-30", "input_cost_per_token": 5.0001e-07, "input_dbu_cost_per_token": 7.143e-06, "litellm_provider": "databricks", @@ -17937,6 +18543,7 @@ "databricks/databricks-meta-llama-3-1-405b-instruct": { "cache_creation_input_token_cost": 5.00003e-06, "cache_read_input_token_cost": 5.00003e-06, + "deprecation_date": "2026-02-15", "input_cost_per_token": 5.00003e-06, "input_dbu_cost_per_token": 7.1429e-05, "litellm_provider": "databricks", @@ -17990,6 +18597,7 @@ "databricks/databricks-meta-llama-3-70b-instruct": { "cache_creation_input_token_cost": 1.00002e-06, "cache_read_input_token_cost": 1.00002e-06, + "deprecation_date": "2024-07-23", "input_cost_per_token": 1.00002e-06, "input_dbu_cost_per_token": 1.4286e-05, "litellm_provider": "databricks", @@ -18008,6 +18616,7 @@ "databricks/databricks-mixtral-8x7b-instruct": { "cache_creation_input_token_cost": 5.0001e-07, "cache_read_input_token_cost": 5.0001e-07, + "deprecation_date": "2025-04-30", "input_cost_per_token": 5.0001e-07, "input_dbu_cost_per_token": 7.143e-06, "litellm_provider": "databricks", @@ -18026,6 +18635,7 @@ "databricks/databricks-mpt-30b-instruct": { "cache_creation_input_token_cost": 1.00002e-06, "cache_read_input_token_cost": 1.00002e-06, + "deprecation_date": "2024-08-30", "input_cost_per_token": 1.00002e-06, "input_dbu_cost_per_token": 1.4286e-05, "litellm_provider": "databricks", @@ -18044,6 +18654,7 @@ "databricks/databricks-mpt-7b-instruct": { "cache_creation_input_token_cost": 5.0001e-07, "cache_read_input_token_cost": 5.0001e-07, + "deprecation_date": "2024-08-30", "input_cost_per_token": 5.0001e-07, "input_dbu_cost_per_token": 7.143e-06, "litellm_provider": "databricks", @@ -18059,6 +18670,74 @@ "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true }, + "databricks/databricks-qwen35-122b-a10b": { + "cache_creation_input_token_cost": 2.2001e-07, + "cache_read_input_token_cost": 2.2001e-07, + "input_cost_per_token": 2.2001e-07, + "input_dbu_cost_per_token": 3.143e-06, + "litellm_provider": "databricks", + "max_input_tokens": 262144, + "max_output_tokens": 25000, + "max_tokens": 25000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 2.20003e-06, + "output_dbu_cost_per_token": 3.1429e-05, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false, + "thinking_always_on": true + }, + "databricks/databricks-qwen3-next-80b-a3b-instruct": { + "cache_creation_input_token_cost": 1.5001e-07, + "cache_read_input_token_cost": 1.5001e-07, + "input_cost_per_token": 1.5001e-07, + "input_dbu_cost_per_token": 2.143e-06, + "litellm_provider": "databricks", + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 1.20001e-06, + "output_dbu_cost_per_token": 1.7143e-05, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "databricks/databricks-qwen3-embedding-0-6b": { + "cache_creation_input_token_cost": 2.002e-08, + "cache_read_input_token_cost": 2.002e-08, + "input_cost_per_token": 2.002e-08, + "input_dbu_cost_per_token": 2.86e-07, + "litellm_provider": "databricks", + "max_input_tokens": 32768, + "max_tokens": 32768, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_dbu_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving" + }, "dataforseo/search": { "input_cost_per_query": 0.003, "litellm_provider": "dataforseo", @@ -31227,7 +31906,7 @@ "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image": 5e-06, + "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "openai", "max_input_tokens": 32000, @@ -31259,7 +31938,7 @@ "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image": 5e-06, + "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "openai", "max_input_tokens": 32000, @@ -31292,7 +31971,7 @@ "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image": 5e-06, + "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -31325,7 +32004,7 @@ "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image": 5e-06, + "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -31360,7 +32039,7 @@ "cache_read_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, - "input_cost_per_image": 8e-07, + "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -31427,7 +32106,7 @@ "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image": 5e-06, + "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "openai", "max_input_tokens": 32000, @@ -52723,7 +53402,7 @@ "cache_read_input_token_cost": 6e-08, "deprecation_date": "2026-07-23", "input_cost_per_audio_token": 1e-05, - "input_cost_per_image": 8e-07, + "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -52756,7 +53435,7 @@ "cache_read_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, - "input_cost_per_image": 8e-07, + "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, "litellm_provider": "openai", "max_input_tokens": 128000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 616d58970cf..7304ca04eb5 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -5311,7 +5311,7 @@ "cache_read_input_token_cost": 4e-06, "deprecation_date": "2027-03-02", "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image": 5e-06, + "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "azure", "max_input_tokens": 32000, @@ -5344,7 +5344,7 @@ "cache_read_input_token_cost": 4e-06, "deprecation_date": "2027-08-24", "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image": 5e-06, + "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "azure", "max_input_tokens": 32000, @@ -5372,11 +5372,115 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "azure/gpt-realtime-2": { + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "deprecation_date": "2026-08-31", + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image_token": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 2.4e-05, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/gpt-realtime-2.1": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "deprecation_date": "2027-06-25", + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image_token": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 2.4e-05, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/gpt-realtime-2.1-mini": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "deprecation_date": "2027-06-25", + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image_token": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "azure/gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, - "input_cost_per_image": 8e-07, + "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, "litellm_provider": "azure", "max_input_tokens": 32000, @@ -5408,7 +5512,7 @@ "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, - "input_cost_per_image": 8e-07, + "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, "litellm_provider": "azure", "max_input_tokens": 32000, @@ -16995,6 +17099,7 @@ "databricks/databricks-claude-3-7-sonnet": { "cache_creation_input_token_cost": 3.74997e-06, "cache_read_input_token_cost": 3.0002e-07, + "deprecation_date": "2026-04-12", "input_cost_per_token": 2.9999900000000002e-06, "input_dbu_cost_per_token": 4.2857e-05, "litellm_provider": "databricks", @@ -17042,6 +17147,35 @@ "supports_vision": false, "thinking_always_on": true }, + "databricks/databricks-claude-fable-5-1": { + "cache_creation_input_token_cost": 1.250004e-05, + "cache_read_input_token_cost": 2.5004e-07, + "input_cost_per_token": 1.000006e-05, + "input_dbu_cost_per_token": 0.000142858, + "litellm_provider": "databricks", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 5.000002e-05, + "output_dbu_cost_per_token": 0.000714286, + "prompt_cache_min_tokens": 512, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "thinking_always_on": true + }, "databricks/databricks-claude-haiku-4-5": { "cache_creation_input_token_cost": 1.24999e-06, "cache_read_input_token_cost": 1.0003e-07, @@ -17242,6 +17376,7 @@ "databricks/databricks-claude-sonnet-4": { "cache_creation_input_token_cost": 3.74997e-06, "cache_read_input_token_cost": 3.0002e-07, + "deprecation_date": "2026-10-09", "input_cost_per_token": 2.9999900000000002e-06, "input_dbu_cost_per_token": 4.2857e-05, "litellm_provider": "databricks", @@ -17254,13 +17389,13 @@ "mode": "chat", "output_cost_per_token": 1.5000020000000002e-05, "output_dbu_cost_per_token": 0.000214286, + "prompt_cache_min_tokens": 1024, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true, - "prompt_cache_min_tokens": 1024 + "supports_tool_choice": true }, "databricks/databricks-claude-sonnet-4-1": { "cache_creation_input_token_cost": 3.74997e-06, @@ -17417,6 +17552,7 @@ "databricks/databricks-gemini-2-5-flash": { "cache_creation_input_token_cost": 3.0002e-07, "cache_read_input_token_cost": 3.0002e-08, + "deprecation_date": "2026-10-02", "input_cost_per_token": 3.0001999999999996e-07, "input_dbu_cost_per_token": 4.285999999999999e-06, "litellm_provider": "databricks", @@ -17474,6 +17610,48 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "databricks/databricks-gemini-3-1-flash-image": { + "litellm_provider": "databricks", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "metadata": { + "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." + }, + "mode": "chat", + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_vision": true + }, + "databricks/databricks-gemini-3-pro-image": { + "litellm_provider": "databricks", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "metadata": { + "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." + }, + "mode": "chat", + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_vision": true + }, "databricks/databricks-gemini-3-1-pro": { "cache_creation_input_token_cost": 2.49998e-06, "cache_read_input_token_cost": 2.4997e-07, @@ -17534,6 +17712,148 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "databricks/databricks-gemini-3-8-flash": { + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." + }, + "mode": "chat", + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gemini-3-7-flash": { + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." + }, + "mode": "chat", + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gemini-3-6-flash": { + "cache_creation_input_token_cost": 1.87502e-06, + "cache_read_input_token_cost": 1.8753e-07, + "input_cost_per_token": 1.87502e-06, + "input_dbu_cost_per_token": 2.6786e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 9.37503e-06, + "output_dbu_cost_per_token": 0.000133929, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gemini-3-5-flash": { + "cache_creation_input_token_cost": 1.87502e-06, + "cache_read_input_token_cost": 1.8753e-07, + "input_cost_per_token": 1.87502e-06, + "input_dbu_cost_per_token": 2.6786e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 1.124998e-05, + "output_dbu_cost_per_token": 0.000160714, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gemini-3-5-flash-lite": { + "cache_creation_input_token_cost": 3.7499e-07, + "cache_read_input_token_cost": 3.752e-08, + "input_cost_per_token": 3.7499e-07, + "input_dbu_cost_per_token": 5.357e-06, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 3.12501e-06, + "output_dbu_cost_per_token": 4.4643e-05, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "databricks/databricks-gemma-3-12b": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, @@ -17579,16 +17899,53 @@ "supports_tool_choice": true, "supports_vision": false }, + "databricks/databricks-glm-5-3": { + "cache_creation_input_token_cost": 1.4e-06, + "cache_read_input_token_cost": 2.5998e-07, + "input_cost_per_token": 1.4e-06, + "input_dbu_cost_per_token": 2e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 4.39999e-06, + "output_dbu_cost_per_token": 6.2857e-05, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false, + "thinking_always_on": true + }, "databricks/databricks-glm-5-3-flash": { + "cache_creation_input_token_cost": 1.5001e-07, + "cache_read_input_token_cost": 3.003e-08, + "input_cost_per_token": 1.5001e-07, + "input_dbu_cost_per_token": 2.143e-06, "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 131072, "max_tokens": 131072, "metadata": { - "notes": "Databricks has not published pay-per-token DBU rates for this model yet (not on the foundation-model-serving pricing page as of 2026-08-27), so cost fields are omitted until rates are published." + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, "mode": "chat", - "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "output_cost_per_token": 5.0001e-07, + "output_dbu_cost_per_token": 7.143e-06, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supported_modalities": [ "text", "image" @@ -17600,7 +17957,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "thinking_always_on": true }, "databricks/databricks-gpt-5": { "cache_creation_input_token_cost": 1.24999e-06, @@ -17618,7 +17976,9 @@ "output_cost_per_token": 9.999990000000002e-06, "output_dbu_cost_per_token": 0.000142857, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true }, "databricks/databricks-gpt-5-1": { "cache_creation_input_token_cost": 1.24999e-06, @@ -17636,11 +17996,14 @@ "output_cost_per_token": 9.999990000000002e-06, "output_dbu_cost_per_token": 0.000142857, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true }, "databricks/databricks-gpt-5-1-codex-max": { "cache_creation_input_token_cost": 1.24999e-06, "cache_read_input_token_cost": 1.2502e-07, + "deprecation_date": "2026-07-16", "input_cost_per_token": 1.24999e-06, "input_dbu_cost_per_token": 1.7857e-05, "litellm_provider": "databricks", @@ -17659,6 +18022,7 @@ "databricks/databricks-gpt-5-1-codex-mini": { "cache_creation_input_token_cost": 2.4997e-07, "cache_read_input_token_cost": 2.499e-08, + "deprecation_date": "2026-07-16", "input_cost_per_token": 2.4997e-07, "input_dbu_cost_per_token": 3.571e-06, "litellm_provider": "databricks", @@ -17690,11 +18054,14 @@ "output_cost_per_token": 1.4e-05, "output_dbu_cost_per_token": 0.0002, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true }, "databricks/databricks-gpt-5-2-codex": { "cache_creation_input_token_cost": 1.75e-06, "cache_read_input_token_cost": 1.75e-07, + "deprecation_date": "2026-07-16", "input_cost_per_token": 1.75e-06, "input_dbu_cost_per_token": 2.5e-05, "litellm_provider": "databricks", @@ -17726,7 +18093,9 @@ "output_cost_per_token": 1.4e-05, "output_dbu_cost_per_token": 0.0002, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true }, "databricks/databricks-gpt-5-4": { "cache_creation_input_token_cost": 2.49998e-06, @@ -17734,7 +18103,7 @@ "input_cost_per_token": 2.49998e-06, "input_dbu_cost_per_token": 3.5714e-05, "litellm_provider": "databricks", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "metadata": { @@ -17744,7 +18113,18 @@ "output_cost_per_token": 1.5000020000000002e-05, "output_dbu_cost_per_token": 0.000214286, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true }, "databricks/databricks-gpt-5-4-mini": { "cache_creation_input_token_cost": 7.4998e-07, @@ -17762,7 +18142,18 @@ "output_cost_per_token": 4.50002e-06, "output_dbu_cost_per_token": 6.4286e-05, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true }, "databricks/databricks-gpt-5-4-nano": { "cache_creation_input_token_cost": 1.9999e-07, @@ -17780,7 +18171,163 @@ "output_cost_per_token": 1.24999e-06, "output_dbu_cost_per_token": 1.7857e-05, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gpt-5-6-sol": { + "cache_creation_input_token_cost": 5.00003e-06, + "cache_read_input_token_cost": 3.9998e-07, + "input_cost_per_token": 4.00001e-06, + "input_dbu_cost_per_token": 5.7143e-05, + "litellm_provider": "databricks", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields. Rates reflect OpenAI's promotional pricing in effect through November 21, 2026; afterwards input, cache and Batch rates are 25% higher and output rates 50% higher." + }, + "mode": "chat", + "output_cost_per_token": 1.999998e-05, + "output_dbu_cost_per_token": 0.000285714, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gpt-5-6-terra": { + "cache_creation_input_token_cost": 3.12501e-06, + "cache_read_input_token_cost": 2.4997e-07, + "input_cost_per_token": 2.49998e-06, + "input_dbu_cost_per_token": 3.5714e-05, + "litellm_provider": "databricks", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 1.500002e-05, + "output_dbu_cost_per_token": 0.000214286, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gpt-5-6-luna": { + "cache_creation_input_token_cost": 1.24999e-06, + "cache_read_input_token_cost": 1.0003e-07, + "input_cost_per_token": 1.00002e-06, + "input_dbu_cost_per_token": 1.4286e-05, + "litellm_provider": "databricks", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 5.99998e-06, + "output_dbu_cost_per_token": 8.5714e-05, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gpt-5-5": { + "cache_creation_input_token_cost": 5.00003e-06, + "cache_read_input_token_cost": 5.0001e-07, + "input_cost_per_token": 5.00003e-06, + "input_dbu_cost_per_token": 7.1429e-05, + "litellm_provider": "databricks", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "responses", + "output_cost_per_token": 2.999997e-05, + "output_dbu_cost_per_token": 0.000428571, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "databricks/databricks-gpt-5-5-pro": { + "cache_creation_input_token_cost": 2.999997e-05, + "cache_read_input_token_cost": 2.999997e-05, + "input_cost_per_token": 2.999997e-05, + "input_dbu_cost_per_token": 0.000428571, + "litellm_provider": "databricks", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "responses", + "output_cost_per_token": 0.00018000003, + "output_dbu_cost_per_token": 0.002571429, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true }, "databricks/databricks-gpt-5-mini": { "cache_creation_input_token_cost": 2.4997e-07, @@ -17798,7 +18345,9 @@ "output_cost_per_token": 1.9999700000000004e-06, "output_dbu_cost_per_token": 2.8571e-05, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true }, "databricks/databricks-gpt-5-nano": { "cache_creation_input_token_cost": 4.998e-08, @@ -17816,7 +18365,9 @@ "output_cost_per_token": 3.9998000000000007e-07, "output_dbu_cost_per_token": 5.714000000000001e-06, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true }, "databricks/databricks-gpt-oss-120b": { "cache_creation_input_token_cost": 1.5001e-07, @@ -17852,6 +18403,32 @@ "output_dbu_cost_per_token": 4.285999999999999e-06, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, + "databricks/databricks-grok-4-6": { + "cache_creation_input_token_cost": 2.49998e-06, + "cache_read_input_token_cost": 6.2503e-07, + "input_cost_per_token": 2.49998e-06, + "input_dbu_cost_per_token": 3.5714e-05, + "litellm_provider": "databricks", + "max_input_tokens": 500000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 7.50001e-06, + "output_dbu_cost_per_token": 0.000107143, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, "databricks/databricks-gte-large-en": { "cache_creation_input_token_cost": 1.2999e-07, "cache_read_input_token_cost": 1.2999e-07, @@ -17869,6 +18446,34 @@ "output_vector_size": 1024, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, + "databricks/databricks-inkling": { + "cache_creation_input_token_cost": 1.00002e-06, + "cache_read_input_token_cost": 1.7003e-07, + "input_cost_per_token": 1.00002e-06, + "input_dbu_cost_per_token": 1.4286e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1000000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 4.04999e-06, + "output_dbu_cost_per_token": 5.7857e-05, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "thinking_always_on": true + }, "databricks/databricks-kimi-k3": { "cache_creation_input_token_cost": 2.99999e-06, "cache_read_input_token_cost": 3.0002e-07, @@ -17901,6 +18506,7 @@ "databricks/databricks-llama-2-70b-chat": { "cache_creation_input_token_cost": 5.0001e-07, "cache_read_input_token_cost": 5.0001e-07, + "deprecation_date": "2024-10-30", "input_cost_per_token": 5.0001e-07, "input_dbu_cost_per_token": 7.143e-06, "litellm_provider": "databricks", @@ -17937,6 +18543,7 @@ "databricks/databricks-meta-llama-3-1-405b-instruct": { "cache_creation_input_token_cost": 5.00003e-06, "cache_read_input_token_cost": 5.00003e-06, + "deprecation_date": "2026-02-15", "input_cost_per_token": 5.00003e-06, "input_dbu_cost_per_token": 7.1429e-05, "litellm_provider": "databricks", @@ -17990,6 +18597,7 @@ "databricks/databricks-meta-llama-3-70b-instruct": { "cache_creation_input_token_cost": 1.00002e-06, "cache_read_input_token_cost": 1.00002e-06, + "deprecation_date": "2024-07-23", "input_cost_per_token": 1.00002e-06, "input_dbu_cost_per_token": 1.4286e-05, "litellm_provider": "databricks", @@ -18008,6 +18616,7 @@ "databricks/databricks-mixtral-8x7b-instruct": { "cache_creation_input_token_cost": 5.0001e-07, "cache_read_input_token_cost": 5.0001e-07, + "deprecation_date": "2025-04-30", "input_cost_per_token": 5.0001e-07, "input_dbu_cost_per_token": 7.143e-06, "litellm_provider": "databricks", @@ -18026,6 +18635,7 @@ "databricks/databricks-mpt-30b-instruct": { "cache_creation_input_token_cost": 1.00002e-06, "cache_read_input_token_cost": 1.00002e-06, + "deprecation_date": "2024-08-30", "input_cost_per_token": 1.00002e-06, "input_dbu_cost_per_token": 1.4286e-05, "litellm_provider": "databricks", @@ -18044,6 +18654,7 @@ "databricks/databricks-mpt-7b-instruct": { "cache_creation_input_token_cost": 5.0001e-07, "cache_read_input_token_cost": 5.0001e-07, + "deprecation_date": "2024-08-30", "input_cost_per_token": 5.0001e-07, "input_dbu_cost_per_token": 7.143e-06, "litellm_provider": "databricks", @@ -18059,6 +18670,74 @@ "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true }, + "databricks/databricks-qwen35-122b-a10b": { + "cache_creation_input_token_cost": 2.2001e-07, + "cache_read_input_token_cost": 2.2001e-07, + "input_cost_per_token": 2.2001e-07, + "input_dbu_cost_per_token": 3.143e-06, + "litellm_provider": "databricks", + "max_input_tokens": 262144, + "max_output_tokens": 25000, + "max_tokens": 25000, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 2.20003e-06, + "output_dbu_cost_per_token": 3.1429e-05, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false, + "thinking_always_on": true + }, + "databricks/databricks-qwen3-next-80b-a3b-instruct": { + "cache_creation_input_token_cost": 1.5001e-07, + "cache_read_input_token_cost": 1.5001e-07, + "input_cost_per_token": 1.5001e-07, + "input_dbu_cost_per_token": 2.143e-06, + "litellm_provider": "databricks", + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "chat", + "output_cost_per_token": 1.20001e-06, + "output_dbu_cost_per_token": 1.7143e-05, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "databricks/databricks-qwen3-embedding-0-6b": { + "cache_creation_input_token_cost": 2.002e-08, + "cache_read_input_token_cost": 2.002e-08, + "input_cost_per_token": 2.002e-08, + "input_dbu_cost_per_token": 2.86e-07, + "litellm_provider": "databricks", + "max_input_tokens": 32768, + "max_tokens": 32768, + "metadata": { + "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." + }, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_dbu_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving" + }, "dataforseo/search": { "input_cost_per_query": 0.003, "litellm_provider": "dataforseo", @@ -31227,7 +31906,7 @@ "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image": 5e-06, + "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "openai", "max_input_tokens": 32000, @@ -31259,7 +31938,7 @@ "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image": 5e-06, + "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "openai", "max_input_tokens": 32000, @@ -31292,7 +31971,7 @@ "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image": 5e-06, + "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -31325,7 +32004,7 @@ "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image": 5e-06, + "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -31360,7 +32039,7 @@ "cache_read_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, - "input_cost_per_image": 8e-07, + "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -31427,7 +32106,7 @@ "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image": 5e-06, + "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "openai", "max_input_tokens": 32000, @@ -52723,7 +53402,7 @@ "cache_read_input_token_cost": 6e-08, "deprecation_date": "2026-07-23", "input_cost_per_audio_token": 1e-05, - "input_cost_per_image": 8e-07, + "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -52756,7 +53435,7 @@ "cache_read_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, - "input_cost_per_image": 8e-07, + "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, "litellm_provider": "openai", "max_input_tokens": 128000, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 311ba7aebc0..0f8084643ea 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -4630,6 +4630,28 @@ def test_generic_cost_per_token_grok_46_long_context(_local_model_cost_map): assert completion_cost == pytest.approx(1_000 * 1.2e-05) +@pytest.mark.parametrize( + ("model", "provider", "image_token_rate"), + [ + ("gpt-realtime-2.1", "openai", 5e-06), + ("gpt-realtime-2.1-mini", "openai", 8e-07), + ("azure/gpt-realtime-2.1", "azure", 5e-06), + ("azure/gpt-realtime-2.1-mini", "azure", 8e-07), + ], +) +def test_realtime_image_tokens_priced_per_token(model, provider, image_token_rate, _local_model_cost_map): + """Realtime image input is billed per 1M image tokens, not per image.""" + usage = Usage( + prompt_tokens=1_100, + completion_tokens=0, + total_tokens=1_100, + prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=100, image_tokens=1_000), + ) + prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider=provider) + text_rate = litellm.model_cost[model]["input_cost_per_token"] + assert prompt_cost == pytest.approx(100 * text_rate + 1_000 * image_token_rate) + + @pytest.mark.parametrize( ("response_quality", "requested_quality", "expected_cost"), [ diff --git a/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py b/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py index e72642f7a04..0b251be5408 100644 --- a/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py +++ b/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py @@ -18,6 +18,10 @@ NEW_MODELS: Final = ( "databricks/databricks-claude-opus-5", "databricks/databricks-claude-sonnet-5", "databricks/databricks-claude-fable-5", + "databricks/databricks-claude-fable-5-1", + "databricks/databricks-gpt-5-6-sol", + "databricks/databricks-gpt-5-6-terra", + "databricks/databricks-gpt-5-6-luna", ) DOLLARS_PER_DBU: Final = Decimal("0.070") @@ -28,6 +32,7 @@ PRICE_FIELDS: Final = ( "cache_read_input_token_cost", ) PUBLISHED_DBU_PER_MILLION: Final = { + "databricks/databricks-claude-fable-5-1": ("142.858", "714.286", "178.572", "3.572"), "databricks/databricks-claude-fable-5": ("142.858", "714.286", "178.572", "14.286"), "databricks/databricks-claude-opus-5": ("71.429", "357.143", "89.286", "7.143"), "databricks/databricks-claude-opus-4-8": ("71.429", "357.143", "89.286", "7.143"), @@ -52,9 +57,17 @@ PUBLISHED_DBU_PER_MILLION: Final = { "databricks/databricks-gpt-5-2": ("25.000", "200.000", "25.000", "2.500"), "databricks/databricks-gpt-5-2-codex": ("25.000", "200.000", "25.000", "2.500"), "databricks/databricks-gpt-5-3-codex": ("25.000", "200.000", "25.000", "2.500"), + "databricks/databricks-gpt-5-6-sol": ("57.143", "285.714", "71.429", "5.714"), + "databricks/databricks-gpt-5-6-terra": ("35.714", "214.286", "44.643", "3.571"), + "databricks/databricks-gpt-5-6-luna": ("14.286", "85.714", "17.857", "1.429"), + "databricks/databricks-gpt-5-5": ("71.429", "428.571", "71.429", "7.143"), + "databricks/databricks-gpt-5-5-pro": ("428.571", "2571.429", "428.571", "428.571"), "databricks/databricks-gpt-5-4": ("35.714", "214.286", "35.714", "3.571"), "databricks/databricks-gpt-5-4-mini": ("10.714", "64.286", "10.714", "1.071"), "databricks/databricks-gpt-5-4-nano": ("2.857", "17.857", "2.857", "0.286"), + "databricks/databricks-gemini-3-6-flash": ("26.786", "133.929", "26.786", "2.679"), + "databricks/databricks-gemini-3-5-flash": ("26.786", "160.714", "26.786", "2.679"), + "databricks/databricks-gemini-3-5-flash-lite": ("5.357", "44.643", "5.357", "0.536"), "databricks/databricks-gemini-3-1-pro": ("35.714", "214.286", "35.714", "3.571"), "databricks/databricks-gemini-3-pro": ("35.714", "214.286", "35.714", "3.571"), "databricks/databricks-gemini-3-flash": ("8.929", "53.571", "8.929", "0.893"), @@ -65,6 +78,13 @@ PUBLISHED_DBU_PER_MILLION: Final = { "databricks/databricks-deepseek-v4-flash-0731": ("2.000", "4.000", "2.000", "0.400"), "databricks/databricks-deepseek-v4-pro-0813": ("18.857", "56.571", "18.857", "1.886"), "databricks/databricks-glm-5-2": ("20.000", "62.857", "20.000", "3.714"), + "databricks/databricks-glm-5-3": ("20.000", "62.857", "20.000", "3.714"), + "databricks/databricks-glm-5-3-flash": ("2.143", "7.143", "2.143", "0.429"), + "databricks/databricks-inkling": ("14.286", "57.857", "14.286", "2.429"), + "databricks/databricks-grok-4-6": ("35.714", "107.143", "35.714", "8.929"), + "databricks/databricks-qwen35-122b-a10b": ("3.143", "31.429", "3.143", "3.143"), + "databricks/databricks-qwen3-next-80b-a3b-instruct": ("2.143", "17.143", "2.143", "2.143"), + "databricks/databricks-qwen3-embedding-0-6b": ("0.286", "0", "0.286", "0.286"), } PROMOTIONAL_DISCOUNT: Final = 0.80 PROMOTION_EXPIRES: Final = "2027-01-31" @@ -73,6 +93,10 @@ ENTRIES_STORING_PROMOTIONAL_RATE: Final = ( "databricks/databricks-gemini-2-5-flash", ) ENTRIES_STORING_LIST_RATE_DESPITE_PROMOTION: Final = ( + "databricks/databricks-gemini-3-6-flash", + "databricks/databricks-gemini-3-5-flash", + "databricks/databricks-gemini-3-5-flash-lite", + "databricks/databricks-grok-4-6", "databricks/databricks-gemini-3-1-pro", "databricks/databricks-gemini-3-pro", "databricks/databricks-gemini-3-flash", From c707f2fe5d771c240f03ef18882229773b3db90d Mon Sep 17 00:00:00 2001 From: mateo Date: Fri, 4 Sep 2026 14:32:11 +0000 Subject: [PATCH 55/88] fix(model_prices): databricks gpt-5-3-codex is served via the Responses API Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 2 +- model_prices_and_context_window.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7304ca04eb5..01bb7d434f5 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18089,7 +18089,7 @@ "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, - "mode": "chat", + "mode": "responses", "output_cost_per_token": 1.4e-05, "output_dbu_cost_per_token": 0.0002, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7304ca04eb5..01bb7d434f5 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18089,7 +18089,7 @@ "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, - "mode": "chat", + "mode": "responses", "output_cost_per_token": 1.4e-05, "output_dbu_cost_per_token": 0.0002, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", From 0c29f510bcdaa9f6d3cfb007c7ebda00458676ff Mon Sep 17 00:00:00 2001 From: mateo Date: Fri, 4 Sep 2026 15:03:29 +0000 Subject: [PATCH 56/88] fix(registry): drop gpt-image-2 text output price, add openrouter minimax-m3 and qwen3.7-plus Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 39 +++++++++++++++++-- model_prices_and_context_window.json | 39 +++++++++++++++++-- .../test_gpt_image_cost_calculator.py | 19 ++------- tests/test_litellm/test_utils.py | 4 +- 4 files changed, 76 insertions(+), 25 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 01bb7d434f5..1a92d78b053 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -8296,7 +8296,6 @@ "input_cost_per_image_token": 8e-06, "litellm_provider": "azure", "mode": "image_generation", - "output_cost_per_token": 1e-05, "output_cost_per_image_token": 3e-05, "supported_endpoints": [ "/v1/images/generations", @@ -8312,7 +8311,6 @@ "input_cost_per_image_token": 8e-06, "litellm_provider": "azure", "mode": "image_generation", - "output_cost_per_token": 1e-05, "output_cost_per_image_token": 3e-05, "supported_endpoints": [ "/v1/images/generations", @@ -29225,7 +29223,6 @@ "input_cost_per_token": 5e-06, "litellm_provider": "openai", "mode": "image_generation", - "output_cost_per_token": 1e-05, "input_cost_per_image_token": 8e-06, "output_cost_per_image_token": 3e-05, "supported_endpoints": [ @@ -29240,7 +29237,6 @@ "input_cost_per_token": 5e-06, "litellm_provider": "openai", "mode": "image_generation", - "output_cost_per_token": 1e-05, "input_cost_per_image_token": 8e-06, "output_cost_per_image_token": 3e-05, "supported_endpoints": [ @@ -60797,5 +60793,40 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true + }, + "openrouter/minimax/minimax-m3": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 512000, + "max_tokens": 512000, + "mode": "chat", + "source": "https://openrouter.ai/minimax/minimax-m3", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "cache_read_input_token_cost": 6e-08, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3.7-plus": { + "input_cost_per_token": 3.2e-07, + "output_cost_per_token": 1.28e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.7-plus", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "cache_read_input_token_cost": 6.4e-08, + "supports_prompt_caching": true, + "cache_creation_input_token_cost": 4e-07 } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 01bb7d434f5..1a92d78b053 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -8296,7 +8296,6 @@ "input_cost_per_image_token": 8e-06, "litellm_provider": "azure", "mode": "image_generation", - "output_cost_per_token": 1e-05, "output_cost_per_image_token": 3e-05, "supported_endpoints": [ "/v1/images/generations", @@ -8312,7 +8311,6 @@ "input_cost_per_image_token": 8e-06, "litellm_provider": "azure", "mode": "image_generation", - "output_cost_per_token": 1e-05, "output_cost_per_image_token": 3e-05, "supported_endpoints": [ "/v1/images/generations", @@ -29225,7 +29223,6 @@ "input_cost_per_token": 5e-06, "litellm_provider": "openai", "mode": "image_generation", - "output_cost_per_token": 1e-05, "input_cost_per_image_token": 8e-06, "output_cost_per_image_token": 3e-05, "supported_endpoints": [ @@ -29240,7 +29237,6 @@ "input_cost_per_token": 5e-06, "litellm_provider": "openai", "mode": "image_generation", - "output_cost_per_token": 1e-05, "input_cost_per_image_token": 8e-06, "output_cost_per_image_token": 3e-05, "supported_endpoints": [ @@ -60797,5 +60793,40 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true + }, + "openrouter/minimax/minimax-m3": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 512000, + "max_tokens": 512000, + "mode": "chat", + "source": "https://openrouter.ai/minimax/minimax-m3", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "cache_read_input_token_cost": 6e-08, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3.7-plus": { + "input_cost_per_token": 3.2e-07, + "output_cost_per_token": 1.28e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.7-plus", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "cache_read_input_token_cost": 6.4e-08, + "supports_prompt_caching": true, + "cache_creation_input_token_cost": 4e-07 } } diff --git a/tests/test_litellm/test_gpt_image_cost_calculator.py b/tests/test_litellm/test_gpt_image_cost_calculator.py index d3ec0673fe3..86a721f8743 100644 --- a/tests/test_litellm/test_gpt_image_cost_calculator.py +++ b/tests/test_litellm/test_gpt_image_cost_calculator.py @@ -172,8 +172,7 @@ class TestGPTImageCostCalculator: image_tokens=500, ), completion_tokens_details=CompletionTokensDetailsWrapper( - text_tokens=1000, - image_tokens=4000, + image_tokens=5000, ), ) @@ -189,12 +188,7 @@ class TestGPTImageCostCalculator: custom_llm_provider="openai", ) - # GPT Image 2 pricing: - # Text input: 100 * $5/1M = 0.0005 - # Image input: 500 * $8/1M = 0.004 - # Text output: 1000 * $10/1M = 0.01 - # Image output: 4000 * $30/1M = 0.12 - expected_cost = 0.0005 + 0.004 + 0.01 + 0.12 + expected_cost = 100 * 5e-6 + 500 * 8e-6 + 5000 * 3e-5 assert abs(cost - expected_cost) < 1e-6, f"Expected {expected_cost}, got {cost}" @@ -429,10 +423,7 @@ class TestGPTImage2OutputImageTokensNoBreakdown: f"are likely being priced at the text output_cost_per_token rate." ) - def test_gpt_image_2_chat_usage_without_breakdown_is_costed_not_zero(self): - """A chat ``Usage`` with ``completion_tokens_details=None`` must still be - costed via ``generic_cost_per_token`` (output at the text rate) rather than - erroring or silently returning 0.0.""" + def test_gpt_image_2_chat_usage_without_breakdown_uses_image_rate(self): from litellm.llms.openai.image_generation.cost_calculator import ( cost_calculator, ) @@ -460,9 +451,7 @@ class TestGPTImage2OutputImageTokensNoBreakdown: custom_llm_provider="openai", ) - # No output breakdown -> output priced at the text rate (output_cost_per_token): - # text in 100*$5/1M + image in 500*$8/1M + output 5000*$10/1M - expected_cost = 100 * 5e-6 + 500 * 8e-6 + 5000 * 1e-5 + expected_cost = 100 * 5e-6 + 500 * 8e-6 + 5000 * 3e-5 assert abs(cost - expected_cost) < 1e-6, f"Expected {expected_cost}, got {cost}" diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 38a09415ed0..a170bbee8e2 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -430,7 +430,7 @@ def test_gpt_image_2_provider_and_model_info(local_model_cost_map): assert model_info["mode"] == "image_generation" assert model_info["input_cost_per_token"] == 5e-06 assert model_info["input_cost_per_image_token"] == 8e-06 - assert model_info["output_cost_per_token"] == 1e-05 + assert model_info["output_cost_per_token"] == 0 assert model_info["output_cost_per_image_token"] == 3e-05 assert ( "/v1/images/generations" @@ -472,7 +472,7 @@ def test_azure_gpt_image_2_model_info(local_model_cost_map): assert model_info["mode"] == "image_generation" assert model_info["input_cost_per_token"] == 5e-06 assert model_info["input_cost_per_image_token"] == 8e-06 - assert model_info["output_cost_per_token"] == 1e-05 + assert model_info["output_cost_per_token"] == 0 assert model_info["output_cost_per_image_token"] == 3e-05 From 6c27754455b384a267bae21923a4f50a28d9d6f6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 09:26:51 -0700 Subject: [PATCH 57/88] fix(anthropic): bill an uncostable partial pass-through stream at zero cost instead of dropping its usage --- .../anthropic_passthrough_logging_handler.py | 22 ++++++++++++++----- ...t_anthropic_passthrough_logging_handler.py | 13 +++++++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index dae52bab956..30b75a7b482 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -240,18 +240,28 @@ class AnthropicPassthroughLoggingHandler: usage: Final = cast(Usage | None, getattr(partial_response, "usage", None)) if partial_response is None or usage is None: return + litellm_logging_obj.record_partial_usage_for_failure( + usage=usage, + response_cost=AnthropicPassthroughLoggingHandler._cost_partial_stream_or_zero( + partial_response=partial_response, model=model, logging_obj=litellm_logging_obj + ), + ) + + @staticmethod + def _cost_partial_stream_or_zero( + partial_response: ModelResponse | TextCompletionResponse, model: str, logging_obj: LiteLLMLoggingObj + ) -> float: try: - response_cost: Final = AnthropicPassthroughLoggingHandler._compute_response_cost( + return AnthropicPassthroughLoggingHandler._compute_response_cost( litellm_model_response=partial_response, - model=AnthropicPassthroughLoggingHandler._resolve_costing_model(model, litellm_logging_obj), - logging_obj=litellm_logging_obj, + model=AnthropicPassthroughLoggingHandler._resolve_costing_model(model, logging_obj), + logging_obj=logging_obj, ) - except Exception as e: # noqa: BLE001 # an uncostable partial stream must still log as a failure + except Exception as e: # noqa: BLE001 # an uncostable partial stream still bills its tokens, at zero cost verbose_proxy_logger.warning( "Anthropic passthrough: could not cost the partial usage of a failed stream (model=%s): %s", model, e ) - return - litellm_logging_obj.record_partial_usage_for_failure(usage=usage, response_cost=response_cost) + return 0.0 @staticmethod def _compute_response_cost( diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index 480da1d040e..d721be62efe 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -2505,6 +2505,19 @@ class TestRecordPartialUsageForFailure: assert usage.prompt_tokens == 52 assert logging_obj.model_call_details["response_cost"] > 0 + def test_stashes_partial_usage_at_zero_cost_when_model_is_unpriced(self): + logging_obj = self._make_logging_obj() + + AnthropicPassthroughLoggingHandler.record_partial_usage_for_failure( + litellm_logging_obj=logging_obj, + request_body={"model": "claude-unpriced-test-model", "stream": True}, + all_chunks=self._interrupted_chunks(), + ) + + usage = logging_obj.model_call_details["combined_usage_object"] + assert usage.prompt_tokens == 52 + assert logging_obj.model_call_details["response_cost"] == 0.0 + def test_leaves_logging_obj_untouched_when_nothing_streamed(self): logging_obj = self._make_logging_obj() From dbf8fe0f4e19f9425a7dd2753c3d96410efe6beb Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 4 Sep 2026 10:10:47 -0700 Subject: [PATCH 58/88] test: repair four chronically failing CI tests test_no_linear_scans_in_router: #39468 added config_deployments() and heuristic_v2_router_limit_violation(), which both scan the whole model_list from admin-only paths (model add/upsert), so add them to the allowlist. The allowlist becomes a mapping so each exemption carries its reason as data. test_missing_model_parameter_curl: a request with no model is rejected by the proxy when nothing can serve it and by the router when a wildcard or default deployment exists, and by the upstream provider when a wildcard forwards it, so the message text is not a stable contract. Assert the contract that holds in every case: HTTP 400 with a non-empty error message. test_model_group_info_e2e: /model_group/info resolves wildcards, so it can never return "anthropic/*" verbatim. cc3f9cd65b7 rewrote the assertion to expect the raw pattern after claude-3-5-haiku-20241022 left the price map, which made it unsatisfiable. Assert the expansion instead. test_should_derive_ocr_mapping_status_from_live_tests: the audit needs a native bridge built with the trace-parity feature, which CI never builds, so skip with the harness's own diagnostic instead of erroring. Extract that check out of ensure_trace_bridge as trace_bridge_error so a pytest run reports the state without kicking off a maturin rebuild. --- .../test_router_index_management.py | 10 ++-- .../shared/native_build.py | 17 ++++--- .../test_openai_error_handling.py | 13 ++++-- tests/test_models.py | 18 ++++---- tests/test_rust_python_harness.py | 46 +++++++++++++++++++ 5 files changed, 82 insertions(+), 22 deletions(-) diff --git a/tests/router_unit_tests/test_router_index_management.py b/tests/router_unit_tests/test_router_index_management.py index 87ddaadaf3d..35d295d581a 100644 --- a/tests/router_unit_tests/test_router_index_management.py +++ b/tests/router_unit_tests/test_router_index_management.py @@ -237,10 +237,12 @@ class TestRouterIndexManagement: - model_name_to_deployment_indices for O(1) + O(k) model_name lookups """ # Methods that are allowed to iterate through self.model_list - ALLOWED_METHODS = [ - "_get_deployment_by_litellm_model", # Edge case: lookup by litellm_params.model (not indexed) - "_finalize_adaptive_router_if_configured", # Init-time prefix scan for "auto_router/adaptive_router" (no index for prefix match) - ] + ALLOWED_METHODS = { + "_get_deployment_by_litellm_model": "lookup by litellm_params.model, which is not indexed", + "_finalize_adaptive_router_if_configured": 'init-time prefix scan for "auto_router/adaptive_router"; no index for prefix match', + "config_deployments": "filters the whole list on model_info.db_model; admin path only (model add/upsert)", + "heuristic_v2_router_limit_violation": "counts heuristic_v2 routers across the whole list; admin path only (auto-router init/upsert)", + } # Get path to router.py router_file = os.path.join( diff --git a/tests/rust-python-harness/shared/native_build.py b/tests/rust-python-harness/shared/native_build.py index 2ca7131c2c1..7df0f999847 100644 --- a/tests/rust-python-harness/shared/native_build.py +++ b/tests/rust-python-harness/shared/native_build.py @@ -73,6 +73,16 @@ def _rebuild(repo_root: Path) -> tuple[bool, str]: return completed.returncode == 0, "\n".join(lines[-_FAILURE_OUTPUT_LINES:]) +def trace_bridge_error() -> str | None: + """Why the installed bridge cannot serve trace parity, or None when it can. Never rebuilds.""" + bridge: Final = get_native_bridge() + if bridge is None: + return "native Rust bridge is not importable" + if getattr(bridge, "_trace", None) is None: + return f"native Rust bridge does not expose _trace; it must be built with the {BRIDGE_FEATURE} feature" + return None + + def ensure_trace_bridge(repo_root: Path) -> str | None: native_path: Final = _native_module_path() native_mtime: Final = native_path.stat().st_mtime if native_path is not None and native_path.exists() else None @@ -84,9 +94,4 @@ def ensure_trace_bridge(repo_root: Path) -> str | None: if not succeeded: return f"native Rust bridge rebuild failed:\n{output}" _drop_imported_bridge() - bridge: Final = get_native_bridge() - if bridge is None: - return "native Rust bridge is not importable" - if getattr(bridge, "_trace", None) is None: - return f"native Rust bridge does not expose _trace; it must be built with the {BRIDGE_FEATURE} feature" - return None + return trace_bridge_error() diff --git a/tests/store_model_in_db_tests/test_openai_error_handling.py b/tests/store_model_in_db_tests/test_openai_error_handling.py index 554ddf49cce..9a18d7f3420 100644 --- a/tests/store_model_in_db_tests/test_openai_error_handling.py +++ b/tests/store_model_in_db_tests/test_openai_error_handling.py @@ -106,15 +106,22 @@ def test_missing_model_parameter_curl(curl_command): # Run the curl command and capture the output key = generate_key_sync() curl_command = curl_command.replace("sk-1234", key) - result = subprocess.run(curl_command, shell=True, capture_output=True, text=True) + result = subprocess.run( + f'{curl_command} -s -w "\\n%{{http_code}}"', + shell=True, + capture_output=True, + text=True, + ) + body, _, status_code = result.stdout.rpartition("\n") # Parse the JSON response - response = json.loads(result.stdout) + response = json.loads(body) # Check that we got an error response assert "error" in response print("error in response", json.dumps(response, indent=4)) - assert "litellm.BadRequestError" in response["error"]["message"] + assert status_code == "400", f"expected HTTP 400, got {status_code}: {response}" + assert isinstance(response["error"]["message"], str) and response["error"]["message"] @pytest.mark.asyncio diff --git a/tests/test_models.py b/tests/test_models.py index 151fb70b665..186752af2bc 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -487,6 +487,9 @@ async def test_get_personal_models_for_user(): async def test_model_group_info_e2e(): """ Test /model/group/info endpoint + + The proxy config declares a wildcard "anthropic/*" deployment, and the endpoint resolves + wildcards into the concrete models they cover, so the raw pattern is never returned. """ async with aiohttp.ClientSession() as session: models = await get_models(session=session, key="sk-1234") @@ -495,16 +498,13 @@ async def test_model_group_info_e2e(): model_group_info = await get_model_group_info(session=session, key="sk-1234") print(model_group_info) - # Check that the endpoint returns data and contains the wildcard - # anthropic model group from the proxy config - has_anthropic_wildcard = False - for model in model_group_info["data"]: - if model["model_group"] == "anthropic/*": - has_anthropic_wildcard = True + model_groups = [m["model_group"] for m in model_group_info["data"]] - assert has_anthropic_wildcard, ( - f"Expected 'anthropic/*' in model groups, got: " - f"{[m['model_group'] for m in model_group_info['data']]}" + assert "anthropic/*" not in model_groups, ( + f"Expected 'anthropic/*' to be expanded, but it was returned verbatim: {model_groups}" + ) + assert any(m.startswith("anthropic/") for m in model_groups), ( + f"Expected concrete anthropic models from the 'anthropic/*' config entry, got: {model_groups}" ) diff --git a/tests/test_rust_python_harness.py b/tests/test_rust_python_harness.py index b27d1c83597..179660dd4e9 100644 --- a/tests/test_rust_python_harness.py +++ b/tests/test_rust_python_harness.py @@ -2,6 +2,7 @@ from __future__ import annotations import importlib from pathlib import Path +from types import SimpleNamespace from typing import Final import pytest @@ -13,6 +14,7 @@ mapping_validator = importlib.import_module("tests.rust-python-harness.strategie mappings = importlib.import_module("tests.rust-python-harness.strategies.unit_tests_mapping.mappings") ocr_mapping = importlib.import_module("tests.rust-python-harness.strategies.unit_tests_mapping.cases.ocr") cli = importlib.import_module("tests.rust-python-harness.cli") +native_build = importlib.import_module("tests.rust-python-harness.shared.native_build") audit_mapping = mapping_validator.audit_mapping UNIT_TEST_CONTRACTS = mappings.UNIT_TEST_CONTRACTS @@ -119,7 +121,51 @@ def test_should_leave_functions_without_mapping_contracts_unimplemented() -> Non assert "messages" not in UNIT_TEST_CONTRACTS +def test_should_report_a_bridge_that_cannot_be_imported() -> None: + with pytest.MonkeyPatch.context() as patch: + patch.setattr(native_build, "get_native_bridge", lambda: None) + message = native_build.trace_bridge_error() + + assert message is not None + assert "not importable" in message + + +def test_should_report_a_bridge_built_without_the_trace_feature() -> None: + with pytest.MonkeyPatch.context() as patch: + patch.setattr(native_build, "get_native_bridge", lambda: SimpleNamespace(_trace=None)) + message = native_build.trace_bridge_error() + + assert message is not None + assert native_build.BRIDGE_FEATURE in message + + +def test_should_accept_a_bridge_built_with_the_trace_feature() -> None: + with pytest.MonkeyPatch.context() as patch: + patch.setattr(native_build, "get_native_bridge", lambda: SimpleNamespace(_trace=object())) + + assert native_build.trace_bridge_error() is None + + +def test_should_not_rebuild_the_bridge_while_reporting_its_state() -> None: + rebuilds: list[object] = [] + + def fake_rebuild(repo_root: object) -> tuple[bool, str]: + rebuilds.append(repo_root) + return True, "" + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(native_build, "_rebuild", fake_rebuild) + patch.setattr(native_build, "get_native_bridge", lambda: None) + native_build.trace_bridge_error() + + assert rebuilds == [] + + def test_should_derive_ocr_mapping_status_from_live_tests() -> None: + bridge_error: Final = native_build.trace_bridge_error() + if bridge_error is not None: + pytest.skip(bridge_error) + report = audit_mapping(OCR_CONTRACT, repo_root=REPO_ROOT) assert report.is_valid, ( From a5a78670d047011f38dd8c11e7d2df818d2fba32 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 4 Sep 2026 10:18:15 -0700 Subject: [PATCH 59/88] test: address review notes on the chronic-test repairs Drop the two new docstrings, annotate the new locals Final, and replace the mutable call recorder with a rebuild stub that fails the test if it is ever reached. --- tests/rust-python-harness/shared/native_build.py | 1 - tests/test_models.py | 6 ++---- tests/test_rust_python_harness.py | 16 ++++++---------- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/tests/rust-python-harness/shared/native_build.py b/tests/rust-python-harness/shared/native_build.py index 7df0f999847..8693cf3bac2 100644 --- a/tests/rust-python-harness/shared/native_build.py +++ b/tests/rust-python-harness/shared/native_build.py @@ -74,7 +74,6 @@ def _rebuild(repo_root: Path) -> tuple[bool, str]: def trace_bridge_error() -> str | None: - """Why the installed bridge cannot serve trace parity, or None when it can. Never rebuilds.""" bridge: Final = get_native_bridge() if bridge is None: return "native Rust bridge is not importable" diff --git a/tests/test_models.py b/tests/test_models.py index 186752af2bc..64c7dcd83da 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -6,6 +6,7 @@ import asyncio import aiohttp import os import dotenv +from typing import Final from dotenv import load_dotenv load_dotenv() @@ -487,9 +488,6 @@ async def test_get_personal_models_for_user(): async def test_model_group_info_e2e(): """ Test /model/group/info endpoint - - The proxy config declares a wildcard "anthropic/*" deployment, and the endpoint resolves - wildcards into the concrete models they cover, so the raw pattern is never returned. """ async with aiohttp.ClientSession() as session: models = await get_models(session=session, key="sk-1234") @@ -498,7 +496,7 @@ async def test_model_group_info_e2e(): model_group_info = await get_model_group_info(session=session, key="sk-1234") print(model_group_info) - model_groups = [m["model_group"] for m in model_group_info["data"]] + model_groups: Final = [m["model_group"] for m in model_group_info["data"]] assert "anthropic/*" not in model_groups, ( f"Expected 'anthropic/*' to be expanded, but it was returned verbatim: {model_groups}" diff --git a/tests/test_rust_python_harness.py b/tests/test_rust_python_harness.py index 179660dd4e9..85b45c07bc2 100644 --- a/tests/test_rust_python_harness.py +++ b/tests/test_rust_python_harness.py @@ -124,7 +124,7 @@ def test_should_leave_functions_without_mapping_contracts_unimplemented() -> Non def test_should_report_a_bridge_that_cannot_be_imported() -> None: with pytest.MonkeyPatch.context() as patch: patch.setattr(native_build, "get_native_bridge", lambda: None) - message = native_build.trace_bridge_error() + message: Final = native_build.trace_bridge_error() assert message is not None assert "not importable" in message @@ -133,7 +133,7 @@ def test_should_report_a_bridge_that_cannot_be_imported() -> None: def test_should_report_a_bridge_built_without_the_trace_feature() -> None: with pytest.MonkeyPatch.context() as patch: patch.setattr(native_build, "get_native_bridge", lambda: SimpleNamespace(_trace=None)) - message = native_build.trace_bridge_error() + message: Final = native_build.trace_bridge_error() assert message is not None assert native_build.BRIDGE_FEATURE in message @@ -147,18 +147,14 @@ def test_should_accept_a_bridge_built_with_the_trace_feature() -> None: def test_should_not_rebuild_the_bridge_while_reporting_its_state() -> None: - rebuilds: list[object] = [] - - def fake_rebuild(repo_root: object) -> tuple[bool, str]: - rebuilds.append(repo_root) - return True, "" + def forbidden_rebuild(repo_root: object) -> tuple[bool, str]: + raise AssertionError("trace_bridge_error must not rebuild the native bridge") with pytest.MonkeyPatch.context() as patch: - patch.setattr(native_build, "_rebuild", fake_rebuild) + patch.setattr(native_build, "_rebuild", forbidden_rebuild) patch.setattr(native_build, "get_native_bridge", lambda: None) - native_build.trace_bridge_error() - assert rebuilds == [] + assert native_build.trace_bridge_error() is not None def test_should_derive_ocr_mapping_status_from_live_tests() -> None: From 4bd3cd9e06c46b8e8aa0459de93e98c527674985 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 4 Sep 2026 11:05:58 -0700 Subject: [PATCH 60/88] ci: report every failing test in a job instead of stopping at the first Drops `-x` from all 24 pytest invocations in .circleci/config.yml. With `-x`, a job stops at its first failure, so a second broken test in the same suite stays invisible until the first is fixed and CI is re-run. That turns one round trip into N when a job has several broken tests. This is exactly what happened in #39770: fixing test_missing_model_parameter_curl in tests/store_model_in_db_tests/test_openai_error_handling.py immediately unmasked test_chat_completion_bad_model_with_spend_logs in the same file, which had been failing for a long time without ever being reported. Only `-x` is removed; -v/-vv/-s/-n/--reruns and every other flag are untouched. --- .circleci/config.yml | 48 ++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dfc539fb80e..6e368a3debe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -575,7 +575,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ + -v \ --junitxml=test-results/junit.xml \ --durations=5 \ -k \"langfuse\"" @@ -630,7 +630,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ + -v \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ @@ -737,7 +737,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ + -v \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ @@ -782,7 +782,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ + -v \ --junitxml=test-results/junit.xml \ --durations=5 \ -k \"assistants\"" @@ -909,7 +909,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x -s \ + -vv -s \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5" @@ -999,7 +999,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x -s \ + -vv -s \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ @@ -1054,7 +1054,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ + -v \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 8 \ @@ -1090,7 +1090,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x \ + -vv \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ @@ -1134,7 +1134,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x \ + -vv \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ @@ -1178,7 +1178,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x -s \ + -vv -s \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ @@ -1222,7 +1222,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x -s \ + -vv -s \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ @@ -1267,7 +1267,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x \ + -vv \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ @@ -1312,7 +1312,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ + -v \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 4" @@ -1391,7 +1391,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x -s \ + -vv -s \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5" @@ -1444,7 +1444,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x -s \ + -vv -s \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 -n 2 \ @@ -1705,7 +1705,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ + -v \ --junitxml=test-results/junit-2.xml \ --durations=5" no_output_timeout: 15m @@ -1794,7 +1794,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -s -v -x \ + -s -v \ --junitxml=test-results/junit.xml \ -n 4 \ --durations=5" @@ -2012,7 +2012,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ + -v \ --junitxml=test-results/junit-2.xml \ --durations=5" no_output_timeout: 15m @@ -2092,7 +2092,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x \ + -vv \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m @@ -2195,7 +2195,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x \ + -vv \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m @@ -2266,7 +2266,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x \ + -vv \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m @@ -2350,7 +2350,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x \ + -vv \ --junitxml=test-results/junit-2.xml \ --durations=5" no_output_timeout: 15m @@ -2446,7 +2446,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ + -v \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m @@ -2516,7 +2516,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x -s \ + -vv -s \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m From 7d3b03d00654600907826772653cfd57a1be1284 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 4 Sep 2026 11:09:01 -0700 Subject: [PATCH 61/88] test(caching): drive the redis stall burst off the clock, not asyncio.wait_for test_event_loop_stall_timeout_burst_keeps_breaker_closed built its timeout burst by wrapping a healthy fake call in asyncio.wait_for. Before 3.12, wait_for returns the inner result when the inner future also completed while the loop was blocked, so no call timed out, the burst never materialised, and the test's own liveness guard failed with 0 >= 3. The fake now checks its own client deadline against the clock, the way a client library does, so the stall produces a real redis TimeoutError burst on every interpreter. The breaker itself is unchanged: its duration gate is plain time.time() bookkeeping and never depended on the version. --- tests/test_litellm/caching/test_redis_cache.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index 2a0119bcfb8..2f412e7382b 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -823,15 +823,26 @@ async def test_event_loop_stall_timeout_burst_keeps_breaker_closed(): Every operation already waiting on the loop times out together when the loop resumes, so a purely consecutive threshold is satisfied instantly even though the Redis on the other end (here an in-process fake that answers immediately) is healthy. + + The fake checks its own client deadline against the clock, the way a client library + does, rather than wrapping the call in asyncio.wait_for: before 3.12 wait_for returns + the inner result when the inner future also completed during the stall, so the burst + never materialises and the test cannot exercise the duration gate. """ import time as time_mod + from redis.exceptions import TimeoutError as RedisTimeoutError + from litellm.caching.redis_cache import RedisCircuitBreaker, _run_under_circuit_breaker breaker = RedisCircuitBreaker(failure_threshold=3, recovery_timeout=60, timeout_min_duration=5.0) async def healthy_redis_call_with_client_timeout(): - return await asyncio.wait_for(asyncio.sleep(0.001, result="ok"), timeout=0.05) + deadline = time_mod.monotonic() + 0.05 + await asyncio.sleep(0.001) + if time_mod.monotonic() > deadline: + raise RedisTimeoutError("read timed out") + return "ok" async def stall_the_loop(): await asyncio.sleep(0) @@ -842,7 +853,7 @@ async def test_event_loop_stall_timeout_burst_keeps_breaker_closed(): stall_the_loop(), return_exceptions=True, ) - timeouts = [r for r in results if isinstance(r, asyncio.TimeoutError)] + timeouts = [r for r in results if isinstance(r, RedisTimeoutError)] assert len(timeouts) >= breaker.failure_threshold, "the stall must time out a full burst" assert breaker.is_open() is False, "a healthy Redis behind one loop stall must stay in the pool" From 4f3b02360e6a0ff23cfa821782f7fea2d3d7a90b Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 11:21:37 -0700 Subject: [PATCH 62/88] test(organization): type the legacy update helper's request body precisely --- .../proxy/management_endpoints/test_organization_endpoints.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py index 5c2a0bdde3d..dc500df6fd6 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py @@ -963,7 +963,9 @@ async def test_v2_serializes_model_max_budget_on_budget_write(monkeypatch): assert json.loads(written) == {"gpt-4o": {"max_budget": 10}} -async def _run_legacy_update_organization(monkeypatch, *, body: dict, existing_budget_id: str): +async def _run_legacy_update_organization( + monkeypatch: pytest.MonkeyPatch, *, body: dict[str, object], existing_budget_id: str +) -> AsyncMock: from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.management_endpoints import organization_endpoints from litellm.proxy.management_endpoints.organization_endpoints import update_organization From caa1ab0e60d2047ba155e1c9c69db62693ceeff6 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 11:27:57 -0700 Subject: [PATCH 63/88] fix(guardrails): store an unpriced Bedrock counter as unknown, not free A counter missing from the cost map entry was priced at 0.0 per unit, so the rollup recorded it as known-free usage. It now stamps None for that counter and the rollup writes NULL, while the per-request guardrail_cost that feeds spend and budgets still sums only the known prices. Claude-Session: https://claude.ai/code/session_01EX13mWex6RaBo9PYnkAtFW --- .../llm_cost_calc/guardrail_cost.py | 25 +++++++++++++------ litellm/types/utils.py | 7 +++--- .../llm_cost_calc/test_guardrail_cost.py | 23 ++++++++++++----- .../test_bedrock_guardrails.py | 17 +++++++++++-- .../proxy/guardrails/test_usage_tracking.py | 24 ++++++++++++++++++ 5 files changed, 77 insertions(+), 19 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py b/litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py index 64e82053c94..54cdf2cb8ff 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py +++ b/litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py @@ -36,16 +36,17 @@ class GuardrailCostByUnitEntry(BaseModel): model_config = ConfigDict(extra="ignore", frozen=True) - guardrail_cost_by_unit: Mapping[str, Annotated[float, Field(ge=0, allow_inf_nan=False)]] | None = None + guardrail_cost_by_unit: Mapping[str, Annotated[float, Field(ge=0, allow_inf_nan=False)] | None] | None = None guardrail_cost_in_spend: bool | None = True _GUARDRAIL_COST_BY_UNIT_ADAPTER: Final[TypeAdapter[GuardrailCostByUnitEntry]] = TypeAdapter(GuardrailCostByUnitEntry) -def billed_guardrail_cost_by_unit(raw: object) -> Mapping[str, float] | None: +def billed_guardrail_cost_by_unit(raw: object) -> Mapping[str, float | None] | None: """Per-counter USD the daily rollup may record for one raw ``guardrail_information`` - entry; None when the entry is unpriced, report-only, or malformed.""" + entry; None when the entry is unpriced, report-only, or malformed, and None per + counter the hook had no price for.""" try: entry: Final = _GUARDRAIL_COST_BY_UNIT_ADAPTER.validate_python(raw) except ValidationError as e: @@ -66,20 +67,28 @@ def _bedrock_guardrail_pricing(aws_region_name: str | None) -> GuardrailPricing return None +def _priced_units(units: int, price_per_unit: float | None) -> float | None: + return None if price_per_unit is None else units * price_per_unit + + def bedrock_guardrail_cost_by_unit( usage_units: Mapping[str, int], aws_region_name: str | None -) -> Mapping[str, float] | None: - """USD per counter, keyed like ``usage_units``; None when no pricing entry exists.""" +) -> Mapping[str, float | None] | None: + """USD per counter, keyed like ``usage_units``; None when no pricing entry exists, + and None for a counter the entry has no price for, since only an explicit 0.0 means free.""" pricing: Final = _bedrock_guardrail_pricing(aws_region_name) if pricing is None: return None return { # mutable-ok: stamped into guardrail_information, which safe_dumps only serializes as a plain dict - counter: units * pricing.guardrail_cost_per_unit.get(counter, 0.0) for counter, units in usage_units.items() + counter: _priced_units(units, pricing.guardrail_cost_per_unit.get(counter)) + for counter, units in usage_units.items() } -def guardrail_cost_total(cost_by_unit: Mapping[str, float] | None) -> float: - return sum(cost_by_unit.values()) if cost_by_unit is not None else 0.0 +def guardrail_cost_total(cost_by_unit: Mapping[str, float | None] | None) -> float: + """The scalar the spend path bills: unknown-priced counters count as 0 here, the + rollup keeps them unknown.""" + return sum(cost for cost in cost_by_unit.values() if cost is not None) if cost_by_unit is not None else 0.0 def bedrock_guardrail_cost(usage_units: Mapping[str, int], aws_region_name: str | None) -> float: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 8a6b1c13b2d..6c645226e2e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3142,10 +3142,11 @@ class StandardLoggingGuardrailInformation(TypedDict, total=False): provider hook. Summed into the request's ``response_cost`` so it counts against spend and budgets like token cost, unless ``guardrail_cost_in_spend`` is False.""" - guardrail_cost_by_unit: ReadOnly[Mapping[str, float] | None] + guardrail_cost_by_unit: ReadOnly[Mapping[str, float | None] | None] """``guardrail_cost`` split per ``guardrail_usage`` counter, so the daily per-counter usage rollup can carry cost at its own grain. Absent when the - hook had no pricing for the invocation.""" + hook had no pricing for the invocation; a counter is None when the pricing + entry has no price for it, which the rollup stores as unknown rather than $0.""" guardrail_cost_in_spend: ReadOnly[bool | None] """Whether ``guardrail_cost`` participates in the request's ``response_cost`` and @@ -3198,7 +3199,7 @@ class GuardrailTracingDetail(TypedDict, total=False): guardrail_action: str | None guardrail_usage: ReadOnly[Mapping[str, int] | None] guardrail_cost: ReadOnly[float | None] - guardrail_cost_by_unit: ReadOnly[Mapping[str, float] | None] + guardrail_cost_by_unit: ReadOnly[Mapping[str, float | None] | None] guardrail_cost_in_spend: ReadOnly[bool | None] diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py index 6e9920d6f1d..af2f169157e 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py @@ -8,6 +8,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import ( bedrock_guardrail_cost_by_unit, billed_guardrail_cost_by_unit, cost_breakdown_with_guardrail, + guardrail_cost_total, guardrail_information_cost, ) @@ -60,16 +61,19 @@ def test_bedrock_guardrail_cost_no_pricing_entry(monkeypatch): def test_bedrock_guardrail_cost_by_unit_prices_every_counter_it_was_given(synthetic_cost_map): """LIT-5652: the daily rollup stores one row per counter, so pricing must come - back at that grain, keyed exactly like the usage (free and unknown counters - included at 0.0) and summing to the scalar the spend path bills.""" + back at that grain, keyed exactly like the usage. An explicit 0.0 in the cost + map is free; a counter the map does not list is unknown (None), never free, + while the scalar the spend path bills still sums only the known prices.""" usage = {"contentPolicyUnits": 2, "topicPolicyUnits": 1, "wordPolicyUnits": 5, "someFutureCounter": 3} by_unit = bedrock_guardrail_cost_by_unit(usage_units=usage, aws_region_name="us-east-1") assert by_unit is not None assert by_unit.keys() == usage.keys() assert by_unit["contentPolicyUnits"] == pytest.approx(0.0003) assert by_unit["topicPolicyUnits"] == pytest.approx(0.00015) - assert (by_unit["wordPolicyUnits"], by_unit["someFutureCounter"]) == (0.0, 0.0) - assert sum(by_unit.values()) == pytest.approx( + assert by_unit["wordPolicyUnits"] == 0.0 + assert by_unit["someFutureCounter"] is None + assert guardrail_cost_total(by_unit) == pytest.approx(0.00045) + assert guardrail_cost_total(by_unit) == pytest.approx( bedrock_guardrail_cost(usage_units=usage, aws_region_name="us-east-1") ) @@ -83,8 +87,15 @@ def test_bedrock_guardrail_cost_by_unit_is_none_without_pricing_so_unpriced_is_n def test_billed_guardrail_cost_by_unit_reads_the_hook_stamp(): - entry = {"guardrail_name": "bedrock", "guardrail_cost_by_unit": {"contentPolicyUnits": 0.15, "wordPolicyUnits": 0}} - assert billed_guardrail_cost_by_unit(entry) == {"contentPolicyUnits": 0.15, "wordPolicyUnits": 0.0} + entry = { + "guardrail_name": "bedrock", + "guardrail_cost_by_unit": {"contentPolicyUnits": 0.15, "wordPolicyUnits": 0, "someFutureCounter": None}, + } + assert billed_guardrail_cost_by_unit(entry) == { + "contentPolicyUnits": 0.15, + "wordPolicyUnits": 0.0, + "someFutureCounter": None, + } @pytest.mark.parametrize( diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index ec8996a8489..1ed24c9a59b 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -5095,18 +5095,31 @@ def test_build_tracing_detail_surfaces_usage_counters_and_cost(monkeypatch): detail = guardrail._build_tracing_detail( { "action": "GUARDRAIL_INTERVENED", - "usage": {"topicPolicyUnits": 1, "contentPolicyUnits": 2, "wordPolicyUnits": 0, "oddball": "not-an-int"}, + "usage": { + "topicPolicyUnits": 1, + "contentPolicyUnits": 2, + "wordPolicyUnits": 0, + "someFutureCounter": 3, + "oddball": "not-an-int", + }, }, aws_region_name="us-east-1", ) - assert detail["guardrail_usage"] == {"topicPolicyUnits": 1, "contentPolicyUnits": 2, "wordPolicyUnits": 0} + assert detail["guardrail_usage"] == { + "topicPolicyUnits": 1, + "contentPolicyUnits": 2, + "wordPolicyUnits": 0, + "someFutureCounter": 3, + } assert detail["guardrail_cost"] == pytest.approx(0.00045) by_unit = detail["guardrail_cost_by_unit"] assert by_unit is not None and by_unit.keys() == detail["guardrail_usage"].keys() assert by_unit["topicPolicyUnits"] == pytest.approx(0.00015) assert by_unit["contentPolicyUnits"] == pytest.approx(0.0003) assert by_unit["wordPolicyUnits"] == 0.0 + assert by_unit["someFutureCounter"] is None + assert by_unit["wordPolicyUnits"] == 0.0 def test_build_tracing_detail_omits_cost_by_unit_when_unpriced_but_keeps_scalar_zero(monkeypatch): diff --git a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py index 50845385443..347c65cf819 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py @@ -373,6 +373,30 @@ async def test_cost_rolled_up_per_counter_alongside_units(): assert costs["wordPolicyUnits"] == (0.0, {"increment": 0.0}) +@pytest.mark.asyncio +async def test_counter_the_hook_could_not_price_is_stored_unknown_not_free(): + """A counter the cost map does not list arrives stamped as None. Its row must + carry NULL, while the priced counter on the same request keeps its cost.""" + prisma = _prisma() + logs = [ + _payload( + "r1", + usage={"contentPolicyUnits": 1000, "someFutureCounter": 3}, + cost_by_unit={"contentPolicyUnits": 0.15, "someFutureCounter": None}, + ) + ] + + await process_spend_logs_guardrail_usage(prisma, logs) + + assert _units_upserts(prisma) == { + ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "contentPolicyUnits"): 1000, + ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "someFutureCounter"): 3, + } + costs = _cost_upserts(prisma) + assert costs["contentPolicyUnits"] == (pytest.approx(0.15), {"increment": pytest.approx(0.15)}) + assert costs["someFutureCounter"] == (None, None) + + @pytest.mark.asyncio async def test_unpriced_increment_makes_the_rows_cost_unknown_not_partial(): """A payload with usage but no per-counter cost (a hook without pricing, a From d05d2a6f0519a4a0ca752e6881ae8290e6173576 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 11:33:26 -0700 Subject: [PATCH 64/88] fix(ui): clamp server-paginated DataTable page index when rowCount shrinks Server-mode tables kept whatever page index the user was on after the server's total dropped below it, for example after deleting the last rows of the final page or when a refetch came back empty. The footer then read "Page 2 of 1" and "Showing 26-25 of 25" with Previous and First enabled over an empty body, and every one of the 13 server-mode consumers was exposed since none of them clamped The shared DataTable now snaps the controlled page index to the last valid page as soon as a non-loading rowCount no longer reaches it, so the fix applies to every consumer without per-table clamps. Loading responses are ignored so a pending fetch never bounces the user to page 1 --- .../shared/DataTable/DataTable.test.tsx | 67 ++++++++++++++++++- .../components/shared/DataTable/DataTable.tsx | 20 +++++- 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx index 149554a3ac3..162fc39a632 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx @@ -1,4 +1,4 @@ -import type { ColumnDef, ExpandedState } from "@tanstack/react-table"; +import type { ColumnDef, ExpandedState, OnChangeFn, PaginationState } from "@tanstack/react-table"; import { render, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { useState } from "react"; @@ -274,6 +274,71 @@ describe("DataTable pagination", () => { await user.click(screen.getByTestId("pagination-next")); expect(onPaginationChange).toHaveBeenCalledTimes(1); }); + + type ServerPageHarnessProps = { + rowCount: number; + isLoading?: boolean; + initialPageIndex: number; + onChange: (next: PaginationState) => void; + }; + + function ServerPageHarness({ rowCount, isLoading = false, initialPageIndex, onChange }: ServerPageHarnessProps) { + const [pagination, setPagination] = useState({ pageIndex: initialPageIndex, pageSize: 10 }); + const handleChange: OnChangeFn = (updater) => { + const next = typeof updater === "function" ? updater(pagination) : updater; + onChange(next); + setPagination(next); + }; + return ( + + ); + } + + it("server mode snaps to the last page when rowCount no longer reaches the current page", async () => { + const onChange = vi.fn(); + render(); + + await waitFor(() => expect(onChange).toHaveBeenCalledWith({ pageIndex: 1, pageSize: 10 })); + expect(onChange).toHaveBeenCalledTimes(1); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 11-15 of 15"); + expect(screen.getByText("Page 2 of 2")).toBeInTheDocument(); + expect(screen.getByTestId("pagination-next")).toBeDisabled(); + }); + + it("server mode falls back to the first page when rowCount drops to zero", async () => { + const onChange = vi.fn(); + render(); + + await waitFor(() => expect(onChange).toHaveBeenCalledWith({ pageIndex: 0, pageSize: 10 })); + expect(onChange).toHaveBeenCalledTimes(1); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("No results"); + expect(screen.getByText("Page 1 of 1")).toBeInTheDocument(); + expect(screen.getByTestId("pagination-first")).toBeDisabled(); + expect(screen.getByTestId("pagination-prev")).toBeDisabled(); + }); + + it("server mode leaves the page index alone while loading and clamps once the response lands", async () => { + const onChange = vi.fn(); + const { rerender } = render(); + + expect(screen.getByText("Page 3 of 1")).toBeInTheDocument(); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(onChange).not.toHaveBeenCalled(); + + rerender(); + + await waitFor(() => expect(onChange).toHaveBeenCalledWith({ pageIndex: 1, pageSize: 10 })); + expect(onChange).toHaveBeenCalledTimes(1); + expect(screen.getByText("Page 2 of 2")).toBeInTheDocument(); + }); }); describe("DataTable filtering", () => { diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx index 17a5fe42d1c..58f9657a001 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx @@ -16,6 +16,7 @@ import { getSortedRowModel, type Header, type OnChangeFn, + type PaginationState, type Row, type RowData, type RowSelectionState, @@ -26,7 +27,7 @@ import { } from "@tanstack/react-table"; import { SearchX } from "lucide-react"; import * as React from "react"; -import { Fragment, useState } from "react"; +import { Fragment, useEffect, useState } from "react"; import { Skeleton } from "@/components/ui/skeleton"; import { @@ -417,6 +418,21 @@ function useControllable( return { value: internal, onChange: setInternal }; } +function useServerPageClamp( + active: boolean, + rowCount: number | undefined, + pagination: { value: PaginationState; onChange: OnChangeFn }, +): void { + const { pageIndex, pageSize } = pagination.value; + const { onChange } = pagination; + useEffect(() => { + if (!active || rowCount === undefined) return; + const lastPageIndex = Math.max(Math.ceil(rowCount / pageSize) - 1, 0); + if (pageIndex <= lastPageIndex) return; + onChange({ pageIndex: lastPageIndex, pageSize }); + }, [active, rowCount, pageIndex, pageSize, onChange]); +} + function useDataTableInstance( props: DataTableResolvedProps, ): Table { @@ -433,6 +449,7 @@ function useDataTableInstance( pagination, onPaginationChange, rowCount, + isLoading = false, pageSizeOptions = DEFAULT_PAGE_SIZE_OPTIONS, filterMode = "none", columnFilters, @@ -457,6 +474,7 @@ function useDataTableInstance( pageIndex: 0, pageSize: pageSizeOptions[0] ?? 25, }); + useServerPageClamp(paginationMode === "server" && !isLoading, rowCount, paginationState); const filterState = useControllable( columnFilters, onColumnFiltersChange, From dd01abc4390f494eedc5fe448b70c7b3de06a1c7 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:00:47 -0700 Subject: [PATCH 65/88] feat(team): report per-user spend within a team for JWT traffic (#39771) * feat(team): report per-user spend within a team for JWT traffic Add GET /team/spend/by_user, which groups raw spend logs by (team_id, user) so JWT/SSO requests with no virtual key are attributed to the user inside each selected team. Team admins see every member, plain members see only their own row. The Team Usage page gets a Spend Per User Within Team card with CSV export backed by the same endpoint. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(team): cover /team/spend/by_user in behavior suite, tf audit allowlist and EntityUsage unit test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(team): drop explanatory docstrings from /team/spend/by_user and regen schema.d.ts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 2 + .../management_endpoints/team_endpoints.py | 123 +++++++++++++ .../management_endpoints/team_endpoints.py | 21 +++ .../endpointaudit/coverage_allowlist.txt | 1 + .../management/test_team_spend_by_user.py | 58 ++++++ .../proxy/auth/test_route_checks.py | 35 ++++ .../test_team_endpoints.py | 172 ++++++++++++++++++ .../EntityUsage/EntityUsage.test.tsx | 25 +++ .../components/EntityUsage/EntityUsage.tsx | 19 ++ .../EntityUsage/TeamUserSpendCard.tsx | 109 +++++++++++ .../EntityUsage/teamUserSpend.test.ts | 93 ++++++++++ .../components/EntityUsage/teamUserSpend.ts | 55 ++++++ .../src/components/networking.tsx | 18 ++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 114 ++++++++++++ 14 files changed, 845 insertions(+) create mode 100644 tests/proxy_behavior/management/test_team_spend_by_user.py create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/TeamUserSpendCard.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/teamUserSpend.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/teamUserSpend.ts diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 832d941f5b5..b33e2fe7ff6 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -670,6 +670,7 @@ class LiteLLMRoutes(enum.Enum): "/team/permissions_bulk_update", "/team/daily/activity", "/team/daily/activity/aggregated", + "/team/spend/by_user", # gateway request counts (SGR); deployment-wide, admin-only "/gateway/daily/activity", # model @@ -832,6 +833,7 @@ class LiteLLMRoutes(enum.Enum): "/team/permissions_update", "/team/daily/activity", "/team/daily/activity/aggregated", + "/team/spend/by_user", "/team/{team_id}/members/me", "/model/new", "/model/update", diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 90d7539b38d..a504c1c5e43 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -170,6 +170,8 @@ from litellm.types.proxy.management_endpoints.team_endpoints import ( TeamMemberAddResult, TeamMemberInfoResponse, TeamMetadataSchemaResponse, + TeamUserSpendResponse, + TeamUserSpendRow, UpdateTeamMemberPermissionsRequest, ) @@ -6231,3 +6233,124 @@ async def get_team_daily_activity_aggregated( timezone_offset_minutes=timezone, include_entity_breakdown=True, ) + + +def _team_user_spend_sql(*, team_count: int, restrict_to_user: bool) -> str: + team_placeholders: Final = ", ".join(f"${i}" for i in range(3, 3 + team_count)) + user_clause: Final = f' AND sl."user" = ${3 + team_count}' if restrict_to_user else "" + return f""" + SELECT + sl.team_id, + sl."user" AS user_id, + u.user_email, + u.user_alias, + SUM(sl.spend)::float AS spend, + SUM(sl.prompt_tokens)::bigint AS prompt_tokens, + SUM(sl.completion_tokens)::bigint AS completion_tokens, + SUM(sl.total_tokens)::bigint AS total_tokens, + COUNT(*)::bigint AS api_requests, + COUNT(*) FILTER (WHERE sl.status IS DISTINCT FROM 'failure')::bigint AS successful_requests, + COUNT(*) FILTER (WHERE sl.status = 'failure')::bigint AS failed_requests + FROM "LiteLLM_SpendLogs" sl + LEFT JOIN "LiteLLM_UserTable" u ON u.user_id = sl."user" + WHERE sl."startTime" >= $1::timestamp + AND sl."startTime" < $2::timestamp + INTERVAL '1 day' + AND sl.team_id IN ({team_placeholders}){user_clause} + GROUP BY sl.team_id, sl."user", u.user_email, u.user_alias + ORDER BY spend DESC, sl.team_id, sl."user" + """ + + +class _TeamUserSpendDbRow(TypedDict): + team_id: ReadOnly[str] + user_id: ReadOnly[str | None] + user_email: ReadOnly[str | None] + user_alias: ReadOnly[str | None] + spend: ReadOnly[float] + prompt_tokens: ReadOnly[int] + completion_tokens: ReadOnly[int] + total_tokens: ReadOnly[int] + api_requests: ReadOnly[int] + successful_requests: ReadOnly[int] + failed_requests: ReadOnly[int] + + +@router.get( + "/team/spend/by_user", + response_model=TeamUserSpendResponse, + tags=["team management"], # mutable-ok: fastapi route tags must be a list +) +async def get_team_spend_by_user( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + team_ids: str | None = None, + start_date: str | None = None, + end_date: str | None = None, +) -> TeamUserSpendResponse: + """ + Spend per user within the given teams, attributed per request from spend logs. + + Proxy admins may query any team. Team admins and members holding the + `/team/daily/activity` permission see every user of the requested teams; + other members only see their own row. + """ + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + + if prisma_client is None: + raise _daily_activity_error(status_code=500, message=CommonProxyErrors.db_not_connected_error.value) + + range_error: Final = _aggregated_date_range_error(start_date, end_date) + if range_error is not None or start_date is None or end_date is None: + raise _daily_activity_error(status_code=400, message=range_error or "Please provide start_date and end_date") + + if not team_ids: + raise _daily_activity_error(status_code=400, message="Please provide team_ids") + + scope: Final = await _resolve_team_daily_activity_scope( + team_ids=team_ids, + exclude_team_ids=None, + api_key=None, + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + scoped_team_ids: Final = tuple(scope.team_ids or ()) + if not scoped_team_ids: + return TeamUserSpendResponse(start_date=start_date, end_date=end_date, results=()) + + own_user_only: Final = scope.api_key_filter is not None + user_param: Final = (user_api_key_dict.user_id or "",) if own_user_only else () + rows: Final[Sequence[_TeamUserSpendDbRow]] = await prisma_client.db.query_raw( + _team_user_spend_sql(team_count=len(scoped_team_ids), restrict_to_user=own_user_only), + start_date, + end_date, + *scoped_team_ids, + *user_param, + ) + results: Final = tuple( + TeamUserSpendRow( + team_id=row["team_id"], + team_alias=_team_alias_or_none(scope.team_alias_metadata.get(row["team_id"])), + user_id=row["user_id"] or "", + user_email=row["user_email"], + user_alias=row["user_alias"], + spend=row["spend"], + prompt_tokens=row["prompt_tokens"], + completion_tokens=row["completion_tokens"], + total_tokens=row["total_tokens"], + api_requests=row["api_requests"], + successful_requests=row["successful_requests"], + failed_requests=row["failed_requests"], + ) + for row in rows + ) + return TeamUserSpendResponse(start_date=start_date, end_date=end_date, results=results) + + +def _team_alias_or_none(metadata: Mapping[str, object] | None) -> str | None: + alias: Final = metadata.get("team_alias") if metadata is not None else None + return alias if isinstance(alias, str) else None diff --git a/litellm/types/proxy/management_endpoints/team_endpoints.py b/litellm/types/proxy/management_endpoints/team_endpoints.py index 2417868fb29..a282430bb11 100644 --- a/litellm/types/proxy/management_endpoints/team_endpoints.py +++ b/litellm/types/proxy/management_endpoints/team_endpoints.py @@ -143,3 +143,24 @@ class TeamMetadataSchemaResponse(BaseModel): """Response for GET /team/metadata_schema; ``fields`` is empty when no schema is configured.""" fields: tuple[TeamMetadataFieldSchema, ...] + + +class TeamUserSpendRow(BaseModel): + team_id: str + team_alias: str | None = None + user_id: str + user_email: str | None = None + user_alias: str | None = None + spend: float = 0.0 + prompt_tokens: int = 0 + completion_tokens: int = 0 + total_tokens: int = 0 + api_requests: int = 0 + successful_requests: int = 0 + failed_requests: int = 0 + + +class TeamUserSpendResponse(BaseModel): + start_date: str + end_date: str + results: tuple[TeamUserSpendRow, ...] diff --git a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt index 052962e078e..6bc8947e89f 100644 --- a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt +++ b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt @@ -28,6 +28,7 @@ GET /tag/user-agent/per-user-analytics GET /tag/wau GET /team/daily/activity GET /team/daily/activity/aggregated +GET /team/spend/by_user GET /team/spend/report GET /user/daily/activity GET /user/daily/activity/aggregated diff --git a/tests/proxy_behavior/management/test_team_spend_by_user.py b/tests/proxy_behavior/management/test_team_spend_by_user.py new file mode 100644 index 00000000000..1d6aab04003 --- /dev/null +++ b/tests/proxy_behavior/management/test_team_spend_by_user.py @@ -0,0 +1,58 @@ +import pytest + +from .actors import Actor + +pytestmark = pytest.mark.asyncio(loop_scope="session") + + +# GET /team/spend/by_user shares the team-scope resolver with +# /team/daily/activity, so the membership matrix must hold here too. team_ids +# is mandatory on this route (a per-user rollup with no team is meaningless), +# so the bare query is 400 for everyone instead of defaulting to own teams. +_MEMBERS = { + "alpha": { + Actor.TEAM_ADMIN, + Actor.INTERNAL_USER, + Actor.OWNER, + Actor.UNRELATED_SAME_ORG, + Actor.SERVICE_ACCOUNT, + }, + "beta": {Actor.CROSS_ORG_USER}, +} + + +def _expected(actor: Actor, team: str) -> int: + if team == "none": + return 400 + if actor == Actor.PROXY_ADMIN: + return 200 + return 200 if actor in _MEMBERS.get(team, set()) else 404 + + +_CASES = [ + (f"{team}/{actor.value}", actor, team, _expected(actor, team)) + for team in ("none", "alpha", "beta") + for actor in Actor +] + +_DATES = "start_date=2024-01-01&end_date=2024-12-31" + + +@pytest.mark.parametrize( + "actor,team,expected_status", + [(a, t, s) for (_id, a, t, s) in _CASES], + ids=[c[0] for c in _CASES], +) +async def test_team_spend_by_user_matrix(actor: Actor, team: str, expected_status: int, proxy_client, world): + team_id = {"alpha": world.team_alpha_id, "beta": world.team_beta_id}.get(team) + query = _DATES if team_id is None else f"{_DATES}&team_ids={team_id}" + + resp = await proxy_client.get( + f"/team/spend/by_user?{query}", + headers={"Authorization": f"Bearer {world.keys[actor].cleartext}"}, + ) + assert resp.status_code == expected_status, f"{actor.value} -> {team}: {resp.status_code} {resp.text}" + if expected_status == 200: + body = resp.json() + assert (body["start_date"], body["end_date"]) == ("2024-01-01", "2024-12-31") + assert all(row["team_id"] == team_id for row in body["results"]) diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 71ccef620e5..48926cb7bc2 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -3312,6 +3312,41 @@ def test_user_daily_activity_routes_reachable_by_non_admin(route, user_role): ) +@pytest.mark.parametrize( + "user_role", + [ + LitellmUserRoles.INTERNAL_USER.value, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value, + ], +) +def test_team_spend_by_user_reachable_by_non_admin(user_role): + user_obj = LiteLLM_UserTable( + user_id="test_user", + user_email="test@example.com", + user_role=user_role, + ) + valid_token = UserAPIKeyAuth(user_id="test_user", user_role=user_role) + request = MagicMock(spec=Request) + request.query_params = {} + + def outcome(route: str) -> str: + try: + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=user_role, + route=route, + request=request, + valid_token=valid_token, + request_data={}, + ) + except Exception as exc: + return f"denied: {exc}" + return "allowed" + + assert outcome("/team/spend/by_user") == "allowed" + assert outcome("/team/spend/by_key").startswith("denied: Only proxy admin") + + def test_user_daily_activity_aggregated_not_covered_by_prefix_match(): """check_route_access is exact-match plus explicit wildcards, so listing the parent /user/daily/activity does not implicitly cover the /aggregated diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 019ebc9807c..ab4cd74e092 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -13598,3 +13598,175 @@ async def test_team_member_update_skips_invalidation_when_no_budget_fields_sent( assert await real_cache.async_get_cache(key="team-1_member-1") == "still-fresh-membership" assert real_spend_counter_cache.in_memory_cache.get_cache(key="spend:team_member:member-1:team-1") == 1.5 + + +def _team_spend_by_user_team(team_id: str, team_alias: str, member: Member, permissions: list[str]) -> MagicMock: + team = MagicMock(spec=LiteLLM_TeamTable) + team.team_id = team_id + team.team_alias = team_alias + team.members_with_roles = [member] + team.team_member_permissions = permissions + team.model_dump.return_value = { + "team_id": team_id, + "team_alias": team_alias, + "members_with_roles": [{"user_id": member.user_id, "role": member.role}], + "team_member_permissions": permissions, + } + return team + + +def _team_spend_by_user_caller(user_id: str, teams: list[str]) -> LiteLLM_UserTable: + return LiteLLM_UserTable( + user_id=user_id, user_email=f"{user_id}@example.com", teams=teams, user_role="internal_user" + ) + + +def _team_spend_by_user_db_row(team_id: str, user_id: str, spend: float, requests: int) -> dict: + return { + "team_id": team_id, + "user_id": user_id, + "user_email": f"{user_id}@example.com", + "user_alias": None, + "spend": spend, + "prompt_tokens": 10 * requests, + "completion_tokens": 5 * requests, + "total_tokens": 15 * requests, + "api_requests": requests, + "successful_requests": requests - 1, + "failed_requests": 1, + } + + +@pytest.mark.asyncio +async def test_get_team_spend_by_user_admin_groups_spend_logs_by_team_and_user(mock_db_client): + from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user + + admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="alice", role="admin"), []) + beta = _team_spend_by_user_team("team-beta", "Team Beta", Member(user_id="alice", role="user"), []) + mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha, beta]) + mock_db_client.db.query_raw = AsyncMock( + return_value=[ + _team_spend_by_user_db_row("team-alpha", "alice", 0.5, 3), + _team_spend_by_user_db_row("team-alpha", "bob", 0.25, 2), + _team_spend_by_user_db_row("team-beta", "alice", 0.1, 1), + ] + ) + + response = await get_team_spend_by_user( + user_api_key_dict=admin, + team_ids="team-alpha,team-beta", + start_date="2026-09-01", + end_date="2026-09-04", + ) + + sql, *params = mock_db_client.db.query_raw.call_args.args + assert params == ["2026-09-01", "2026-09-04", "team-alpha", "team-beta"] + assert 'FROM "LiteLLM_SpendLogs" sl' in sql + assert 'sl."startTime" >= $1::timestamp' in sql + assert "sl.\"startTime\" < $2::timestamp + INTERVAL '1 day'" in sql + assert "sl.team_id IN ($3, $4)" in sql + assert 'GROUP BY sl.team_id, sl."user"' in sql + assert 'sl."user" = $' not in sql + + assert response.start_date == "2026-09-01" + assert response.end_date == "2026-09-04" + assert [(r.team_id, r.team_alias, r.user_id, r.user_email, r.spend, r.api_requests) for r in response.results] == [ + ("team-alpha", "Team Alpha", "alice", "alice@example.com", 0.5, 3), + ("team-alpha", "Team Alpha", "bob", "bob@example.com", 0.25, 2), + ("team-beta", "Team Beta", "alice", "alice@example.com", 0.1, 1), + ] + assert (response.results[0].successful_requests, response.results[0].failed_requests) == (2, 1) + assert (response.results[0].prompt_tokens, response.results[0].completion_tokens) == (30, 15) + + +@pytest.mark.asyncio +async def test_get_team_spend_by_user_team_admin_sees_every_member(mock_db_client): + from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user + + caller = UserAPIKeyAuth(user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER) + alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="alice", role="admin"), []) + mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha]) + mock_db_client.db.query_raw = AsyncMock(return_value=[]) + mock_db_client.db.litellm_usertable.find_unique = AsyncMock( + return_value=_team_spend_by_user_caller("alice", ["team-alpha"]) + ) + + await get_team_spend_by_user( + user_api_key_dict=caller, team_ids="team-alpha", start_date="2026-09-01", end_date="2026-09-04" + ) + + sql, *params = mock_db_client.db.query_raw.call_args.args + assert params == ["2026-09-01", "2026-09-04", "team-alpha"] + assert 'sl."user" = $' not in sql + + +@pytest.mark.asyncio +async def test_get_team_spend_by_user_plain_member_only_sees_own_row(mock_db_client): + from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user + + caller = UserAPIKeyAuth(user_id="bob", user_role=LitellmUserRoles.INTERNAL_USER) + alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="bob", role="user"), ["/key/info"]) + mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha]) + mock_db_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_db_client.db.query_raw = AsyncMock(return_value=[_team_spend_by_user_db_row("team-alpha", "bob", 0.25, 2)]) + mock_db_client.db.litellm_usertable.find_unique = AsyncMock( + return_value=_team_spend_by_user_caller("bob", ["team-alpha"]) + ) + + response = await get_team_spend_by_user( + user_api_key_dict=caller, team_ids="team-alpha", start_date="2026-09-01", end_date="2026-09-04" + ) + + sql, *params = mock_db_client.db.query_raw.call_args.args + assert params == ["2026-09-01", "2026-09-04", "team-alpha", "bob"] + assert "sl.team_id IN ($3)" in sql + assert 'AND sl."user" = $4' in sql + assert [(r.user_id, r.spend) for r in response.results] == [("bob", 0.25)] + + +@pytest.mark.asyncio +async def test_get_team_spend_by_user_member_of_other_team_gets_404(mock_db_client): + from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user + + caller = UserAPIKeyAuth(user_id="bob", user_role=LitellmUserRoles.INTERNAL_USER) + mock_db_client.db.query_raw = AsyncMock(return_value=[]) + mock_db_client.db.litellm_usertable.find_unique = AsyncMock( + return_value=_team_spend_by_user_caller("bob", ["team-alpha"]) + ) + + with pytest.raises(HTTPException) as exc_info: + await get_team_spend_by_user( + user_api_key_dict=caller, team_ids="team-beta", start_date="2026-09-01", end_date="2026-09-04" + ) + + assert exc_info.value.status_code == 404 + mock_db_client.db.query_raw.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "team_ids,start_date,end_date,expected_error", + [ + (None, "2026-09-01", "2026-09-04", "team_ids"), + ("", "2026-09-01", "2026-09-04", "team_ids"), + ("team-alpha", None, "2026-09-04", "start_date and end_date"), + ("team-alpha", "2026-09-04", "2026-09-01", "on or after"), + ("team-alpha", "2020-01-01", "2026-12-31", "at most 400 days"), + ("team-alpha", "nope", "2026-09-04", "valid YYYY-MM-DD"), + ], +) +async def test_get_team_spend_by_user_rejects_bad_input(mock_db_client, team_ids, start_date, end_date, expected_error): + from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user + + mock_db_client.db.query_raw = AsyncMock(return_value=[]) + admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + + with pytest.raises(HTTPException) as exc_info: + await get_team_spend_by_user( + user_api_key_dict=admin, team_ids=team_ids, start_date=start_date, end_date=end_date + ) + + assert exc_info.value.status_code == 400 + assert expected_error in str(exc_info.value.detail) + mock_db_client.db.query_raw.assert_not_called() diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx index 5bb48a78437..2a6c2ede478 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx @@ -3,6 +3,7 @@ import userEvent from "@testing-library/user-event"; import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { ReactNode } from "react"; import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers"; +import useTeams from "@/app/(dashboard)/hooks/useTeams"; import * as networking from "@/components/networking"; import EntityUsage from "./EntityUsage"; @@ -60,6 +61,10 @@ vi.mock("./TopModelView", () => ({ ), })); +vi.mock("./TeamUserSpendCard", () => ({ + default: ({ teamIds }: { teamIds: string[] }) =>
{`team-user-spend:${teamIds.join("|")}`}
, +})); + vi.mock("@/components/EntityUsageExport/EntityUsageExportModal", () => ({ default: () =>
Entity Usage Export Modal
, })); @@ -460,6 +465,26 @@ describe("EntityUsage", () => { }); }); + it("feeds the per-user spend card every visible team except the dashboard team, only for teams", async () => { + const mockUseTeams = vi.mocked(useTeams); + const teamsResult = (teams: { team_id: string }[]) => + ({ teams, setTeams: vi.fn() }) as unknown as ReturnType; + mockUseTeams.mockReturnValue( + teamsResult([{ team_id: "team-alpha" }, { team_id: "litellm-dashboard" }, { team_id: "team-beta" }]), + ); + + render(); + expect(await screen.findByText("team-user-spend:team-alpha|team-beta")).toBeInTheDocument(); + + cleanup(); + mockUseTeams.mockReturnValue(teamsResult([])); + render(); + await waitFor(() => { + expect(mockTagDailyActivityCall).toHaveBeenCalled(); + }); + expect(screen.queryByText(/^team-user-spend:/)).not.toBeInTheDocument(); + }); + it("should render with organization entity type and call organization API", async () => { render(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx index ef3943e5b71..273e478528e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx @@ -43,6 +43,7 @@ import EndpointUsage from "../EndpointUsage/EndpointUsage"; import ModelViewToggle, { ModelViewType } from "../ModelViewToggle"; import TopKeyView from "@/components/UsagePage/components/EntityUsage/TopKeyView"; import TopModelView from "./TopModelView"; +import TeamUserSpendCard from "./TeamUserSpendCard"; interface EntityMetrics { metrics: { @@ -275,6 +276,13 @@ const EntityUsage: React.FC = ({ const capitalizedEntityLabel = entityType.charAt(0).toUpperCase() + entityType.slice(1); const showFlatCost = entityType === "team" && hasFlatCost(spendData.metadata); + const userSpendTeamIds = useMemo( + () => + selectedTags.length > 0 + ? selectedTags + : (teams ?? []).map((team) => team.team_id).filter((id) => id !== "litellm-dashboard"), + [selectedTags, teams], + ); const providerSpend = useMemo(() => getProviderSpend(spendData.results), [spendData.results]); const entityBreakdownColumns = useMemo[]>( () => [ @@ -530,6 +538,17 @@ const EntityUsage: React.FC = ({
+ {entityType === "team" && ( +
+ +
+ )} + {/* Top API Keys */}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/TeamUserSpendCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/TeamUserSpendCard.tsx new file mode 100644 index 00000000000..ed90e144efb --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/TeamUserSpendCard.tsx @@ -0,0 +1,109 @@ +import { useQuery } from "@tanstack/react-query"; +import type { ColumnDef } from "@tanstack/react-table"; +import { Download } from "lucide-react"; +import React, { useMemo } from "react"; + +import { teamSpendByUserCall } from "@/components/networking"; +import { DataTable } from "@/components/shared/DataTable"; +import { MoneyCell } from "@/components/shared/table_cells"; +import { Button } from "@/components/ui/button"; +import { Card as ShadcnCard, CardContent } from "@/components/ui/card"; + +import { + buildTeamUserSpendCsv, + downloadCsv, + sortBySpendDesc, + teamLabel, + teamUserSpendCsvFileName, + teamUserSpendRowId, + userLabel, + type TeamUserSpendRow, +} from "./teamUserSpend"; + +interface TeamUserSpendCardProps { + accessToken: string | null; + startTime: Date | null; + endTime: Date | null; + teamIds: string[]; +} + +const columns: ColumnDef[] = [ + { header: "Team", accessorFn: teamLabel, id: "team", cell: ({ row }) => teamLabel(row.original) }, + { header: "User", accessorFn: userLabel, id: "user", cell: ({ row }) => userLabel(row.original) }, + { + header: "Spend", + accessorKey: "spend", + meta: { numeric: true }, + cell: ({ row }) => , + }, + { + header: "Requests", + accessorKey: "api_requests", + meta: { numeric: true }, + cell: ({ row }) => row.original.api_requests.toLocaleString(), + }, + { + header: "Successful", + accessorKey: "successful_requests", + meta: { numeric: true, className: "text-success" }, + cell: ({ row }) => row.original.successful_requests.toLocaleString(), + }, + { + header: "Failed", + accessorKey: "failed_requests", + meta: { numeric: true, className: "text-destructive" }, + cell: ({ row }) => row.original.failed_requests.toLocaleString(), + }, + { + header: "Tokens", + accessorKey: "total_tokens", + meta: { numeric: true }, + cell: ({ row }) => row.original.total_tokens.toLocaleString(), + }, +]; + +const TeamUserSpendCard: React.FC = ({ accessToken, startTime, endTime, teamIds }) => { + const hasTeams = teamIds.length > 0; + const { data, isLoading } = useQuery({ + queryKey: ["teamSpendByUser", startTime?.toISOString(), endTime?.toISOString(), teamIds], + queryFn: () => + accessToken && startTime && endTime ? teamSpendByUserCall(accessToken, startTime, endTime, teamIds) : null, + enabled: Boolean(accessToken && startTime && endTime) && hasTeams, + }); + const rows = useMemo(() => sortBySpendDesc(data?.results ?? []), [data]); + + return ( + + +
+
+

Spend Per User Within Team

+

+ Attributed per request from spend logs, so it includes JWT/SSO traffic that does not use a virtual key +

+
+ +
+ + + + ); +}; + +export default TeamUserSpendCard; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/teamUserSpend.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/teamUserSpend.test.ts new file mode 100644 index 00000000000..36d442c617f --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/teamUserSpend.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "vitest"; + +import type { TeamUserSpendResponse } from "@/components/networking"; + +import { + buildTeamUserSpendCsv, + sortBySpendDesc, + teamUserSpendCsvFileName, + teamUserSpendRowId, + userLabel, + type TeamUserSpendRow, +} from "./teamUserSpend"; + +const row = (overrides: Partial): TeamUserSpendRow => ({ + team_id: "team-alpha", + team_alias: "Team Alpha", + user_id: "alice@example.com", + user_email: "alice@example.com", + user_alias: null, + spend: 0.5, + prompt_tokens: 10, + completion_tokens: 5, + total_tokens: 15, + api_requests: 3, + successful_requests: 2, + failed_requests: 1, + ...overrides, +}); + +const aliceInBeta: Partial = { + team_id: "team-beta", + team_alias: "Team Beta", + spend: 0.1, + api_requests: 1, +}; +const bobInAlpha: Partial = { + user_id: "bob", + user_email: null, + user_alias: "Bob", + spend: 0.25, + api_requests: 2, +}; + +const response: TeamUserSpendResponse = { + start_date: "2026-09-01", + end_date: "2026-09-04", + results: [row(aliceInBeta), row({}), row(bobInAlpha)], +}; + +describe("teamUserSpend", () => { + it("keeps the same user as separate rows per team", () => { + const ids = response.results.map(teamUserSpendRowId); + expect(new Set(ids).size).toBe(3); + expect(ids[0]).not.toBe(ids[1]); + }); + + it("labels a user by email, then alias, then id, then a placeholder", () => { + expect(userLabel(row({}))).toBe("alice@example.com"); + expect(userLabel(row({ user_email: null, user_alias: "Bob", user_id: "u1" }))).toBe("Bob"); + expect(userLabel(row({ user_email: null, user_alias: null, user_id: "u1" }))).toBe("u1"); + expect(userLabel(row({ user_email: null, user_alias: null, user_id: "" }))).toBe("(no user)"); + }); + + it("sorts by spend descending without mutating the input", () => { + const before = [...response.results]; + expect(sortBySpendDesc(response.results).map((r) => r.spend)).toEqual([0.5, 0.25, 0.1]); + expect(response.results).toEqual(before); + }); + + it("writes one CSV line per (team, user) with the team kept on every line", () => { + const lines = buildTeamUserSpendCsv(response).split(/\r?\n/); + expect(lines[0]).toBe( + "Start Date,End Date,Team,Team ID,User,User ID,User Email,Spend (USD),Requests,Successful,Failed,Prompt Tokens,Completion Tokens,Total Tokens", + ); + expect(lines.slice(1)).toEqual([ + "2026-09-01,2026-09-04,Team Alpha,team-alpha,alice@example.com,alice@example.com,alice@example.com,0.5,3,2,1,10,5,15", + "2026-09-01,2026-09-04,Team Alpha,team-alpha,Bob,bob,,0.25,2,2,1,10,5,15", + "2026-09-01,2026-09-04,Team Beta,team-beta,alice@example.com,alice@example.com,alice@example.com,0.1,1,2,1,10,5,15", + ]); + }); + + it("neutralises spreadsheet formulas in user-controlled cells", () => { + const csv = buildTeamUserSpendCsv({ + ...response, + results: [row({ user_alias: null, user_email: "=HYPERLINK(1)" })], + }); + expect(csv).toContain("'=HYPERLINK(1)"); + }); + + it("names the file after the exported range", () => { + expect(teamUserSpendCsvFileName(response)).toBe("team_user_spend_2026-09-01_to_2026-09-04.csv"); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/teamUserSpend.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/teamUserSpend.ts new file mode 100644 index 00000000000..d0b47a4e5c0 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/teamUserSpend.ts @@ -0,0 +1,55 @@ +import Papa from "papaparse"; + +import type { TeamUserSpendResponse } from "@/components/networking"; + +export type TeamUserSpendRow = TeamUserSpendResponse["results"][number]; + +export const NO_USER_LABEL = "(no user)"; + +export const userLabel = (row: TeamUserSpendRow): string => { + const identity = row.user_email || row.user_alias; + return identity || row.user_id || NO_USER_LABEL; +}; + +export const teamLabel = (row: TeamUserSpendRow): string => row.team_alias || row.team_id; + +export const teamUserSpendRowId = (row: TeamUserSpendRow): string => `${row.team_id}\u0000${row.user_id}`; + +export const sortBySpendDesc = (rows: readonly TeamUserSpendRow[]): TeamUserSpendRow[] => + [...rows].sort((a, b) => b.spend - a.spend || teamLabel(a).localeCompare(teamLabel(b))); + +export const buildTeamUserSpendCsv = (response: TeamUserSpendResponse): string => + Papa.unparse( + sortBySpendDesc(response.results).map((row) => ({ + "Start Date": response.start_date, + "End Date": response.end_date, + Team: teamLabel(row), + "Team ID": row.team_id, + User: userLabel(row), + "User ID": row.user_id, + "User Email": row.user_email ?? "", + "Spend (USD)": row.spend, + Requests: row.api_requests, + Successful: row.successful_requests, + Failed: row.failed_requests, + "Prompt Tokens": row.prompt_tokens, + "Completion Tokens": row.completion_tokens, + "Total Tokens": row.total_tokens, + })), + { escapeFormulae: true }, + ); + +export const teamUserSpendCsvFileName = (response: TeamUserSpendResponse): string => + `team_user_spend_${response.start_date}_to_${response.end_date}.csv`; + +export const downloadCsv = (csv: string, fileName: string): void => { + const blob = new Blob([csv], { type: "text/csv;charset=utf-8;" }); + const url = window.URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = fileName; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + window.URL.revokeObjectURL(url); +}; diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 1384679a88a..697216c5254 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -81,6 +81,7 @@ import { EmailEventSettingsResponse, EmailEventSettingsUpdateRequest } from "./e import type { SkillRegisterRequest } from "./claude_code_plugins/types"; import type { ModelBudgetUsage, ModelMaxBudget } from "./key_team_helpers/ModelMaxBudgetEditor"; import type { ObjectPermission } from "./object_permission_types"; +import type { components } from "@/lib/http/schema"; import { jsonFields } from "./common_components/check_openapi_schema"; import type { MCPUserEnvVarsStatus } from "./mcp_tools/types"; import type { @@ -1535,6 +1536,23 @@ export const teamDailyActivityAggregatedCall = async ( } }; +export type TeamUserSpendResponse = components["schemas"]["TeamUserSpendResponse"]; + +export const teamSpendByUserCall = async ( + accessToken: string, + startTime: Date, + endTime: Date, + teamIds: string[], +): Promise => + apiClient.get(`/team/spend/by_user`, { + accessToken, + query: { + start_date: formatDate(startTime), + end_date: formatDate(endTime), + team_ids: teamIds.join(","), + }, + }); + export const organizationDailyActivityCall = async ( accessToken: string, startTime: Date, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 8c72872bdd2..f4cb88bbae1 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -15531,6 +15531,30 @@ export interface paths { patch?: never; trace?: never; }; + "/team/spend/by_user": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Team Spend By User + * @description Spend per user within the given teams, attributed per request from spend logs. + * + * Proxy admins may query any team. Team admins and members holding the + * `/team/daily/activity` permission see every user of the requested teams; + * other members only see their own row. + */ + get: operations["get_team_spend_by_user_team_spend_by_user_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/team/spend/report": { parameters: { query?: never; @@ -36769,6 +36793,63 @@ export interface components { /** Team Id */ team_id: string; }; + /** TeamUserSpendResponse */ + TeamUserSpendResponse: { + /** End Date */ + end_date: string; + /** Results */ + results: components["schemas"]["TeamUserSpendRow"][]; + /** Start Date */ + start_date: string; + }; + /** TeamUserSpendRow */ + TeamUserSpendRow: { + /** + * Api Requests + * @default 0 + */ + api_requests: number; + /** + * Completion Tokens + * @default 0 + */ + completion_tokens: number; + /** + * Failed Requests + * @default 0 + */ + failed_requests: number; + /** + * Prompt Tokens + * @default 0 + */ + prompt_tokens: number; + /** + * Spend + * @default 0 + */ + spend: number; + /** + * Successful Requests + * @default 0 + */ + successful_requests: number; + /** Team Alias */ + team_alias?: string | null; + /** Team Id */ + team_id: string; + /** + * Total Tokens + * @default 0 + */ + total_tokens: number; + /** User Alias */ + user_alias?: string | null; + /** User Email */ + user_email?: string | null; + /** User Id */ + user_id: string; + }; /** * TestCustomCodeGuardrailRequest * @description Request model for testing custom code guardrails. @@ -58550,6 +58631,39 @@ export interface operations { }; }; }; + get_team_spend_by_user_team_spend_by_user_get: { + parameters: { + query?: { + team_ids?: string | null; + start_date?: string | null; + end_date?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["TeamUserSpendResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_team_spend_report_team_spend_report_get: { parameters: { query?: { From 7b96a11e5f5605c3bd21b5aa7e9b9161530b0b96 Mon Sep 17 00:00:00 2001 From: mateo Date: Fri, 4 Sep 2026 19:19:08 +0000 Subject: [PATCH 66/88] feat(registry): add OpenRouter catalog gaps, Fireworks DeepSeek V4 Flash Vision, Together MiniMax M2.7 and Qwen2.5 7B Turbo pricing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 2184 ++++++++++++++++- model_prices_and_context_window.json | 2184 ++++++++++++++++- model_prices_and_context_window.schema.json | 20 + 3 files changed, 4386 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1a92d78b053..aa4f9b87377 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -42472,7 +42472,12 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 3e-07, + "max_input_tokens": 32768, + "max_tokens": 32768, + "source": "https://www.together.ai/pricing" }, "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { "deprecation_date": "2026-07-10", @@ -42894,6 +42899,16 @@ "supports_tool_choice": true, "supports_vision": true }, + "together_ai/MiniMaxAI/MiniMax-M2.7": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 196608, + "max_tokens": 196608, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://www.together.ai/pricing" + }, "together_ai/Prism-ML/Ternary-Bonsai-27B": { "input_cost_per_token": 0.0, "litellm_provider": "together_ai", @@ -56337,6 +56352,19 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-vision-exp": { + "cache_read_input_token_cost": 7e-09, + "input_cost_per_token": 2.2e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 6.6e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/models/kimi-k3": { "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, @@ -56374,6 +56402,19 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/deepseek-v4-flash-vision-exp": { + "cache_read_input_token_cost": 7e-09, + "input_cost_per_token": 2.2e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 6.6e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/glm-5p2-fast": { "cache_read_input_token_cost": 2.1e-07, "input_cost_per_token": 2.1e-06, @@ -60828,5 +60869,2146 @@ "cache_read_input_token_cost": 6.4e-08, "supports_prompt_caching": true, "cache_creation_input_token_cost": 4e-07 + }, + "openrouter/openai/gpt-6-astra": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "cache_read_input_token_cost": 1e-06, + "cache_creation_input_token_cost": 1.25e-05, + "input_cost_per_token_above_272k_tokens": 2e-05, + "output_cost_per_token_above_272k_tokens": 7.5e-05, + "cache_read_input_token_cost_above_272k_tokens": 2e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-6-astra", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3.8-flash": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 4.7e-07, + "cache_read_input_token_cost": 1.6e-08, + "cache_creation_input_token_cost": 2e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.8-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/z-ai/glm-5.3-flash": { + "input_cost_per_token": 7.5e-08, + "output_cost_per_token": 2.5e-07, + "cache_read_input_token_cost": 1.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 1310720, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-5.3-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/deepseek/deepseek-v4-flash-vision-exp": { + "input_cost_per_token": 2.2e-07, + "output_cost_per_token": 6.6e-07, + "cache_read_input_token_cost": 7e-09, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "source": "https://openrouter.ai/deepseek/deepseek-v4-flash-vision-exp", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/z-ai/glm-5.3": { + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "cache_read_input_token_cost": 1.4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1310720, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-5.3", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3.8-27b": { + "input_cost_per_token": 4.2e-07, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 8.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.8-27b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3.8-2.4t-a95b": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 2.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.8-2.4t-a95b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/nvidia/nemotron-3.5-lightning:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3.5-lightning:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3.8-max": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 2.5e-07, + "cache_creation_input_token_cost": 2.5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.8-max", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/deepseek/deepseek-v4-flash-0731": { + "input_cost_per_token": 6.5e-08, + "output_cost_per_token": 1.8e-07, + "cache_read_input_token_cost": 1.6e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 1310720, + "max_output_tokens": 943718, + "max_tokens": 943718, + "mode": "chat", + "source": "https://openrouter.ai/deepseek/deepseek-v4-flash-0731", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3.7-flash": { + "input_cost_per_token": 3e-08, + "output_cost_per_token": 1.3e-07, + "cache_read_input_token_cost": 6e-09, + "cache_creation_input_token_cost": 3.8e-08, + "input_cost_per_token_above_256k_tokens": 2e-07, + "output_cost_per_token_above_256k_tokens": 8e-07, + "cache_read_input_token_cost_above_256k_tokens": 4e-08, + "cache_creation_input_token_cost_above_256k_tokens": 2.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.7-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/poolside/laguna-s-2.1": { + "input_cost_per_token": 9e-08, + "output_cost_per_token": 1.8e-07, + "cache_read_input_token_cost": 9e-09, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/poolside/laguna-s-2.1", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/poolside/laguna-s-2.1:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/poolside/laguna-s-2.1:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false + }, + "openrouter/moonshotai/kimi-k3": { + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "max_tokens": 943718, + "mode": "chat", + "source": "https://openrouter.ai/moonshotai/kimi-k3", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/poolside/laguna-xs-2.1": { + "input_cost_per_token": 6e-08, + "output_cost_per_token": 1.2e-07, + "cache_read_input_token_cost": 3e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/poolside/laguna-xs-2.1", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/poolside/laguna-xs-2.1:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/poolside/laguna-xs-2.1:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false + }, + "openrouter/google/gemini-3.1-flash-lite-image": { + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.5e-06, + "output_cost_per_image_token": 3e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 65536, + "max_output_tokens": 58982, + "max_tokens": 58982, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.1-flash-lite-image", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/google/gemini-3.1-flash-image": { + "input_cost_per_token": 5e-07, + "output_cost_per_token": 3e-06, + "output_cost_per_image_token": 6e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.1-flash-image", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/google/gemini-3-pro-image": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05, + "cache_read_input_token_cost": 2e-07, + "cache_creation_input_token_cost": 3.75e-07, + "input_cost_per_audio_token": 2e-06, + "output_cost_per_image_token": 0.00012, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3-pro-image", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/z-ai/glm-5.2": { + "input_cost_per_token": 9.66e-07, + "output_cost_per_token": 3.036e-06, + "cache_read_input_token_cost": 1.932e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-5.2", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/z-ai/glm-5.2:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 256000, + "max_output_tokens": 230400, + "max_tokens": 230400, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-5.2:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/moonshotai/kimi-k2.7-code": { + "input_cost_per_token": 6.6e-07, + "output_cost_per_token": 3.4e-06, + "cache_read_input_token_cost": 1.8e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/moonshotai/kimi-k2.7-code", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/nvidia/nemotron-3.5-content-safety": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 117964, + "max_tokens": 117964, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3.5-content-safety", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/nvidia/nemotron-3.5-content-safety:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3.5-content-safety:free", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_vision": true + }, + "openrouter/nvidia/nemotron-3-ultra-550b-a55b": { + "input_cost_per_token": 6.25e-07, + "output_cost_per_token": 3.125e-06, + "cache_read_input_token_cost": 1.875e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3-ultra-550b-a55b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/nvidia/nemotron-3-ultra-550b-a55b:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3-ultra-550b-a55b:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false + }, + "openrouter/minimax/minimax-m3:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "max_tokens": 943718, + "mode": "chat", + "source": "https://openrouter.ai/minimax/minimax-m3:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3.7-max": { + "input_cost_per_token": 1.475e-06, + "output_cost_per_token": 4.425e-06, + "cache_read_input_token_cost": 2.95e-07, + "cache_creation_input_token_cost": 1.84375e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.7-max", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/mistralai/mistral-medium-3-5": { + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 7.5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 209715, + "max_tokens": 209715, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/mistral-medium-3-5", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true + }, + "openrouter/nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 256000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_audio_input": true + }, + "openrouter/qwen/qwen3.5-plus-20260420": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.8e-06, + "cache_creation_input_token_cost": 3.75e-07, + "input_cost_per_token_above_256k_tokens": 3.75e-07, + "output_cost_per_token_above_256k_tokens": 2.25e-06, + "cache_creation_input_token_cost_above_256k_tokens": 4.6875e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.5-plus-20260420", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3.6-flash": { + "input_cost_per_token": 1.875e-07, + "output_cost_per_token": 1.125e-06, + "cache_creation_input_token_cost": 2.34375e-07, + "input_cost_per_token_above_256k_tokens": 7.5e-07, + "output_cost_per_token_above_256k_tokens": 3e-06, + "cache_creation_input_token_cost_above_256k_tokens": 9.375e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.6-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3.6-35b-a3b": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 9e-07, + "cache_read_input_token_cost": 5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.6-35b-a3b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3.6-max-preview": { + "input_cost_per_token": 1.027e-06, + "output_cost_per_token": 6.162e-06, + "cache_creation_input_token_cost": 1.28375e-06, + "input_cost_per_token_above_128k_tokens": 1.58e-06, + "output_cost_per_token_above_128k_tokens": 9.48e-06, + "cache_creation_input_token_cost_above_128k_tokens": 1.975e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.6-max-preview", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3.6-27b": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 3e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.6-27b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.5-pro": { + "input_cost_per_token": 3e-05, + "output_cost_per_token": 0.00018, + "input_cost_per_token_above_272k_tokens": 6e-05, + "output_cost_per_token_above_272k_tokens": 0.00027, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.5-pro", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true + }, + "openrouter/deepseek/deepseek-v4-flash": { + "input_cost_per_token": 8.778e-08, + "output_cost_per_token": 1.7556e-07, + "cache_read_input_token_cost": 1.7556e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "source": "https://openrouter.ai/deepseek/deepseek-v4-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/moonshotai/kimi-k2.6": { + "input_cost_per_token": 9.5e-07, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 1.6e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/moonshotai/kimi-k2.6", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/google/gemma-4-26b-a4b-it": { + "input_cost_per_token": 7e-08, + "output_cost_per_token": 3.4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/google/gemma-4-26b-a4b-it", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/google/gemma-4-26b-a4b-it:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/google/gemma-4-26b-a4b-it:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/google/gemma-4-31b-it": { + "input_cost_per_token": 9e-08, + "output_cost_per_token": 3.4e-07, + "cache_read_input_token_cost": 5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/google/gemma-4-31b-it", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/google/gemma-4-31b-it:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/google/gemma-4-31b-it:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/z-ai/glm-5v-turbo": { + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2.4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 202752, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-5v-turbo", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/minimax/minimax-m2.7": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 6e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/minimax/minimax-m2.7", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/minimax/minimax-m2.7:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 196608, + "max_output_tokens": 176947, + "max_tokens": 176947, + "mode": "chat", + "source": "https://openrouter.ai/minimax/minimax-m2.7:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/mistralai/mistral-small-2603": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "cache_read_input_token_cost": 1.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 209715, + "max_tokens": 209715, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/mistral-small-2603", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/z-ai/glm-5-turbo": { + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2.4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 202752, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-5-turbo", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/nvidia/nemotron-3-super-120b-a12b": { + "input_cost_per_token": 8.5e-08, + "output_cost_per_token": 4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3-super-120b-a12b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/nvidia/nemotron-3-super-120b-a12b:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3-super-120b-a12b:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3.5-9b": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.5-9b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/openai/gpt-5.4-pro": { + "input_cost_per_token": 3e-05, + "output_cost_per_token": 0.00018, + "input_cost_per_token_above_272k_tokens": 6e-05, + "output_cost_per_token_above_272k_tokens": 0.00027, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.4-pro", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true + }, + "openrouter/google/gemini-3.1-flash-image-preview": { + "input_cost_per_token": 5e-07, + "output_cost_per_token": 3e-06, + "output_cost_per_image_token": 6e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 65536, + "max_output_tokens": 58982, + "max_tokens": 58982, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.1-flash-image-preview", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/google/gemini-3.1-pro-preview-customtools": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05, + "cache_read_input_token_cost": 2e-07, + "cache_creation_input_token_cost": 3.75e-07, + "input_cost_per_audio_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "output_cost_per_token_above_200k_tokens": 1.8e-05, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.1-pro-preview-customtools", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3-max-thinking": { + "input_cost_per_token": 7.8e-07, + "output_cost_per_token": 3.9e-06, + "input_cost_per_token_above_128k_tokens": 1.95e-06, + "output_cost_per_token_above_128k_tokens": 9.75e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-max-thinking", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3-coder-next": { + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 8e-07, + "cache_read_input_token_cost": 7e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-coder-next", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/minimax/minimax-m2-her": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 3e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 65536, + "max_output_tokens": 2048, + "max_tokens": 2048, + "mode": "chat", + "source": "https://openrouter.ai/minimax/minimax-m2-her", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-audio": { + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1e-05, + "input_cost_per_audio_token": 3.2e-05, + "output_cost_per_audio_token": 6.4e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-audio", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_audio_input": true + }, + "openrouter/openai/gpt-audio-mini": { + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.4e-06, + "input_cost_per_audio_token": 6e-07, + "output_cost_per_audio_token": 2.4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-audio-mini", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_audio_input": true + }, + "openrouter/nvidia/nemotron-3-nano-30b-a3b": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2e-07, + "cache_read_input_token_cost": 3e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3-nano-30b-a3b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/z-ai/glm-4.6v": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 9e-07, + "cache_read_input_token_cost": 5.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-4.6v", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/google/gemini-3-pro-image-preview": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05, + "cache_read_input_token_cost": 2e-07, + "cache_creation_input_token_cost": 3.75e-07, + "input_cost_per_audio_token": 2e-06, + "output_cost_per_image_token": 0.00012, + "litellm_provider": "openrouter", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3-pro-image-preview", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.1-codex": { + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.3e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.1-codex", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.1-codex-mini": { + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 3e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.1-codex-mini", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/moonshotai/kimi-k2-thinking": { + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 1.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 100352, + "max_tokens": 100352, + "mode": "chat", + "source": "https://openrouter.ai/moonshotai/kimi-k2-thinking", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/mistralai/voxtral-small-24b-2507": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, + "cache_read_input_token_cost": 1e-08, + "input_cost_per_audio_token": 0.0001, + "litellm_provider": "openrouter", + "max_input_tokens": 32768, + "max_output_tokens": 26214, + "max_tokens": 26214, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/voxtral-small-24b-2507", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_pdf_input": true, + "supports_audio_input": true, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-oss-safeguard-20b": { + "input_cost_per_token": 7.5e-08, + "output_cost_per_token": 3e-07, + "cache_read_input_token_cost": 3.75e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-oss-safeguard-20b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3-vl-32b-instruct": { + "input_cost_per_token": 1.04e-07, + "output_cost_per_token": 4.16e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-vl-32b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3-vl-8b-thinking": { + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 2.1e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-vl-8b-thinking", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3-vl-8b-instruct": { + "input_cost_per_token": 1.17e-07, + "output_cost_per_token": 4.55e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-vl-8b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/google/gemini-2.5-flash-image": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 8.33333333333333e-08, + "input_cost_per_audio_token": 1e-06, + "output_cost_per_image_token": 3e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-2.5-flash-image", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3-vl-30b-a3b-thinking": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2.4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-vl-30b-a3b-thinking", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3-vl-30b-a3b-instruct": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-vl-30b-a3b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/openai/gpt-5-pro": { + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 0.00012, + "litellm_provider": "openrouter", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5-pro", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true + }, + "openrouter/qwen/qwen3-vl-235b-a22b-thinking": { + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-vl-235b-a22b-thinking", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3-vl-235b-a22b-instruct": { + "input_cost_per_token": 2.1e-07, + "output_cost_per_token": 1.9e-06, + "cache_read_input_token_cost": 1e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-vl-235b-a22b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3-max": { + "input_cost_per_token": 7.8e-07, + "output_cost_per_token": 3.9e-06, + "cache_read_input_token_cost": 1.56e-07, + "cache_creation_input_token_cost": 9.75e-07, + "input_cost_per_token_above_128k_tokens": 1.95e-06, + "output_cost_per_token_above_128k_tokens": 9.75e-06, + "cache_read_input_token_cost_above_128k_tokens": 3.9e-07, + "cache_creation_input_token_cost_above_128k_tokens": 2.4375e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-max", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/deepseek/deepseek-v3.1-terminus": { + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 1.35e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 163840, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/deepseek/deepseek-v3.1-terminus", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3-coder-flash": { + "input_cost_per_token": 1.95e-07, + "output_cost_per_token": 9.75e-07, + "cache_read_input_token_cost": 3.9e-08, + "cache_creation_input_token_cost": 2.4375e-07, + "input_cost_per_token_above_128k_tokens": 5.2e-07, + "output_cost_per_token_above_128k_tokens": 2.6e-06, + "cache_read_input_token_cost_above_128k_tokens": 1.04e-07, + "cache_creation_input_token_cost_above_128k_tokens": 6.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-coder-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3-next-80b-a3b-thinking": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-next-80b-a3b-thinking", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3-next-80b-a3b-instruct": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1.1e-06, + "cache_read_input_token_cost": 7e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-next-80b-a3b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen-plus-2025-07-28": { + "input_cost_per_token": 2.6e-07, + "output_cost_per_token": 7.8e-07, + "input_cost_per_token_above_256k_tokens": 7.8e-07, + "output_cost_per_token_above_256k_tokens": 2.34e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen-plus-2025-07-28", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/moonshotai/kimi-k2-0905": { + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 100352, + "max_tokens": 100352, + "mode": "chat", + "source": "https://openrouter.ai/moonshotai/kimi-k2-0905", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3-30b-a3b-thinking-2507": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2.4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 81920, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-30b-a3b-thinking-2507", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/mistralai/mistral-medium-3.1": { + "input_cost_per_token": 4e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 4e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 104857, + "max_tokens": 104857, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/mistral-medium-3.1", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/z-ai/glm-4.5v": { + "input_cost_per_token": 6e-07, + "output_cost_per_token": 1.8e-06, + "cache_read_input_token_cost": 1.1e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 65536, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-4.5v", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/mistralai/codestral-2508": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 9e-07, + "cache_read_input_token_cost": 3e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 256000, + "max_output_tokens": 204800, + "max_tokens": 204800, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/codestral-2508", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3-coder-30b-a3b-instruct": { + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2.8e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-coder-30b-a3b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3-30b-a3b-instruct-2507": { + "input_cost_per_token": 4.815e-08, + "output_cost_per_token": 1.9305e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-30b-a3b-instruct-2507", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/z-ai/glm-4.5": { + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.2e-06, + "cache_read_input_token_cost": 1.1e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 98304, + "max_tokens": 98304, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-4.5", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/z-ai/glm-4.5-air": { + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 8.5e-07, + "cache_read_input_token_cost": 2.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 98304, + "max_tokens": 98304, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-4.5-air", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/moonshotai/kimi-k2": { + "input_cost_per_token": 5.7e-07, + "output_cost_per_token": 2.3e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 100352, + "max_tokens": 100352, + "mode": "chat", + "source": "https://openrouter.ai/moonshotai/kimi-k2", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "openrouter/minimax/minimax-m1": { + "input_cost_per_token": 5.5e-07, + "output_cost_per_token": 2.2e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 40000, + "max_tokens": 40000, + "mode": "chat", + "source": "https://openrouter.ai/minimax/minimax-m1", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false + }, + "openrouter/openai/o3-pro": { + "input_cost_per_token": 2e-05, + "output_cost_per_token": 8e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "source": "https://openrouter.ai/openai/o3-pro", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true + }, + "openrouter/google/gemini-2.5-pro-preview": { + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost": 3.75e-07, + "input_cost_per_audio_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 1.5e-05, + "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-2.5-pro-preview", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "supports_prompt_caching": true + }, + "openrouter/mistralai/mistral-medium-3": { + "input_cost_per_token": 4e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 4e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 104857, + "max_tokens": 104857, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/mistral-medium-3", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/google/gemini-2.5-pro-preview-05-06": { + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost": 3.75e-07, + "input_cost_per_audio_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 1.5e-05, + "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_tokens": 65535, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-2.5-pro-preview-05-06", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "supports_prompt_caching": true + }, + "openrouter/meta-llama/llama-guard-4-12b": { + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 163840, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/meta-llama/llama-guard-4-12b", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3-30b-a3b": { + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-30b-a3b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3-8b": { + "input_cost_per_token": 1.17e-07, + "output_cost_per_token": 4.55e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-8b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3-14b": { + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 2.4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-14b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3-32b": { + "input_cost_per_token": 8e-08, + "output_cost_per_token": 2.8e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-32b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3-235b-a22b": { + "input_cost_per_token": 4.55e-07, + "output_cost_per_token": 1.82e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-235b-a22b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/openai/o4-mini-high": { + "input_cost_per_token": 1.1e-06, + "output_cost_per_token": 4.4e-06, + "cache_read_input_token_cost": 2.75e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "source": "https://openrouter.ai/openai/o4-mini-high", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/meta-llama/llama-4-maverick": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6.96e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 115200, + "max_tokens": 115200, + "mode": "chat", + "source": "https://openrouter.ai/meta-llama/llama-4-maverick", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/meta-llama/llama-4-scout": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1310720, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/meta-llama/llama-4-scout", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/openai/o1-pro": { + "input_cost_per_token": 0.00015, + "output_cost_per_token": 0.0006, + "litellm_provider": "openrouter", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "source": "https://openrouter.ai/openai/o1-pro", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true + }, + "openrouter/google/gemma-3-4b-it": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/google/gemma-3-4b-it", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/google/gemma-3-12b-it": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/google/gemma-3-12b-it", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/google/gemma-3-27b-it": { + "input_cost_per_token": 8e-08, + "output_cost_per_token": 4.5e-07, + "cache_read_input_token_cost": 4e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 117964, + "max_tokens": 117964, + "mode": "chat", + "source": "https://openrouter.ai/google/gemma-3-27b-it", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/mistralai/mistral-saba": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-07, + "cache_read_input_token_cost": 2e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 32768, + "max_output_tokens": 26214, + "max_tokens": 26214, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/mistral-saba", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen2.5-vl-72b-instruct": { + "input_cost_per_token": 8e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 115200, + "max_tokens": 115200, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen2.5-vl-72b-instruct", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen-plus": { + "input_cost_per_token": 2.6e-07, + "output_cost_per_token": 7.8e-07, + "cache_read_input_token_cost": 5.2e-08, + "cache_creation_input_token_cost": 3.25e-07, + "input_cost_per_token_above_256k_tokens": 7.8e-07, + "output_cost_per_token_above_256k_tokens": 2.34e-06, + "cache_read_input_token_cost_above_256k_tokens": 1.56e-07, + "cache_creation_input_token_cost_above_256k_tokens": 9.75e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen-plus", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/mistralai/mistral-small-24b-instruct-2501": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 8e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/mistral-small-24b-instruct-2501", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/deepseek/deepseek-r1-distill-llama-70b": { + "input_cost_per_token": 8e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 8192, + "max_output_tokens": 7372, + "max_tokens": 7372, + "mode": "chat", + "source": "https://openrouter.ai/deepseek/deepseek-r1-distill-llama-70b", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_vision": false + }, + "openrouter/minimax/minimax-01": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.1e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000192, + "max_output_tokens": 900172, + "max_tokens": 900172, + "mode": "chat", + "source": "https://openrouter.ai/minimax/minimax-01", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_vision": true + }, + "openrouter/meta-llama/llama-3.3-70b-instruct": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3.2e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/meta-llama/llama-3.3-70b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/openai/gpt-4o-2024-11-20": { + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-4o-2024-11-20", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_web_search": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/mistralai/mistral-large-2407": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 2e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 104857, + "max_tokens": 104857, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/mistral-large-2407", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen-2.5-7b-instruct": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 32768, + "max_output_tokens": 29491, + "max_tokens": 29491, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen-2.5-7b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/meta-llama/llama-3.2-1b-instruct": { + "input_cost_per_token": 2.7e-08, + "output_cost_per_token": 2.01e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 60000, + "max_output_tokens": 54000, + "max_tokens": 54000, + "mode": "chat", + "source": "https://openrouter.ai/meta-llama/llama-3.2-1b-instruct", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_vision": false + }, + "openrouter/meta-llama/llama-3.2-3b-instruct": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 3.3e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 117964, + "max_tokens": 117964, + "mode": "chat", + "source": "https://openrouter.ai/meta-llama/llama-3.2-3b-instruct", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen-2.5-72b-instruct": { + "input_cost_per_token": 3.6e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen-2.5-72b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/openai/gpt-4o-2024-08-06": { + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-4o-2024-08-06", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_web_search": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/meta-llama/llama-3.1-70b-instruct": { + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/meta-llama/llama-3.1-70b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/meta-llama/llama-3.1-8b-instruct": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 8e-08, + "cache_read_input_token_cost": 2.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 117964, + "max_tokens": 117964, + "mode": "chat", + "source": "https://openrouter.ai/meta-llama/llama-3.1-8b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/mistralai/mistral-nemo": { + "input_cost_per_token": 1.9e-08, + "output_cost_per_token": 3e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/mistral-nemo", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/openai/gpt-4o-mini-2024-07-18": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "cache_read_input_token_cost": 7.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-4o-mini-2024-07-18", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_web_search": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/google/gemma-2-27b-it": { + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 6.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 8192, + "max_output_tokens": 2048, + "max_tokens": 2048, + "mode": "chat", + "source": "https://openrouter.ai/google/gemma-2-27b-it", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/openai/gpt-4-turbo": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-4-turbo", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/openai/gpt-4-turbo-preview": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-4-turbo-preview", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/openai/gpt-3.5-turbo-instruct": { + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 4095, + "max_output_tokens": 3685, + "max_tokens": 3685, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-3.5-turbo-instruct", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_response_schema": true, + "supports_vision": false } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1a92d78b053..aa4f9b87377 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -42472,7 +42472,12 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 3e-07, + "max_input_tokens": 32768, + "max_tokens": 32768, + "source": "https://www.together.ai/pricing" }, "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { "deprecation_date": "2026-07-10", @@ -42894,6 +42899,16 @@ "supports_tool_choice": true, "supports_vision": true }, + "together_ai/MiniMaxAI/MiniMax-M2.7": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 196608, + "max_tokens": 196608, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://www.together.ai/pricing" + }, "together_ai/Prism-ML/Ternary-Bonsai-27B": { "input_cost_per_token": 0.0, "litellm_provider": "together_ai", @@ -56337,6 +56352,19 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-vision-exp": { + "cache_read_input_token_cost": 7e-09, + "input_cost_per_token": 2.2e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 6.6e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/models/kimi-k3": { "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, @@ -56374,6 +56402,19 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/deepseek-v4-flash-vision-exp": { + "cache_read_input_token_cost": 7e-09, + "input_cost_per_token": 2.2e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 6.6e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/glm-5p2-fast": { "cache_read_input_token_cost": 2.1e-07, "input_cost_per_token": 2.1e-06, @@ -60828,5 +60869,2146 @@ "cache_read_input_token_cost": 6.4e-08, "supports_prompt_caching": true, "cache_creation_input_token_cost": 4e-07 + }, + "openrouter/openai/gpt-6-astra": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "cache_read_input_token_cost": 1e-06, + "cache_creation_input_token_cost": 1.25e-05, + "input_cost_per_token_above_272k_tokens": 2e-05, + "output_cost_per_token_above_272k_tokens": 7.5e-05, + "cache_read_input_token_cost_above_272k_tokens": 2e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-6-astra", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3.8-flash": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 4.7e-07, + "cache_read_input_token_cost": 1.6e-08, + "cache_creation_input_token_cost": 2e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.8-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/z-ai/glm-5.3-flash": { + "input_cost_per_token": 7.5e-08, + "output_cost_per_token": 2.5e-07, + "cache_read_input_token_cost": 1.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 1310720, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-5.3-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/deepseek/deepseek-v4-flash-vision-exp": { + "input_cost_per_token": 2.2e-07, + "output_cost_per_token": 6.6e-07, + "cache_read_input_token_cost": 7e-09, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "source": "https://openrouter.ai/deepseek/deepseek-v4-flash-vision-exp", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/z-ai/glm-5.3": { + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "cache_read_input_token_cost": 1.4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1310720, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-5.3", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3.8-27b": { + "input_cost_per_token": 4.2e-07, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 8.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.8-27b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3.8-2.4t-a95b": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 2.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.8-2.4t-a95b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/nvidia/nemotron-3.5-lightning:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3.5-lightning:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3.8-max": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 2.5e-07, + "cache_creation_input_token_cost": 2.5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.8-max", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/deepseek/deepseek-v4-flash-0731": { + "input_cost_per_token": 6.5e-08, + "output_cost_per_token": 1.8e-07, + "cache_read_input_token_cost": 1.6e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 1310720, + "max_output_tokens": 943718, + "max_tokens": 943718, + "mode": "chat", + "source": "https://openrouter.ai/deepseek/deepseek-v4-flash-0731", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3.7-flash": { + "input_cost_per_token": 3e-08, + "output_cost_per_token": 1.3e-07, + "cache_read_input_token_cost": 6e-09, + "cache_creation_input_token_cost": 3.8e-08, + "input_cost_per_token_above_256k_tokens": 2e-07, + "output_cost_per_token_above_256k_tokens": 8e-07, + "cache_read_input_token_cost_above_256k_tokens": 4e-08, + "cache_creation_input_token_cost_above_256k_tokens": 2.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.7-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/poolside/laguna-s-2.1": { + "input_cost_per_token": 9e-08, + "output_cost_per_token": 1.8e-07, + "cache_read_input_token_cost": 9e-09, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/poolside/laguna-s-2.1", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/poolside/laguna-s-2.1:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/poolside/laguna-s-2.1:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false + }, + "openrouter/moonshotai/kimi-k3": { + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "max_tokens": 943718, + "mode": "chat", + "source": "https://openrouter.ai/moonshotai/kimi-k3", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/poolside/laguna-xs-2.1": { + "input_cost_per_token": 6e-08, + "output_cost_per_token": 1.2e-07, + "cache_read_input_token_cost": 3e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/poolside/laguna-xs-2.1", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/poolside/laguna-xs-2.1:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/poolside/laguna-xs-2.1:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false + }, + "openrouter/google/gemini-3.1-flash-lite-image": { + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.5e-06, + "output_cost_per_image_token": 3e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 65536, + "max_output_tokens": 58982, + "max_tokens": 58982, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.1-flash-lite-image", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/google/gemini-3.1-flash-image": { + "input_cost_per_token": 5e-07, + "output_cost_per_token": 3e-06, + "output_cost_per_image_token": 6e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.1-flash-image", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/google/gemini-3-pro-image": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05, + "cache_read_input_token_cost": 2e-07, + "cache_creation_input_token_cost": 3.75e-07, + "input_cost_per_audio_token": 2e-06, + "output_cost_per_image_token": 0.00012, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3-pro-image", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/z-ai/glm-5.2": { + "input_cost_per_token": 9.66e-07, + "output_cost_per_token": 3.036e-06, + "cache_read_input_token_cost": 1.932e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-5.2", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_parallel_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/z-ai/glm-5.2:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 256000, + "max_output_tokens": 230400, + "max_tokens": 230400, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-5.2:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/moonshotai/kimi-k2.7-code": { + "input_cost_per_token": 6.6e-07, + "output_cost_per_token": 3.4e-06, + "cache_read_input_token_cost": 1.8e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/moonshotai/kimi-k2.7-code", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/nvidia/nemotron-3.5-content-safety": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 117964, + "max_tokens": 117964, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3.5-content-safety", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/nvidia/nemotron-3.5-content-safety:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3.5-content-safety:free", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_vision": true + }, + "openrouter/nvidia/nemotron-3-ultra-550b-a55b": { + "input_cost_per_token": 6.25e-07, + "output_cost_per_token": 3.125e-06, + "cache_read_input_token_cost": 1.875e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3-ultra-550b-a55b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/nvidia/nemotron-3-ultra-550b-a55b:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3-ultra-550b-a55b:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false + }, + "openrouter/minimax/minimax-m3:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "max_tokens": 943718, + "mode": "chat", + "source": "https://openrouter.ai/minimax/minimax-m3:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3.7-max": { + "input_cost_per_token": 1.475e-06, + "output_cost_per_token": 4.425e-06, + "cache_read_input_token_cost": 2.95e-07, + "cache_creation_input_token_cost": 1.84375e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.7-max", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/mistralai/mistral-medium-3-5": { + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 7.5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 209715, + "max_tokens": 209715, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/mistral-medium-3-5", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true + }, + "openrouter/nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 256000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_audio_input": true + }, + "openrouter/qwen/qwen3.5-plus-20260420": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.8e-06, + "cache_creation_input_token_cost": 3.75e-07, + "input_cost_per_token_above_256k_tokens": 3.75e-07, + "output_cost_per_token_above_256k_tokens": 2.25e-06, + "cache_creation_input_token_cost_above_256k_tokens": 4.6875e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.5-plus-20260420", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3.6-flash": { + "input_cost_per_token": 1.875e-07, + "output_cost_per_token": 1.125e-06, + "cache_creation_input_token_cost": 2.34375e-07, + "input_cost_per_token_above_256k_tokens": 7.5e-07, + "output_cost_per_token_above_256k_tokens": 3e-06, + "cache_creation_input_token_cost_above_256k_tokens": 9.375e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.6-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3.6-35b-a3b": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 9e-07, + "cache_read_input_token_cost": 5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.6-35b-a3b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3.6-max-preview": { + "input_cost_per_token": 1.027e-06, + "output_cost_per_token": 6.162e-06, + "cache_creation_input_token_cost": 1.28375e-06, + "input_cost_per_token_above_128k_tokens": 1.58e-06, + "output_cost_per_token_above_128k_tokens": 9.48e-06, + "cache_creation_input_token_cost_above_128k_tokens": 1.975e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.6-max-preview", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3.6-27b": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 3e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.6-27b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.5-pro": { + "input_cost_per_token": 3e-05, + "output_cost_per_token": 0.00018, + "input_cost_per_token_above_272k_tokens": 6e-05, + "output_cost_per_token_above_272k_tokens": 0.00027, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.5-pro", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true + }, + "openrouter/deepseek/deepseek-v4-flash": { + "input_cost_per_token": 8.778e-08, + "output_cost_per_token": 1.7556e-07, + "cache_read_input_token_cost": 1.7556e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "source": "https://openrouter.ai/deepseek/deepseek-v4-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/moonshotai/kimi-k2.6": { + "input_cost_per_token": 9.5e-07, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 1.6e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/moonshotai/kimi-k2.6", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/google/gemma-4-26b-a4b-it": { + "input_cost_per_token": 7e-08, + "output_cost_per_token": 3.4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/google/gemma-4-26b-a4b-it", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/google/gemma-4-26b-a4b-it:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/google/gemma-4-26b-a4b-it:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/google/gemma-4-31b-it": { + "input_cost_per_token": 9e-08, + "output_cost_per_token": 3.4e-07, + "cache_read_input_token_cost": 5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/google/gemma-4-31b-it", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/google/gemma-4-31b-it:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/google/gemma-4-31b-it:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/z-ai/glm-5v-turbo": { + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2.4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 202752, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-5v-turbo", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/minimax/minimax-m2.7": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 6e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/minimax/minimax-m2.7", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/minimax/minimax-m2.7:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 196608, + "max_output_tokens": 176947, + "max_tokens": 176947, + "mode": "chat", + "source": "https://openrouter.ai/minimax/minimax-m2.7:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/mistralai/mistral-small-2603": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "cache_read_input_token_cost": 1.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 209715, + "max_tokens": 209715, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/mistral-small-2603", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/z-ai/glm-5-turbo": { + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2.4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 202752, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-5-turbo", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/nvidia/nemotron-3-super-120b-a12b": { + "input_cost_per_token": 8.5e-08, + "output_cost_per_token": 4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3-super-120b-a12b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/nvidia/nemotron-3-super-120b-a12b:free": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3-super-120b-a12b:free", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3.5-9b": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3.5-9b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/openai/gpt-5.4-pro": { + "input_cost_per_token": 3e-05, + "output_cost_per_token": 0.00018, + "input_cost_per_token_above_272k_tokens": 6e-05, + "output_cost_per_token_above_272k_tokens": 0.00027, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.4-pro", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true + }, + "openrouter/google/gemini-3.1-flash-image-preview": { + "input_cost_per_token": 5e-07, + "output_cost_per_token": 3e-06, + "output_cost_per_image_token": 6e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 65536, + "max_output_tokens": 58982, + "max_tokens": 58982, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.1-flash-image-preview", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/google/gemini-3.1-pro-preview-customtools": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05, + "cache_read_input_token_cost": 2e-07, + "cache_creation_input_token_cost": 3.75e-07, + "input_cost_per_audio_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "output_cost_per_token_above_200k_tokens": 1.8e-05, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3.1-pro-preview-customtools", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3-max-thinking": { + "input_cost_per_token": 7.8e-07, + "output_cost_per_token": 3.9e-06, + "input_cost_per_token_above_128k_tokens": 1.95e-06, + "output_cost_per_token_above_128k_tokens": 9.75e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-max-thinking", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3-coder-next": { + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 8e-07, + "cache_read_input_token_cost": 7e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-coder-next", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/minimax/minimax-m2-her": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 3e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 65536, + "max_output_tokens": 2048, + "max_tokens": 2048, + "mode": "chat", + "source": "https://openrouter.ai/minimax/minimax-m2-her", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-audio": { + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1e-05, + "input_cost_per_audio_token": 3.2e-05, + "output_cost_per_audio_token": 6.4e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-audio", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_audio_input": true + }, + "openrouter/openai/gpt-audio-mini": { + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.4e-06, + "input_cost_per_audio_token": 6e-07, + "output_cost_per_audio_token": 2.4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-audio-mini", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_audio_input": true + }, + "openrouter/nvidia/nemotron-3-nano-30b-a3b": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2e-07, + "cache_read_input_token_cost": 3e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/nvidia/nemotron-3-nano-30b-a3b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/z-ai/glm-4.6v": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 9e-07, + "cache_read_input_token_cost": 5.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-4.6v", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/google/gemini-3-pro-image-preview": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05, + "cache_read_input_token_cost": 2e-07, + "cache_creation_input_token_cost": 3.75e-07, + "input_cost_per_audio_token": 2e-06, + "output_cost_per_image_token": 0.00012, + "litellm_provider": "openrouter", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-3-pro-image-preview", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.1-codex": { + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.3e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.1-codex", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-5.1-codex-mini": { + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 3e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5.1-codex-mini", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/moonshotai/kimi-k2-thinking": { + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 1.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 100352, + "max_tokens": 100352, + "mode": "chat", + "source": "https://openrouter.ai/moonshotai/kimi-k2-thinking", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/mistralai/voxtral-small-24b-2507": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, + "cache_read_input_token_cost": 1e-08, + "input_cost_per_audio_token": 0.0001, + "litellm_provider": "openrouter", + "max_input_tokens": 32768, + "max_output_tokens": 26214, + "max_tokens": 26214, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/voxtral-small-24b-2507", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_pdf_input": true, + "supports_audio_input": true, + "supports_prompt_caching": true + }, + "openrouter/openai/gpt-oss-safeguard-20b": { + "input_cost_per_token": 7.5e-08, + "output_cost_per_token": 3e-07, + "cache_read_input_token_cost": 3.75e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-oss-safeguard-20b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3-vl-32b-instruct": { + "input_cost_per_token": 1.04e-07, + "output_cost_per_token": 4.16e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-vl-32b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3-vl-8b-thinking": { + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 2.1e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-vl-8b-thinking", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3-vl-8b-instruct": { + "input_cost_per_token": 1.17e-07, + "output_cost_per_token": 4.55e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-vl-8b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/google/gemini-2.5-flash-image": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 8.33333333333333e-08, + "input_cost_per_audio_token": 1e-06, + "output_cost_per_image_token": 3e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-2.5-flash-image", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3-vl-30b-a3b-thinking": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2.4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-vl-30b-a3b-thinking", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3-vl-30b-a3b-instruct": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-vl-30b-a3b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/openai/gpt-5-pro": { + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 0.00012, + "litellm_provider": "openrouter", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-5-pro", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true + }, + "openrouter/qwen/qwen3-vl-235b-a22b-thinking": { + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-vl-235b-a22b-thinking", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3-vl-235b-a22b-instruct": { + "input_cost_per_token": 2.1e-07, + "output_cost_per_token": 1.9e-06, + "cache_read_input_token_cost": 1e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-vl-235b-a22b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3-max": { + "input_cost_per_token": 7.8e-07, + "output_cost_per_token": 3.9e-06, + "cache_read_input_token_cost": 1.56e-07, + "cache_creation_input_token_cost": 9.75e-07, + "input_cost_per_token_above_128k_tokens": 1.95e-06, + "output_cost_per_token_above_128k_tokens": 9.75e-06, + "cache_read_input_token_cost_above_128k_tokens": 3.9e-07, + "cache_creation_input_token_cost_above_128k_tokens": 2.4375e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-max", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/deepseek/deepseek-v3.1-terminus": { + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 1.35e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 163840, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/deepseek/deepseek-v3.1-terminus", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3-coder-flash": { + "input_cost_per_token": 1.95e-07, + "output_cost_per_token": 9.75e-07, + "cache_read_input_token_cost": 3.9e-08, + "cache_creation_input_token_cost": 2.4375e-07, + "input_cost_per_token_above_128k_tokens": 5.2e-07, + "output_cost_per_token_above_128k_tokens": 2.6e-06, + "cache_read_input_token_cost_above_128k_tokens": 1.04e-07, + "cache_creation_input_token_cost_above_128k_tokens": 6.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-coder-flash", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3-next-80b-a3b-thinking": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-next-80b-a3b-thinking", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3-next-80b-a3b-instruct": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1.1e-06, + "cache_read_input_token_cost": 7e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-next-80b-a3b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen-plus-2025-07-28": { + "input_cost_per_token": 2.6e-07, + "output_cost_per_token": 7.8e-07, + "input_cost_per_token_above_256k_tokens": 7.8e-07, + "output_cost_per_token_above_256k_tokens": 2.34e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen-plus-2025-07-28", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/moonshotai/kimi-k2-0905": { + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 100352, + "max_tokens": 100352, + "mode": "chat", + "source": "https://openrouter.ai/moonshotai/kimi-k2-0905", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3-30b-a3b-thinking-2507": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2.4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 81920, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-30b-a3b-thinking-2507", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/mistralai/mistral-medium-3.1": { + "input_cost_per_token": 4e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 4e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 104857, + "max_tokens": 104857, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/mistral-medium-3.1", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/z-ai/glm-4.5v": { + "input_cost_per_token": 6e-07, + "output_cost_per_token": 1.8e-06, + "cache_read_input_token_cost": 1.1e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 65536, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-4.5v", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/mistralai/codestral-2508": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 9e-07, + "cache_read_input_token_cost": 3e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 256000, + "max_output_tokens": 204800, + "max_tokens": 204800, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/codestral-2508", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen3-coder-30b-a3b-instruct": { + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2.8e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-coder-30b-a3b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3-30b-a3b-instruct-2507": { + "input_cost_per_token": 4.815e-08, + "output_cost_per_token": 1.9305e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-30b-a3b-instruct-2507", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/z-ai/glm-4.5": { + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.2e-06, + "cache_read_input_token_cost": 1.1e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 98304, + "max_tokens": 98304, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-4.5", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/z-ai/glm-4.5-air": { + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 8.5e-07, + "cache_read_input_token_cost": 2.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 98304, + "max_tokens": 98304, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-4.5-air", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/moonshotai/kimi-k2": { + "input_cost_per_token": 5.7e-07, + "output_cost_per_token": 2.3e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 100352, + "max_tokens": 100352, + "mode": "chat", + "source": "https://openrouter.ai/moonshotai/kimi-k2", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "openrouter/minimax/minimax-m1": { + "input_cost_per_token": 5.5e-07, + "output_cost_per_token": 2.2e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 40000, + "max_tokens": 40000, + "mode": "chat", + "source": "https://openrouter.ai/minimax/minimax-m1", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false + }, + "openrouter/openai/o3-pro": { + "input_cost_per_token": 2e-05, + "output_cost_per_token": 8e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "source": "https://openrouter.ai/openai/o3-pro", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true + }, + "openrouter/google/gemini-2.5-pro-preview": { + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost": 3.75e-07, + "input_cost_per_audio_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 1.5e-05, + "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-2.5-pro-preview", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "supports_prompt_caching": true + }, + "openrouter/mistralai/mistral-medium-3": { + "input_cost_per_token": 4e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 4e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 104857, + "max_tokens": 104857, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/mistral-medium-3", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/google/gemini-2.5-pro-preview-05-06": { + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost": 3.75e-07, + "input_cost_per_audio_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 1.5e-05, + "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_tokens": 65535, + "mode": "chat", + "source": "https://openrouter.ai/google/gemini-2.5-pro-preview-05-06", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_audio_input": true, + "supports_prompt_caching": true + }, + "openrouter/meta-llama/llama-guard-4-12b": { + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 163840, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/meta-llama/llama-guard-4-12b", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3-30b-a3b": { + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-30b-a3b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3-8b": { + "input_cost_per_token": 1.17e-07, + "output_cost_per_token": 4.55e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-8b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3-14b": { + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 2.4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-14b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3-32b": { + "input_cost_per_token": 8e-08, + "output_cost_per_token": 2.8e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-32b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen3-235b-a22b": { + "input_cost_per_token": 4.55e-07, + "output_cost_per_token": 1.82e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen3-235b-a22b", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/openai/o4-mini-high": { + "input_cost_per_token": 1.1e-06, + "output_cost_per_token": 4.4e-06, + "cache_read_input_token_cost": 2.75e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "source": "https://openrouter.ai/openai/o4-mini-high", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/meta-llama/llama-4-maverick": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6.96e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 115200, + "max_tokens": 115200, + "mode": "chat", + "source": "https://openrouter.ai/meta-llama/llama-4-maverick", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/meta-llama/llama-4-scout": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1310720, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/meta-llama/llama-4-scout", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/openai/o1-pro": { + "input_cost_per_token": 0.00015, + "output_cost_per_token": 0.0006, + "litellm_provider": "openrouter", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "source": "https://openrouter.ai/openai/o1-pro", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_pdf_input": true + }, + "openrouter/google/gemma-3-4b-it": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/google/gemma-3-4b-it", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/google/gemma-3-12b-it": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/google/gemma-3-12b-it", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/google/gemma-3-27b-it": { + "input_cost_per_token": 8e-08, + "output_cost_per_token": 4.5e-07, + "cache_read_input_token_cost": 4e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 117964, + "max_tokens": 117964, + "mode": "chat", + "source": "https://openrouter.ai/google/gemma-3-27b-it", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/mistralai/mistral-saba": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-07, + "cache_read_input_token_cost": 2e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 32768, + "max_output_tokens": 26214, + "max_tokens": 26214, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/mistral-saba", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen2.5-vl-72b-instruct": { + "input_cost_per_token": 8e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 115200, + "max_tokens": 115200, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen2.5-vl-72b-instruct", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen-plus": { + "input_cost_per_token": 2.6e-07, + "output_cost_per_token": 7.8e-07, + "cache_read_input_token_cost": 5.2e-08, + "cache_creation_input_token_cost": 3.25e-07, + "input_cost_per_token_above_256k_tokens": 7.8e-07, + "output_cost_per_token_above_256k_tokens": 2.34e-06, + "cache_read_input_token_cost_above_256k_tokens": 1.56e-07, + "cache_creation_input_token_cost_above_256k_tokens": 9.75e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen-plus", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/mistralai/mistral-small-24b-instruct-2501": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 8e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/mistral-small-24b-instruct-2501", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/deepseek/deepseek-r1-distill-llama-70b": { + "input_cost_per_token": 8e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 8192, + "max_output_tokens": 7372, + "max_tokens": 7372, + "mode": "chat", + "source": "https://openrouter.ai/deepseek/deepseek-r1-distill-llama-70b", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_reasoning": true, + "supports_vision": false + }, + "openrouter/minimax/minimax-01": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.1e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000192, + "max_output_tokens": 900172, + "max_tokens": 900172, + "mode": "chat", + "source": "https://openrouter.ai/minimax/minimax-01", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_vision": true + }, + "openrouter/meta-llama/llama-3.3-70b-instruct": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3.2e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/meta-llama/llama-3.3-70b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/openai/gpt-4o-2024-11-20": { + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-4o-2024-11-20", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_web_search": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/mistralai/mistral-large-2407": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 2e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 104857, + "max_tokens": 104857, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/mistral-large-2407", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/qwen/qwen-2.5-7b-instruct": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 32768, + "max_output_tokens": 29491, + "max_tokens": 29491, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen-2.5-7b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/meta-llama/llama-3.2-1b-instruct": { + "input_cost_per_token": 2.7e-08, + "output_cost_per_token": 2.01e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 60000, + "max_output_tokens": 54000, + "max_tokens": 54000, + "mode": "chat", + "source": "https://openrouter.ai/meta-llama/llama-3.2-1b-instruct", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_vision": false + }, + "openrouter/meta-llama/llama-3.2-3b-instruct": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 3.3e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 117964, + "max_tokens": 117964, + "mode": "chat", + "source": "https://openrouter.ai/meta-llama/llama-3.2-3b-instruct", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/qwen/qwen-2.5-72b-instruct": { + "input_cost_per_token": 3.6e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/qwen/qwen-2.5-72b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/openai/gpt-4o-2024-08-06": { + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-4o-2024-08-06", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_web_search": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/meta-llama/llama-3.1-70b-instruct": { + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/meta-llama/llama-3.1-70b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/meta-llama/llama-3.1-8b-instruct": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 8e-08, + "cache_read_input_token_cost": 2.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 117964, + "max_tokens": 117964, + "mode": "chat", + "source": "https://openrouter.ai/meta-llama/llama-3.1-8b-instruct", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "supports_prompt_caching": true + }, + "openrouter/mistralai/mistral-nemo": { + "input_cost_per_token": 1.9e-08, + "output_cost_per_token": 3e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/mistralai/mistral-nemo", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/openai/gpt-4o-mini-2024-07-18": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "cache_read_input_token_cost": 7.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-4o-mini-2024-07-18", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_web_search": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_prompt_caching": true + }, + "openrouter/google/gemma-2-27b-it": { + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 6.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 8192, + "max_output_tokens": 2048, + "max_tokens": 2048, + "mode": "chat", + "source": "https://openrouter.ai/google/gemma-2-27b-it", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/openai/gpt-4-turbo": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-4-turbo", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/openai/gpt-4-turbo-preview": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 3e-05, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-4-turbo-preview", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false + }, + "openrouter/openai/gpt-3.5-turbo-instruct": { + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 4095, + "max_output_tokens": 3685, + "max_tokens": 3685, + "mode": "chat", + "source": "https://openrouter.ai/openai/gpt-3.5-turbo-instruct", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_response_schema": true, + "supports_vision": false } } diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 9e370e5406a..a51149bf958 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -79,6 +79,11 @@ "minimum": 0, "description": "USD per token written to the provider's prompt cache." }, + "cache_creation_input_token_cost_above_128k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "cache_creation_input_token_cost_above_1hr": { "type": "number", "minimum": 0, @@ -94,6 +99,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "cache_creation_input_token_cost_above_256k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "cache_creation_input_token_cost_above_272k_tokens": { "type": "number", "minimum": 0, @@ -128,6 +138,11 @@ "minimum": 0, "description": "USD per prompt token served from the provider's prompt cache." }, + "cache_read_input_token_cost_above_128k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "cache_read_input_token_cost_above_200k_tokens": { "type": "number", "minimum": 0, @@ -138,6 +153,11 @@ "minimum": 0, "description": "Priority service-tier rate for the same-named base field." }, + "cache_read_input_token_cost_above_256k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "cache_read_input_token_cost_above_272k_tokens": { "type": "number", "minimum": 0, From 1f0611a8b90a57df101200a981a1029d79017b06 Mon Sep 17 00:00:00 2001 From: mateo Date: Fri, 4 Sep 2026 19:25:10 +0000 Subject: [PATCH 67/88] fix(registry): drop Together MiniMax M2.7 and revert Qwen2.5 7B Turbo pricing, both non-serverless Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../model_prices_and_context_window_backup.json | 17 +---------------- model_prices_and_context_window.json | 17 +---------------- 2 files changed, 2 insertions(+), 32 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index aa4f9b87377..476fe4143c1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -42472,12 +42472,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_tool_choice": true, - "input_cost_per_token": 3e-07, - "output_cost_per_token": 3e-07, - "max_input_tokens": 32768, - "max_tokens": 32768, - "source": "https://www.together.ai/pricing" + "supports_tool_choice": true }, "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { "deprecation_date": "2026-07-10", @@ -42899,16 +42894,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "together_ai/MiniMaxAI/MiniMax-M2.7": { - "cache_read_input_token_cost": 6e-08, - "input_cost_per_token": 3e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 196608, - "max_tokens": 196608, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://www.together.ai/pricing" - }, "together_ai/Prism-ML/Ternary-Bonsai-27B": { "input_cost_per_token": 0.0, "litellm_provider": "together_ai", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index aa4f9b87377..476fe4143c1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -42472,12 +42472,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_tool_choice": true, - "input_cost_per_token": 3e-07, - "output_cost_per_token": 3e-07, - "max_input_tokens": 32768, - "max_tokens": 32768, - "source": "https://www.together.ai/pricing" + "supports_tool_choice": true }, "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { "deprecation_date": "2026-07-10", @@ -42899,16 +42894,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "together_ai/MiniMaxAI/MiniMax-M2.7": { - "cache_read_input_token_cost": 6e-08, - "input_cost_per_token": 3e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 196608, - "max_tokens": 196608, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://www.together.ai/pricing" - }, "together_ai/Prism-ML/Ternary-Bonsai-27B": { "input_cost_per_token": 0.0, "litellm_provider": "together_ai", From 205a5e9d6cfe878fb489f360908fc45efcafe4a1 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:45:24 -0700 Subject: [PATCH 68/88] feat(mcp): use x-mcp--* headers as default upstream credentials for group members (#39717) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../mcp_server/mcp_server_manager.py | 3 + .../mcp_server/rest_endpoints.py | 7 +- .../proxy/_experimental/mcp_server/server.py | 6 +- .../proxy/_experimental/mcp_server/utils.py | 49 ++++++++++---- .../mcp_server/test_mcp_header_alias_utils.py | 64 +++++++++++++++++++ .../mcp_server/test_mcp_server.py | 37 +++++++++++ .../mcp_server/test_rest_endpoints.py | 37 ++++++++++- 7 files changed, 186 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index ce4928ff83d..bfc5f629faf 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -928,6 +928,7 @@ def _resolve_openapi_tool_auth( mcp_server_auth_headers, alias=mcp_server.alias, server_name=mcp_server.server_name, + access_groups=mcp_server.access_groups, ) if mcp_server_auth_headers else None @@ -3296,6 +3297,7 @@ class MCPServerManager: mcp_server_auth_headers, alias=server.alias, server_name=server.server_name, + access_groups=server.access_groups, ) # Fall back to deprecated mcp_auth_header if no server-specific header found @@ -5373,6 +5375,7 @@ class MCPServerManager: mcp_server_auth_headers, alias=mcp_server.alias, server_name=mcp_server.server_name, + access_groups=mcp_server.access_groups, ) # Fall back to deprecated mcp_auth_header if no server-specific header found diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 37474f85fe7..b3469da9071 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -257,7 +257,7 @@ if MCP_AVAILABLE: ) def _get_server_auth_header( - server, + server: MCPServer, mcp_server_auth_headers: dict[str, dict[str, str]] | None, mcp_auth_header: str | None, ) -> dict[str, str] | str | None: @@ -269,8 +269,9 @@ if MCP_AVAILABLE: if mcp_server_auth_headers: server_auth: Final = lookup_mcp_server_auth_in_headers( mcp_server_auth_headers, - alias=getattr(server, "alias", None), - server_name=getattr(server, "server_name", None), + alias=server.alias, + server_name=server.server_name, + access_groups=server.access_groups, ) if server_auth is not None: return server_auth diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 3d7c947a913..975d9642b36 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -1612,7 +1612,10 @@ if MCP_AVAILABLE: ) server_headers: Final = lookup_mcp_server_auth_in_headers( - mcp_server_auth_headers, alias=server.alias, server_name=server.server_name + mcp_server_auth_headers, + alias=server.alias, + server_name=server.server_name, + access_groups=server.access_groups, ) if isinstance(server_headers, str): return bool(server_headers.strip()) @@ -1712,6 +1715,7 @@ if MCP_AVAILABLE: mcp_server_auth_headers, alias=server.alias, server_name=server.server_name, + access_groups=server.access_groups, ) extra_headers: dict[str, str] | None = None diff --git a/litellm/proxy/_experimental/mcp_server/utils.py b/litellm/proxy/_experimental/mcp_server/utils.py index 83883664df5..252756e0458 100644 --- a/litellm/proxy/_experimental/mcp_server/utils.py +++ b/litellm/proxy/_experimental/mcp_server/utils.py @@ -8,11 +8,12 @@ import json import os import re import typing -from collections.abc import Iterable, Iterator, Mapping, MutableMapping, MutableSequence +from collections.abc import Iterable, Iterator, Mapping, MutableMapping, MutableSequence, Sequence from collections.abc import Set as AbstractSet from typing import Any, Final, Protocol from urllib.parse import quote +from litellm._logging import verbose_logger from litellm.types.mcp_server.mcp_server_manager import MCPServer if typing.TYPE_CHECKING: @@ -169,34 +170,58 @@ def sanitize_mcp_alias_for_header(alias: str) -> str: return sanitized.strip("_") +def _header_keys_for_identifier(identifier: str) -> tuple[str, ...]: + lowered: Final = identifier.lower() + sanitized: Final = sanitize_mcp_alias_for_header(identifier) + return (lowered,) if not sanitized or sanitized == lowered else (lowered, sanitized) + + +def _matching_header_key(normalized_headers: Mapping[str, object], identifier: str) -> str | None: + return next((key for key in _header_keys_for_identifier(identifier) if key in normalized_headers), None) + + def lookup_mcp_server_auth_in_headers( mcp_server_auth_headers: Mapping[str, str | dict[str, str]], *, alias: str | None = None, server_name: str | None = None, + access_groups: Sequence[str] | None = None, ) -> str | dict[str, str] | None: """ Resolve server-specific auth headers with case-insensitive matching. Tries the raw alias/server_name (lowercased) and the header-safe sanitized alias so dashboard clients using sanitize_mcp_alias_for_header() still match. + + When no server-level header matches, an ``x-mcp-{access_group}-*`` header is + used as the default for every server in that group. If the server belongs to + several groups that each carry a different credential, nothing is returned so + a token is never forwarded to a server it may not have been meant for. """ if not mcp_server_auth_headers: return None normalized_headers: Final = {k.lower(): v for k, v in mcp_server_auth_headers.items()} - for identifier in (alias, server_name): - if not identifier: - continue - keys_to_try = [identifier.lower()] - sanitized = sanitize_mcp_alias_for_header(identifier) - if sanitized and sanitized not in keys_to_try: - keys_to_try.append(sanitized) - for key in keys_to_try: - if key in normalized_headers: - return normalized_headers[key] - return None + server_keys: Final = ( + _matching_header_key(normalized_headers, identifier) for identifier in (alias, server_name) if identifier + ) + server_key: Final = next((key for key in server_keys if key is not None), None) + if server_key is not None: + return normalized_headers[server_key] + + group_keys: Final = (_matching_header_key(normalized_headers, group) for group in access_groups or ()) + group_matches: Final = tuple(normalized_headers[key] for key in group_keys if key is not None) + if not group_matches: + return None + if any(match != group_matches[0] for match in group_matches[1:]): + verbose_logger.debug( + "Ambiguous MCP group auth headers for server alias=%s (groups=%s); not forwarding any group credential", + alias, + access_groups, + ) + return None + return group_matches[0] MCP_TOOL_ALLOWLIST_ENFORCED_KEY: Final = "tool_allowlist_enforced" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_header_alias_utils.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_header_alias_utils.py index 2627199570b..6c24205c258 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_header_alias_utils.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_header_alias_utils.py @@ -16,3 +16,67 @@ def test_lookup_mcp_server_auth_in_headers_sanitized_alias(): headers = {"github_mcp": {"Authorization": "Bearer token"}} result = lookup_mcp_server_auth_in_headers(headers, alias="GitHub-MCP") assert result == {"Authorization": "Bearer token"} + + +def test_lookup_mcp_server_auth_in_headers_group_header_is_default_for_members(): + headers = {"shared": {"Authorization": "Bearer group-token"}} + assert lookup_mcp_server_auth_in_headers(headers, alias="alpha", server_name="alpha", access_groups=["shared"]) == { + "Authorization": "Bearer group-token" + } + assert lookup_mcp_server_auth_in_headers(headers, alias="beta", server_name="beta", access_groups=["Shared"]) == { + "Authorization": "Bearer group-token" + } + + +def test_lookup_mcp_server_auth_in_headers_group_header_sanitized_group_name(): + headers = {"dev_group": {"Authorization": "Bearer group-token"}} + assert lookup_mcp_server_auth_in_headers(headers, alias="alpha", access_groups=["Dev Group"]) == { + "Authorization": "Bearer group-token" + } + + +def test_lookup_mcp_server_auth_in_headers_server_header_overrides_group_header(): + headers = { + "shared": {"Authorization": "Bearer group-token"}, + "beta": {"Authorization": "Bearer beta-token"}, + } + assert lookup_mcp_server_auth_in_headers(headers, alias="beta", server_name="beta", access_groups=["shared"]) == { + "Authorization": "Bearer beta-token" + } + + +def test_lookup_mcp_server_auth_in_headers_group_header_not_forwarded_outside_group(): + headers = {"shared": {"Authorization": "Bearer group-token"}} + assert ( + lookup_mcp_server_auth_in_headers(headers, alias="gamma", server_name="gamma", access_groups=["other"]) is None + ) + assert lookup_mcp_server_auth_in_headers(headers, alias="gamma", server_name="gamma", access_groups=None) is None + + +def test_lookup_mcp_server_auth_in_headers_alias_colliding_with_group_name_keeps_server_level_match(): + headers = {"shared": {"Authorization": "Bearer shared-token"}} + assert lookup_mcp_server_auth_in_headers(headers, alias="shared", access_groups=["other"]) == { + "Authorization": "Bearer shared-token" + } + assert lookup_mcp_server_auth_in_headers(headers, alias="alpha", access_groups=["shared"]) == { + "Authorization": "Bearer shared-token" + } + assert lookup_mcp_server_auth_in_headers(headers, alias="gamma", access_groups=["other"]) is None + + +def test_lookup_mcp_server_auth_in_headers_conflicting_group_headers_fail_closed(): + headers = { + "shared": {"Authorization": "Bearer group-token"}, + "other": {"Authorization": "Bearer other-token"}, + } + assert lookup_mcp_server_auth_in_headers(headers, alias="delta", access_groups=["shared", "other"]) is None + + +def test_lookup_mcp_server_auth_in_headers_identical_group_headers_resolve(): + headers = { + "shared": {"Authorization": "Bearer group-token"}, + "other": {"Authorization": "Bearer group-token"}, + } + assert lookup_mcp_server_auth_in_headers(headers, alias="delta", access_groups=["shared", "other"]) == { + "Authorization": "Bearer group-token" + } diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index 9a6815a61e5..086ab854e36 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -303,6 +303,43 @@ def test_prepare_mcp_server_headers_case_insensitive_extra_headers(): assert extra_headers == {"Authorization": "Bearer token"} +def test_prepare_mcp_server_headers_group_header_defaults_for_members_only(): + try: + from litellm.proxy._experimental.mcp_server.server import ( + _prepare_mcp_server_headers, + ) + except ImportError: + pytest.skip("MCP server not available") + + def server(alias: str, group: str) -> MCPServer: + return MCPServer( + server_id=f"server-{alias}", + name=alias, + alias=alias, + transport=MCPTransport.http, + access_groups=[group], + ) + + mcp_server_auth_headers = { + "shared": {"Authorization": "Bearer group-token"}, + "beta": {"Authorization": "Bearer beta-token"}, + } + + def resolve(mcp_server: MCPServer): + server_auth_header, _ = _prepare_mcp_server_headers( + server=mcp_server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=None, + oauth2_headers=None, + raw_headers={"x-litellm-api-key": "Bearer sk-litellm-key"}, + ) + return server_auth_header + + assert resolve(server("alpha", "shared")) == {"Authorization": "Bearer group-token"} + assert resolve(server("beta", "shared")) == {"Authorization": "Bearer beta-token"} + assert resolve(server("gamma", "other")) is None + + def test_prepare_mcp_server_headers_passthrough_strips_authorization_without_admission_header(): try: from litellm.proxy._experimental.mcp_server.server import ( diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index d441c05090b..f2c8f8c80c5 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -25,7 +25,8 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.types.mcp import MCPAuth +from litellm.types.mcp import MCPAuth, MCPTransport +from litellm.types.mcp_server.mcp_server_manager import MCPServer def _rendered_log_message(call): @@ -3268,6 +3269,40 @@ class TestConnectionErrorMessage: assert "proxy logs" in message.lower() +class TestGetServerAuthHeaderGroupDefault: + """``x-mcp--authorization`` is the default for group members, the per-server + header still wins, and servers outside the group never see the group credential.""" + + @staticmethod + def _server(alias: str, group: str) -> MCPServer: + return MCPServer( + server_id=f"server-{alias}", + name=alias, + server_name=alias, + alias=alias, + url="https://example.com/mcp", + transport=MCPTransport.http, + access_groups=[group], + ) + + def test_group_header_applies_to_members_and_per_server_header_overrides(self): + headers = { + "shared": {"Authorization": "Bearer group-token"}, + "beta": {"Authorization": "Bearer beta-token"}, + } + assert rest_endpoints._get_server_auth_header(self._server("alpha", "shared"), headers, None) == { + "Authorization": "Bearer group-token" + } + assert rest_endpoints._get_server_auth_header(self._server("beta", "shared"), headers, None) == { + "Authorization": "Bearer beta-token" + } + + def test_group_header_falls_back_to_legacy_header_outside_group(self): + headers = {"shared": {"Authorization": "Bearer group-token"}} + assert rest_endpoints._get_server_auth_header(self._server("gamma", "other"), headers, None) is None + assert rest_endpoints._get_server_auth_header(self._server("gamma", "other"), headers, "legacy") == "legacy" + + class TestToolResponseMcpInfoEnrichment: """The REST tools/list response must expose the user-facing alias and the server_id alongside the internal server_name so clients (agent builder UIs) From 11f272e08bdb954a9938a7370eec2765fe388502 Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 3 Sep 2026 23:30:51 +0000 Subject: [PATCH 69/88] fix(ui): paginate per-user usage with the shared server-side DataTable footer Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../src/components/per_user_usage.test.tsx | 100 ++++++++++++++++++ .../src/components/per_user_usage.tsx | 87 ++++++--------- 2 files changed, 135 insertions(+), 52 deletions(-) diff --git a/ui/litellm-dashboard/src/components/per_user_usage.test.tsx b/ui/litellm-dashboard/src/components/per_user_usage.test.tsx index 9cd199d786c..f92039e55b8 100644 --- a/ui/litellm-dashboard/src/components/per_user_usage.test.tsx +++ b/ui/litellm-dashboard/src/components/per_user_usage.test.tsx @@ -1,4 +1,5 @@ import { render, screen, fireEvent, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { describe, it, expect, vi, beforeEach } from "vitest"; import PerUserUsage from "./per_user_usage"; import * as networking from "./networking"; @@ -94,6 +95,105 @@ describe("PerUserUsage", () => { expect(screen.getByText("u1")).toBeInTheDocument(); }); + describe("server pagination", () => { + const TOTAL_USERS = 120; + + const pageOfUsers = (page: number, pageSize: number): UserRow[] => { + const start = (page - 1) * pageSize; + const count = Math.max(0, Math.min(pageSize, TOTAL_USERS - start)); + return Array.from({ length: count }, (_, index) => userRow(`user-${start + index + 1}`, "curl/8.0", 5)); + }; + + beforeEach(() => { + mockPerUserAnalyticsCall.mockImplementation(async (_token, page = 1, pageSize = 50) => ({ + results: pageOfUsers(page, pageSize), + total_count: TOTAL_USERS, + page, + page_size: pageSize, + total_pages: Math.ceil(TOTAL_USERS / pageSize), + })); + }); + + const lastCall = () => mockPerUserAnalyticsCall.mock.calls[mockPerUserAnalyticsCall.mock.calls.length - 1]; + + it("renders every row the server returns and shows the range from total_count", async () => { + render(); + + expect(await screen.findByText("user-50")).toBeInTheDocument(); + expect(screen.getByText("user-1")).toBeInTheDocument(); + expect(screen.getAllByRole("row")).toHaveLength(51); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 1-50 of 120"); + expect(screen.getByTestId("pagination-prev")).toBeDisabled(); + expect(screen.getByTestId("pagination-next")).toBeEnabled(); + }); + + it("refetches the next page when Next is clicked", async () => { + const user = userEvent.setup(); + render(); + await screen.findByText("user-1"); + + await user.click(screen.getByTestId("pagination-next")); + + expect(await screen.findByText("user-51")).toBeInTheDocument(); + expect(lastCall()).toEqual(["test-token", 2, 50, undefined]); + expect(screen.queryByText("user-1")).not.toBeInTheDocument(); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 51-100 of 120"); + expect(screen.getByTestId("pagination-prev")).toBeEnabled(); + }); + + it("disables Next once the response says this is the last page", async () => { + const user = userEvent.setup(); + render(); + await screen.findByText("user-1"); + + await user.click(screen.getByTestId("pagination-last")); + + expect(await screen.findByText("user-120")).toBeInTheDocument(); + expect(lastCall()).toEqual(["test-token", 3, 50, undefined]); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 101-120 of 120"); + expect(screen.getByTestId("pagination-next")).toBeDisabled(); + }); + + it("refetches with the selected page size and goes back to the first page", async () => { + const user = userEvent.setup(); + render(); + await screen.findByText("user-1"); + await user.click(screen.getByTestId("pagination-next")); + await screen.findByText("user-51"); + + await user.click(screen.getByTestId("pagination-page-size")); + await user.click(await screen.findByRole("option", { name: "100" })); + + expect(await screen.findByText("user-100")).toBeInTheDocument(); + expect(lastCall()).toEqual(["test-token", 1, 100, undefined]); + expect(screen.getAllByRole("row")).toHaveLength(101); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 1-100 of 120"); + }); + + it("goes back to the first page when the tag filter changes", async () => { + const user = userEvent.setup(); + const { rerender } = render(); + await screen.findByText("user-1"); + await user.click(screen.getByTestId("pagination-next")); + await screen.findByText("user-51"); + + rerender(); + + await waitFor(() => { + expect(lastCall()).toEqual(["test-token", 1, 50, ["curl/8.0"]]); + }); + expect(await screen.findByText("user-1")).toBeInTheDocument(); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 1-50 of 120"); + }); + + it("does not request anything without an access token", () => { + render(); + + expect(mockPerUserAnalyticsCall).not.toHaveBeenCalled(); + expect(screen.getByText("No per-user usage data")).toBeInTheDocument(); + }); + }); + it("renders the usage distribution as a stacked bar chart with the explicit palette and users formatter", async () => { render(); diff --git a/ui/litellm-dashboard/src/components/per_user_usage.tsx b/ui/litellm-dashboard/src/components/per_user_usage.tsx index 6f29077de84..6e5cbb26014 100644 --- a/ui/litellm-dashboard/src/components/per_user_usage.tsx +++ b/ui/litellm-dashboard/src/components/per_user_usage.tsx @@ -1,8 +1,7 @@ -import React, { useState, useEffect } from "react"; -import type { ColumnDef } from "@tanstack/react-table"; +import React, { useState, useEffect, useCallback } from "react"; +import type { ColumnDef, OnChangeFn, PaginationState } from "@tanstack/react-table"; import { BarChart } from "@/components/shared/charts"; import { DataTable } from "@/components/shared/DataTable"; -import { Button } from "@/components/ui/button"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { perUserAnalyticsCall } from "./networking"; @@ -42,39 +41,38 @@ const PerUserUsage: React.FC = ({ accessToken, selectedTags, total_pages: 0, }); - const [currentPage, setCurrentPage] = useState(1); - - const fetchPerUserData = async () => { - if (!accessToken) return; - - try { - const response = await perUserAnalyticsCall( - accessToken, - currentPage, - 50, - selectedTags.length > 0 ? selectedTags : undefined, - ); - setPerUserData(response); - } catch (error) { - console.error("Failed to fetch per-user data:", error); - } - }; + const [pagination, setPagination] = useState({ pageIndex: 0, pageSize: 50 }); useEffect(() => { - fetchPerUserData(); - }, [accessToken, selectedTags, currentPage]); + setPagination((prev) => (prev.pageIndex === 0 ? prev : { ...prev, pageIndex: 0 })); + }, [selectedTags]); - const handleNextPage = () => { - if (currentPage < perUserData.total_pages) { - setCurrentPage(currentPage + 1); - } - }; + useEffect(() => { + if (!accessToken) return; - const handlePrevPage = () => { - if (currentPage > 1) { - setCurrentPage(currentPage - 1); - } - }; + let stale = false; + perUserAnalyticsCall( + accessToken, + pagination.pageIndex + 1, + pagination.pageSize, + selectedTags.length > 0 ? selectedTags : undefined, + ) + .then((response) => { + if (!stale) setPerUserData(response); + }) + .catch((error) => console.error("Failed to fetch per-user data:", error)); + + return () => { + stale = true; + }; + }, [accessToken, selectedTags, pagination]); + + const handlePaginationChange = useCallback>((updaterOrValue) => { + setPagination((prev) => { + const next = typeof updaterOrValue === "function" ? updaterOrValue(prev) : updaterOrValue; + return next.pageSize === prev.pageSize ? next : { pageIndex: 0, pageSize: next.pageSize }; + }); + }, []); const columns: ColumnDef[] = [ { @@ -137,30 +135,15 @@ const PerUserUsage: React.FC = ({ accessToken, selectedTags, row.user_id} + paginationMode="server" + pagination={pagination} + onPaginationChange={handlePaginationChange} + rowCount={perUserData.total_count} noDataMessage="No per-user usage data" size="compact" /> - - {perUserData.results.length > 10 && ( -
-

Showing 10 of {perUserData.total_count} results

-
- - -
-
- )} {/* Tab 2: Usage Distribution Histogram */} From fd42bddee66d93bdab85145685b15e63daacb230 Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 3 Sep 2026 23:52:12 +0000 Subject: [PATCH 70/88] fix(ui): reset per-user usage page in the same render as the tag filter change Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../src/components/per_user_usage.test.tsx | 4 ++++ ui/litellm-dashboard/src/components/per_user_usage.tsx | 10 ++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/components/per_user_usage.test.tsx b/ui/litellm-dashboard/src/components/per_user_usage.test.tsx index f92039e55b8..3ce85580ca8 100644 --- a/ui/litellm-dashboard/src/components/per_user_usage.test.tsx +++ b/ui/litellm-dashboard/src/components/per_user_usage.test.tsx @@ -176,12 +176,16 @@ describe("PerUserUsage", () => { await screen.findByText("user-1"); await user.click(screen.getByTestId("pagination-next")); await screen.findByText("user-51"); + const callsBeforeTagChange = mockPerUserAnalyticsCall.mock.calls.length; rerender(); await waitFor(() => { expect(lastCall()).toEqual(["test-token", 1, 50, ["curl/8.0"]]); }); + expect(mockPerUserAnalyticsCall.mock.calls.slice(callsBeforeTagChange)).toEqual([ + ["test-token", 1, 50, ["curl/8.0"]], + ]); expect(await screen.findByText("user-1")).toBeInTheDocument(); expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 1-50 of 120"); }); diff --git a/ui/litellm-dashboard/src/components/per_user_usage.tsx b/ui/litellm-dashboard/src/components/per_user_usage.tsx index 6e5cbb26014..215a2032d4f 100644 --- a/ui/litellm-dashboard/src/components/per_user_usage.tsx +++ b/ui/litellm-dashboard/src/components/per_user_usage.tsx @@ -42,10 +42,12 @@ const PerUserUsage: React.FC = ({ accessToken, selectedTags, }); const [pagination, setPagination] = useState({ pageIndex: 0, pageSize: 50 }); + const [pagedTags, setPagedTags] = useState(selectedTags); - useEffect(() => { + if (pagedTags !== selectedTags) { + setPagedTags(selectedTags); setPagination((prev) => (prev.pageIndex === 0 ? prev : { ...prev, pageIndex: 0 })); - }, [selectedTags]); + } useEffect(() => { if (!accessToken) return; @@ -55,7 +57,7 @@ const PerUserUsage: React.FC = ({ accessToken, selectedTags, accessToken, pagination.pageIndex + 1, pagination.pageSize, - selectedTags.length > 0 ? selectedTags : undefined, + pagedTags.length > 0 ? pagedTags : undefined, ) .then((response) => { if (!stale) setPerUserData(response); @@ -65,7 +67,7 @@ const PerUserUsage: React.FC = ({ accessToken, selectedTags, return () => { stale = true; }; - }, [accessToken, selectedTags, pagination]); + }, [accessToken, pagedTags, pagination]); const handlePaginationChange = useCallback>((updaterOrValue) => { setPagination((prev) => { From 7fde31fe08c56778681901d5ff667bf821882e42 Mon Sep 17 00:00:00 2001 From: ryan Date: Fri, 4 Sep 2026 00:16:06 +0000 Subject: [PATCH 71/88] fix(ui): fall back to the last page when per-user usage shrinks under the current page Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../src/components/per_user_usage.test.tsx | 36 +++++++++++++++---- .../src/components/per_user_usage.tsx | 6 +++- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/ui/litellm-dashboard/src/components/per_user_usage.test.tsx b/ui/litellm-dashboard/src/components/per_user_usage.test.tsx index 3ce85580ca8..d5c2216dfd5 100644 --- a/ui/litellm-dashboard/src/components/per_user_usage.test.tsx +++ b/ui/litellm-dashboard/src/components/per_user_usage.test.tsx @@ -98,20 +98,24 @@ describe("PerUserUsage", () => { describe("server pagination", () => { const TOTAL_USERS = 120; - const pageOfUsers = (page: number, pageSize: number): UserRow[] => { + const pageOfUsers = (page: number, pageSize: number, total: number): UserRow[] => { const start = (page - 1) * pageSize; - const count = Math.max(0, Math.min(pageSize, TOTAL_USERS - start)); + const count = Math.max(0, Math.min(pageSize, total - start)); return Array.from({ length: count }, (_, index) => userRow(`user-${start + index + 1}`, "curl/8.0", 5)); }; - beforeEach(() => { + const serveUsers = (total: number) => { mockPerUserAnalyticsCall.mockImplementation(async (_token, page = 1, pageSize = 50) => ({ - results: pageOfUsers(page, pageSize), - total_count: TOTAL_USERS, + results: pageOfUsers(page, pageSize, total), + total_count: total, page, page_size: pageSize, - total_pages: Math.ceil(TOTAL_USERS / pageSize), + total_pages: Math.ceil(total / pageSize), })); + }; + + beforeEach(() => { + serveUsers(TOTAL_USERS); }); const lastCall = () => mockPerUserAnalyticsCall.mock.calls[mockPerUserAnalyticsCall.mock.calls.length - 1]; @@ -154,6 +158,26 @@ describe("PerUserUsage", () => { expect(screen.getByTestId("pagination-next")).toBeDisabled(); }); + it("falls back to the last existing page when the data shrinks under the current page", async () => { + const user = userEvent.setup(); + render(); + await screen.findByText("user-1"); + await user.click(screen.getByTestId("pagination-next")); + await screen.findByText("user-51"); + + serveUsers(60); + await user.click(screen.getByTestId("pagination-next")); + + expect(await screen.findByText("user-60")).toBeInTheDocument(); + expect(mockPerUserAnalyticsCall.mock.calls.slice(-2)).toEqual([ + ["test-token", 3, 50, undefined], + ["test-token", 2, 50, undefined], + ]); + expect(screen.getAllByRole("row")).toHaveLength(11); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 51-60 of 60"); + expect(screen.getByTestId("pagination-next")).toBeDisabled(); + }); + it("refetches with the selected page size and goes back to the first page", async () => { const user = userEvent.setup(); render(); diff --git a/ui/litellm-dashboard/src/components/per_user_usage.tsx b/ui/litellm-dashboard/src/components/per_user_usage.tsx index 215a2032d4f..57b1a926ec0 100644 --- a/ui/litellm-dashboard/src/components/per_user_usage.tsx +++ b/ui/litellm-dashboard/src/components/per_user_usage.tsx @@ -60,7 +60,11 @@ const PerUserUsage: React.FC = ({ accessToken, selectedTags, pagedTags.length > 0 ? pagedTags : undefined, ) .then((response) => { - if (!stale) setPerUserData(response); + if (stale) return; + setPerUserData(response); + if (response.total_pages > 0 && pagination.pageIndex >= response.total_pages) { + setPagination({ ...pagination, pageIndex: response.total_pages - 1 }); + } }) .catch((error) => console.error("Failed to fetch per-user data:", error)); From 3bdc5ecd0e2775ff7f79a7166f16f5e7cb746371 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 12:47:37 -0700 Subject: [PATCH 72/88] refactor(ui): drop the per-user usage page clamp now handled by the shared DataTable The shared DataTable clamps a server-mode page index whenever rowCount no longer reaches it (#39776), including the empty-dataset case this table's own clamp skipped because it required total_pages > 0. Remove the local clamp and cover the empty case through the component so the wiring into the shared behavior is what the tests prove --- .../src/components/per_user_usage.test.tsx | 24 +++++++++++++++++++ .../src/components/per_user_usage.tsx | 3 --- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/per_user_usage.test.tsx b/ui/litellm-dashboard/src/components/per_user_usage.test.tsx index d5c2216dfd5..30b3059b7d2 100644 --- a/ui/litellm-dashboard/src/components/per_user_usage.test.tsx +++ b/ui/litellm-dashboard/src/components/per_user_usage.test.tsx @@ -178,6 +178,30 @@ describe("PerUserUsage", () => { expect(screen.getByTestId("pagination-next")).toBeDisabled(); }); + it("goes back to the first page when the data disappears under the current page", async () => { + const user = userEvent.setup(); + render(); + await screen.findByText("user-1"); + await user.click(screen.getByTestId("pagination-next")); + await screen.findByText("user-51"); + + serveUsers(0); + await user.click(screen.getByTestId("pagination-next")); + + await waitFor(() => { + expect(lastCall()).toEqual(["test-token", 1, 50, undefined]); + }); + expect(mockPerUserAnalyticsCall.mock.calls.slice(-2)).toEqual([ + ["test-token", 3, 50, undefined], + ["test-token", 1, 50, undefined], + ]); + expect(screen.getByText("No per-user usage data")).toBeInTheDocument(); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("No results"); + expect(screen.getByText("Page 1 of 1")).toBeInTheDocument(); + expect(screen.getByTestId("pagination-first")).toBeDisabled(); + expect(screen.getByTestId("pagination-prev")).toBeDisabled(); + }); + it("refetches with the selected page size and goes back to the first page", async () => { const user = userEvent.setup(); render(); diff --git a/ui/litellm-dashboard/src/components/per_user_usage.tsx b/ui/litellm-dashboard/src/components/per_user_usage.tsx index 57b1a926ec0..f600cd6c45d 100644 --- a/ui/litellm-dashboard/src/components/per_user_usage.tsx +++ b/ui/litellm-dashboard/src/components/per_user_usage.tsx @@ -62,9 +62,6 @@ const PerUserUsage: React.FC = ({ accessToken, selectedTags, .then((response) => { if (stale) return; setPerUserData(response); - if (response.total_pages > 0 && pagination.pageIndex >= response.total_pages) { - setPagination({ ...pagination, pageIndex: response.total_pages - 1 }); - } }) .catch((error) => console.error("Failed to fetch per-user data:", error)); From f5157a63eb030b8130be567b03c49deac226cd70 Mon Sep 17 00:00:00 2001 From: mateo Date: Fri, 4 Sep 2026 19:56:30 +0000 Subject: [PATCH 73/88] test: allow 128k and 256k tiered cache fields in registry schema test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/test_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 580d8dfcc09..14907e17b1b 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -906,7 +906,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_creation_input_audio_token_cost": {"type": "number"}, "cache_creation_input_token_cost": {"type": "number"}, "cache_creation_input_token_cost_above_1hr": {"type": "number"}, + "cache_creation_input_token_cost_above_128k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_200k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_256k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_272k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_272k_tokens_flex": { "type": "number" @@ -917,7 +919,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_creation_input_token_cost_flex": {"type": "number"}, "cache_creation_input_token_cost_priority": {"type": "number"}, "cache_read_input_token_cost": {"type": "number"}, + "cache_read_input_token_cost_above_128k_tokens": {"type": "number"}, "cache_read_input_token_cost_above_200k_tokens": {"type": "number"}, + "cache_read_input_token_cost_above_256k_tokens": {"type": "number"}, "cache_read_input_token_cost_above_272k_tokens": {"type": "number"}, "cache_read_input_token_cost_above_272k_tokens_flex": { "type": "number" From 976ff0a7855c3e533447a7f008430a657c0015a1 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 12:58:33 -0700 Subject: [PATCH 74/88] fix(organization): 422 on negative limits and unparseable budget_duration in v2 update --- .../management_endpoints/common_utils.py | 4 +- .../organization_endpoints.py | 12 ++++++ .../test_organization_endpoints.py | 40 +++++++++++++++++++ 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index 2241884faf1..abf8e287a2f 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -22,7 +22,7 @@ def validate_finite_spend(spend: float | None) -> None: ) -def validate_budget_duration(budget_duration: str | None) -> None: +def validate_budget_duration(budget_duration: str | None, status_code: int = 400) -> None: """Reject budget durations that can't be parsed, are non-positive, or overflow date math, so a bad value can't be persisted and later crash the budget reset job. @@ -44,7 +44,7 @@ def validate_budget_duration(budget_duration: str | None) -> None: get_budget_reset_time(budget_duration=budget_duration) except (ValueError, OverflowError): raise HTTPException( - status_code=400, + status_code=status_code, detail={ "error": f"Invalid budget_duration '{budget_duration}'. Use a format like '1h', '24h', '7d', or '30d'." }, diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py index 5e38a016099..24620bf94ae 100644 --- a/litellm/proxy/management_endpoints/organization_endpoints.py +++ b/litellm/proxy/management_endpoints/organization_endpoints.py @@ -41,6 +41,7 @@ from litellm.proxy.management_endpoints.common_daily_activity import get_daily_a from litellm.proxy.management_endpoints.common_utils import ( _set_object_metadata_field, _user_has_admin_view, + validate_budget_duration, ) from litellm.proxy.management_helpers.object_permission_utils import ( handle_update_object_permission_common, @@ -807,6 +808,17 @@ async def update_organization_v2( status_code=422, detail={"error": f"soft_budget must be a non-negative finite number. Received: {data.soft_budget}"}, ) + for limit_name, limit_value in ( + ("tpm_limit", data.tpm_limit), + ("rpm_limit", data.rpm_limit), + ("max_parallel_requests", data.max_parallel_requests), + ): + if limit_value is not None and limit_value < 0: + raise HTTPException( + status_code=422, + detail={"error": f"{limit_name} must be non-negative. Received: {limit_value}"}, + ) + validate_budget_duration(data.budget_duration, status_code=422) if data.model_max_budget: from litellm.proxy.management_endpoints.key_management_endpoints import ( validate_model_max_budget, diff --git a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py index e2d89a660c2..f9fe4b8af5a 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py @@ -814,6 +814,46 @@ async def test_v2_rejects_negative_max_budget(monkeypatch): assert "max_budget" in str(exc.value.detail) +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["tpm_limit", "rpm_limit", "max_parallel_requests"]) +async def test_v2_rejects_negative_integer_limits(monkeypatch: pytest.MonkeyPatch, field: str): + """v2 rejects negative tpm/rpm/parallel-request limits with a 422 instead of persisting them to the budget row.""" + from litellm.proxy._types import LitellmUserRoles, OrganizationUpdateRequestV2, UserAPIKeyAuth + from litellm.proxy.management_endpoints.organization_endpoints import update_organization_v2 + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", AsyncMock()) + + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin-1") + with pytest.raises(HTTPException) as exc: + await update_organization_v2( + organization_id="org-1", + data=OrganizationUpdateRequestV2.model_validate({field: -1}), + user_api_key_dict=auth, + ) + assert exc.value.status_code == 422 + assert field in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_v2_rejects_unparseable_budget_duration(monkeypatch: pytest.MonkeyPatch): + """v2 rejects a budget_duration the parser can't read with a 422 instead of persisting it alongside a silent + next-midnight fallback reset.""" + from litellm.proxy._types import LitellmUserRoles, OrganizationUpdateRequestV2, UserAPIKeyAuth + from litellm.proxy.management_endpoints.organization_endpoints import update_organization_v2 + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", AsyncMock()) + + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin-1") + with pytest.raises(HTTPException) as exc: + await update_organization_v2( + organization_id="org-1", + data=OrganizationUpdateRequestV2.model_validate({"budget_duration": "bogus"}), + user_api_key_dict=auth, + ) + assert exc.value.status_code == 422 + assert "budget_duration" in str(exc.value.detail) + + @pytest.mark.asyncio async def test_v2_rejects_caller_without_org_access(monkeypatch): """v2 runs the real _verify_org_access guard: a non-admin without ORG_ADMIN on the org gets 403 and no write.""" From 3e4a884b25eeebd76de3f4e850b816f0b3bdc2fe Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 13:03:18 -0700 Subject: [PATCH 75/88] feat(organization): expose PATCH /v2/organization/{organization_id} in the OpenAPI schema --- .../management_endpoints/organization_endpoints.py | 1 - .../test_organization_endpoints.py | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py index 5e38a016099..35a1380a619 100644 --- a/litellm/proxy/management_endpoints/organization_endpoints.py +++ b/litellm/proxy/management_endpoints/organization_endpoints.py @@ -764,7 +764,6 @@ async def handle_update_object_permission( tags=["organization management"], dependencies=[Depends(user_api_key_auth)], response_model=LiteLLM_OrganizationTableWithMembers, - include_in_schema=False, ) async def update_organization_v2( organization_id: str, diff --git a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py index e2d89a660c2..9218cec4308 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py @@ -726,6 +726,20 @@ async def _run_update_organization_v2( return mock_prisma_client +def test_v2_update_route_is_public_in_openapi(): + """PATCH /v2/organization/{organization_id} is a public route: hiding it again (include_in_schema=False) + would drop it from openapi.json, /docs, and the generated UI API types.""" + from fastapi import FastAPI + + from litellm.proxy.management_endpoints.organization_endpoints import router + + app = FastAPI() + app.include_router(router) + v2_path = app.openapi()["paths"].get("/v2/organization/{organization_id}") + assert v2_path is not None + assert "patch" in v2_path + + @pytest.mark.asyncio async def test_v2_update_clears_tpm_limit_and_metadata(monkeypatch): """A cleared tpm_limit is written to the budget row as None; a cleared metadata is written as {}.""" From 0eb2363074bef6fd413c598fa0de20b7d11f10b2 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 13:10:37 -0700 Subject: [PATCH 76/88] test(organization): assert route publicity through the production OpenAPI generator --- .../management_endpoints/test_organization_endpoints.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py index 9218cec4308..6ce9e54cae0 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py @@ -729,13 +729,9 @@ async def _run_update_organization_v2( def test_v2_update_route_is_public_in_openapi(): """PATCH /v2/organization/{organization_id} is a public route: hiding it again (include_in_schema=False) would drop it from openapi.json, /docs, and the generated UI API types.""" - from fastapi import FastAPI + from litellm.proxy.proxy_server import get_openapi_schema - from litellm.proxy.management_endpoints.organization_endpoints import router - - app = FastAPI() - app.include_router(router) - v2_path = app.openapi()["paths"].get("/v2/organization/{organization_id}") + v2_path = get_openapi_schema()["paths"].get("/v2/organization/{organization_id}") assert v2_path is not None assert "patch" in v2_path From 50d6b26a86ea3c374bdd872a644ccdc5e61ceb47 Mon Sep 17 00:00:00 2001 From: mateo Date: Fri, 4 Sep 2026 20:10:58 +0000 Subject: [PATCH 77/88] fix(registry): mark baseten GLM-5.3 as vision-capable per Baseten vision docs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 6 ++++-- model_prices_and_context_window.json | 6 ++++-- tests/test_litellm/test_baseten_glm_5_3_model_metadata.py | 3 ++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 476fe4143c1..7f5038e3073 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -60810,7 +60810,8 @@ "output_cost_per_token": 4.4e-06, "source": "https://www.baseten.co/pricing/", "supported_modalities": [ - "text" + "text", + "image" ], "supported_output_modalities": [ "text" @@ -60818,7 +60819,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "openrouter/minimax/minimax-m3": { "input_cost_per_token": 3e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 476fe4143c1..7f5038e3073 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -60810,7 +60810,8 @@ "output_cost_per_token": 4.4e-06, "source": "https://www.baseten.co/pricing/", "supported_modalities": [ - "text" + "text", + "image" ], "supported_output_modalities": [ "text" @@ -60818,7 +60819,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "openrouter/minimax/minimax-m3": { "input_cost_per_token": 3e-07, diff --git a/tests/test_litellm/test_baseten_glm_5_3_model_metadata.py b/tests/test_litellm/test_baseten_glm_5_3_model_metadata.py index 98a8cf026ec..1dc17067d9f 100644 --- a/tests/test_litellm/test_baseten_glm_5_3_model_metadata.py +++ b/tests/test_litellm/test_baseten_glm_5_3_model_metadata.py @@ -54,7 +54,8 @@ def test_baseten_glm_5_3_specs(): assert info["supports_prompt_caching"] is True assert info["supports_response_schema"] is True assert info["supports_tool_choice"] is True - assert info["supported_modalities"] == ["text"] + assert info["supports_vision"] is True + assert info["supported_modalities"] == ["text", "image"] assert info["supported_output_modalities"] == ["text"] routed_model, provider, _, _ = get_llm_provider(model=MODEL) From 7a717740dd9b02785742b26be9e8feca01c151ec Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 13:11:31 -0700 Subject: [PATCH 78/88] test(organization): assert rejected values write nothing to the DB --- .../test_organization_endpoints.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py index f9fe4b8af5a..2422b2ae5aa 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py @@ -821,7 +821,8 @@ async def test_v2_rejects_negative_integer_limits(monkeypatch: pytest.MonkeyPatc from litellm.proxy._types import LitellmUserRoles, OrganizationUpdateRequestV2, UserAPIKeyAuth from litellm.proxy.management_endpoints.organization_endpoints import update_organization_v2 - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", AsyncMock()) + prisma_mock = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_mock) auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin-1") with pytest.raises(HTTPException) as exc: @@ -832,6 +833,9 @@ async def test_v2_rejects_negative_integer_limits(monkeypatch: pytest.MonkeyPatc ) assert exc.value.status_code == 422 assert field in str(exc.value.detail) + prisma_mock.db.tx.assert_not_called() + prisma_mock.db.litellm_budgettable.update.assert_not_awaited() + prisma_mock.db.litellm_organizationtable.update.assert_not_awaited() @pytest.mark.asyncio @@ -841,7 +845,8 @@ async def test_v2_rejects_unparseable_budget_duration(monkeypatch: pytest.Monkey from litellm.proxy._types import LitellmUserRoles, OrganizationUpdateRequestV2, UserAPIKeyAuth from litellm.proxy.management_endpoints.organization_endpoints import update_organization_v2 - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", AsyncMock()) + prisma_mock = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_mock) auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin-1") with pytest.raises(HTTPException) as exc: @@ -852,6 +857,9 @@ async def test_v2_rejects_unparseable_budget_duration(monkeypatch: pytest.Monkey ) assert exc.value.status_code == 422 assert "budget_duration" in str(exc.value.detail) + prisma_mock.db.tx.assert_not_called() + prisma_mock.db.litellm_budgettable.update.assert_not_awaited() + prisma_mock.db.litellm_organizationtable.update.assert_not_awaited() @pytest.mark.asyncio From 6234399f9e7b0e28eec0edb7269c5537f249f3c8 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 4 Sep 2026 13:16:34 -0700 Subject: [PATCH 79/88] fix(router): keep circuit-open fallbacks out of session pins An open classifier circuit routed through the ordinary heuristic or classifier_fallback path, and both causes are pin-worthy, so a session whose turn landed on the cooldown fallback held that model for the whole session_affinity TTL and never reclassified after the breaker closed. The circuit-open signal now blocks the pin, and _classifier_failure_outcome tags its outcomes through one helper instead of reassigning a Final. --- .../complexity_router/complexity_router.py | 41 +++++++++++-------- .../router_strategy/test_complexity_router.py | 39 ++++++++++++++++++ 2 files changed, 64 insertions(+), 16 deletions(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 2c1097b7af3..7dbb2ddc544 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -313,6 +313,8 @@ _TRUNCATION_MARKER: Final = "..." _TRUNCATION_HEAD_FRACTION: Final = 0.3 _MIN_QUOTED_TURN_CHARS: Final = 120 +_CLASSIFIER_CIRCUIT_OPEN_SIGNAL: Final = "classifier-circuit-open" + _CJK_CHARACTER: Final = re.compile("[぀-ヿㇰ-ㇿ㐀-䶿一-鿿豈-﫿ｦ-ﾝ\U00020000-\U0003ffff]") @@ -757,6 +759,12 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo image), not what the session's traffic looks like, and pinning it would hold every following text turn on the vision-capable model the image forced. A modality pin override is the same fact on a session that already holds a pin, so it must not overwrite the pin it displaced. + + An open classifier circuit is the shortest-lived state of all: the fallback ran because the + breaker skipped the classifier, not because the request got classified, and the cooldown is + seconds against a TTL of an hour that every later turn refreshes. Its cause is whatever the + fallback path reports, so the circuit signal is what marks the decision, and leaving it + unpinned lets the session classify again as soon as the breaker closes. """ return decision is None or ( decision.get("cause") @@ -768,6 +776,7 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo "modality_pin_override", ) and not decision.get("context_escalated") + and _CLASSIFIER_CIRCUIT_OPEN_SIGNAL not in (decision.get("signals") or ()) ) @@ -818,6 +827,10 @@ class ClassificationOutcome(NamedTuple): classifier_cost: float | None = None +def _with_signal(outcome: ClassificationOutcome, signal: str | None) -> ClassificationOutcome: + return outcome if signal is None else outcome._replace(signals=(*outcome.signals, signal)) + + class _ClassifierCircuitBreaker: """Process-local timeout breaker for one complexity-router classifier. @@ -1564,7 +1577,7 @@ class ComplexityRouter(CustomLogger): prompt, system_prompt, scored, - signal="classifier-circuit-open", + signal=_CLASSIFIER_CIRCUIT_OPEN_SIGNAL, ) try: tier, classifier_cost = await self._classify_with_llm(prompt, system_prompt, request_kwargs, messages) @@ -1602,28 +1615,24 @@ class ComplexityRouter(CustomLogger): fallback_tier: Final = self.config.fallback_tier if fallback_tier is not None: verbose_router_logger.warning("ComplexityRouter: %s, routing to fallback_tier %s", reason, fallback_tier) - outcome: Final = ClassificationOutcome( - tier=fallback_tier, - score=None, - signals=(f"classifier-fallback:{fallback_tier}",), - cause="classifier_fallback", + return _with_signal( + ClassificationOutcome( + tier=fallback_tier, + score=None, + signals=(f"classifier-fallback:{fallback_tier}",), + cause="classifier_fallback", + ), + signal, ) - return outcome if signal is None else outcome._replace(signals=(*outcome.signals, signal)) verbose_router_logger.warning( "ComplexityRouter: %s, falling back to %s", reason, self.config.classifier_fallback ) if self.config.classifier_fallback == "default_model": - outcome = self._default_model_fallback_outcome() - return outcome if signal is None else outcome._replace(signals=(*outcome.signals, signal)) + return _with_signal(self._default_model_fallback_outcome(), signal) if scored is not None: - return scored if signal is None else scored._replace(signals=(*scored.signals, signal)) + return _with_signal(scored, signal) tier, score, signals, cause = self._score_and_classify(prompt, system_prompt) - return ClassificationOutcome( - tier=tier, - score=score, - signals=signals if signal is None else (*signals, signal), - cause=cause, - ) + return _with_signal(ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause), signal) async def _classify_with_plugin( self, diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 130a6f5a488..b4c48b53376 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -4750,6 +4750,45 @@ class TestSessionAffinity: # Pinned to the first turn's model, not re-classified down to SIMPLE. assert second.model == "o1-preview" + @pytest.mark.asyncio + async def test_circuit_open_fallback_does_not_pin_the_session(self, mock_router_instance, session_affinity_config): + """Regression: the classifier circuit cools down in seconds while a pin lasts for the whole + TTL, so a session whose only turn landed on the cooldown fallback must classify again once + the breaker closes instead of holding that fallback's model.""" + now = 100.0 + mock_router_instance.cache = DualCache() + mock_router_instance.acompletion = AsyncMock( + side_effect=[TimeoutError("classifier timed out"), _llm_response('{"tier": "REASONING"}')] + ) + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + **session_affinity_config, + "classifier_type": "llm", + "classifier_llm_config": {"model": "haiku-classifier", "timeout_ms": 400}, + }, + ) + router._classifier_circuit_breaker = _ClassifierCircuitBreaker(30.0, clock=lambda: now) + + await router.async_pre_routing_hook( + model="test-model", + request_kwargs=self._request_kwargs("outage-session"), + messages=self.SIMPLE_MESSAGE, + ) + cooled_down_kwargs = self._request_kwargs("cooldown-session") + during_cooldown = await router.async_pre_routing_hook( + model="test-model", request_kwargs=cooled_down_kwargs, messages=self.SIMPLE_MESSAGE + ) + now = 130.0 + after_cooldown = await router.async_pre_routing_hook( + model="test-model", request_kwargs=cooled_down_kwargs, messages=self.SIMPLE_MESSAGE + ) + + assert during_cooldown.model == "gpt-4o-mini" + assert after_cooldown.model == "o1-preview" + assert mock_router_instance.acompletion.await_count == 2 + @pytest.mark.asyncio async def test_a_pinned_turn_reports_the_tier_that_serves_it(self, mock_router_instance, session_affinity_config): mock_router_instance.cache = DualCache() From 8beca1d58d4fa28020b16d672b2b7dc0b8e88ce3 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Fri, 4 Sep 2026 13:33:06 -0700 Subject: [PATCH 80/88] fix(auto-router): route 1M complex tier to GPT Sol (#39797) * feat(ui): add 1M context auto-router preset * feat(ui): use heuristic v2 for 1M preset * fix(ui): keep 1M preset test within lint budget * fix(auto-router): route 1M complex tier to GPT Sol * test(auto-router): update 1M complex tier expectation --- litellm/proxy/public_endpoints/autorouter_presets.json | 4 ++-- .../proxy/public_endpoints/test_public_endpoints.py | 2 +- ui/litellm-dashboard/src/lib/autorouter_presets.test.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/public_endpoints/autorouter_presets.json b/litellm/proxy/public_endpoints/autorouter_presets.json index 6a6642d4211..7d09db31127 100644 --- a/litellm/proxy/public_endpoints/autorouter_presets.json +++ b/litellm/proxy/public_endpoints/autorouter_presets.json @@ -1,12 +1,12 @@ { "1m_context": { "label": "1M Context", - "description": "Routes across models with 1M-token context windows: Luna for simple queries, Terra for medium, Opus 5 for complex, Opus 5 at high thinking for reasoning.", + "description": "Routes across models with 1M-token context windows: Luna for simple queries, Terra for medium, Sol for complex, Opus 5 at high thinking for reasoning.", "complexity_router_config": { "tiers": { "SIMPLE": ["gpt-5.6-luna"], "MEDIUM": ["gpt-5.6-terra"], - "COMPLEX": ["claude-opus-5"], + "COMPLEX": ["gpt-5.6-sol"], "REASONING": ["claude-opus-5"] }, "tier_model_configs": { diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 41439f28638..4a19ad3541c 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -1108,7 +1108,7 @@ def test_get_autorouter_presets_local_mode_serves_bundled_catalog( assert payload["1m_context"]["complexity_router_config"]["tiers"] == { "SIMPLE": ["gpt-5.6-luna"], "MEDIUM": ["gpt-5.6-terra"], - "COMPLEX": ["claude-opus-5"], + "COMPLEX": ["gpt-5.6-sol"], "REASONING": ["claude-opus-5"], } assert payload["1m_context"]["complexity_router_config"]["tier_model_configs"] == { diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index ffede7b2a6b..344964c4434 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -217,12 +217,12 @@ describe("autorouter_presets", () => { }); }); - it("pins the 1M context preset to Luna, Terra, and Opus at high thinking", () => { + it("pins the 1M context preset to Luna, Terra, Sol, and Opus at high thinking", () => { const preset = getPresetByKey("1m_context")!; const expectedTiers = { SIMPLE: ["gpt-5.6-luna"], MEDIUM: ["gpt-5.6-terra"], - COMPLEX: ["claude-opus-5"], + COMPLEX: ["gpt-5.6-sol"], REASONING: ["claude-opus-5"], }; expect(preset.complexity_router_config.classifier_type).toBe("heuristic_v2"); From 6f18a4d81ef4b7f2efd0d61500bcca29fdbe5a3e Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 14:06:34 -0700 Subject: [PATCH 81/88] fix(ui): accept any routing group name the backend accepts The create form rejected names with slashes or spaces even though the proxy stores and routes any non-empty string. Drop the client-only character pattern and trim the name before the required check so a whitespace-only name is still refused Claude-Session: https://claude.ai/code/session_01HkaXiD6gssHnx3kqu1rR8C --- .../routing_groups/RoutingGroupModal.test.tsx | 25 ++++++++++++++++--- .../routing_groups/RoutingGroupModal.tsx | 5 ++-- .../routing_groups/routingGroupPayload.ts | 1 - 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.test.tsx b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.test.tsx index 3699a57a657..322d90b24c5 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.test.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.test.tsx @@ -19,6 +19,13 @@ const EXPECTED_STORED_PAYLOAD: RoutingGroup = { const SEEDED_CREATE: RoutingGroup = { group_name: "", models: ["gemini-pro"], routing_strategy: "simple-shuffle" }; +const EXPECTED_SLASH_AND_SPACE_PAYLOAD: RoutingGroup = { + group_name: "team a/fast chat", + models: ["gemini-pro"], + routing_strategy: "simple-shuffle", + routing_strategy_args: null, +}; + const STORED_GROUP: RoutingGroup = { group_name: "already-taken", models: ["gpt-4o", "claude-sonnet"], @@ -211,16 +218,28 @@ describe("RoutingGroupModal", () => { expect(onSubmit).not.toHaveBeenCalled(); }); - it("rejects a name with characters outside the allowed set", async () => { + it("accepts a name with slashes and spaces, since the backend does", async () => { const user = userEvent.setup(); const { onSubmit } = renderModal({ initialValue: { group_name: "", models: ["gemini-pro"], routing_strategy: "simple-shuffle" }, }); - await typeName(user, "bad name"); + await typeName(user, "team a/fast chat"); await save(user, "Create Group"); - expect(await screen.findByText("Only letters, numbers, dot, underscore, and dash are allowed")).toBeInTheDocument(); + expect(onSubmit).toHaveBeenCalledWith(EXPECTED_SLASH_AND_SPACE_PAYLOAD); + }); + + it("rejects a whitespace-only name as missing", async () => { + const user = userEvent.setup(); + const { onSubmit } = renderModal({ + initialValue: { group_name: "", models: ["gemini-pro"], routing_strategy: "simple-shuffle" }, + }); + + await typeName(user, " "); + await save(user, "Create Group"); + + expect(await screen.findByText("Group name is required")).toBeInTheDocument(); expect(onSubmit).not.toHaveBeenCalled(); }); diff --git a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.tsx b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.tsx index 442275c32f6..5865c59d8bd 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.tsx @@ -23,7 +23,6 @@ import { Textarea } from "@/components/ui/textarea"; import { useZodForm } from "@/lib/forms/useZodForm"; import { GROUP_NAME_MAX_LENGTH, - GROUP_NAME_PATTERN, STRATEGIES_WITH_ARGS, argsForStrategy, buildRoutingGroupPayload, @@ -74,10 +73,10 @@ const RoutingGroupModal: React.FC = ({ const shape = { group_name: z .string() + .trim() .min(1, "Group name is required") .max(GROUP_NAME_MAX_LENGTH, `Must be ${GROUP_NAME_MAX_LENGTH} characters or fewer`) - .regex(GROUP_NAME_PATTERN, "Only letters, numbers, dot, underscore, and dash are allowed") - .refine((value) => !reservedNames.has(value.trim().toLowerCase()), "A group with this name already exists"), + .refine((value) => !reservedNames.has(value.toLowerCase()), "A group with this name already exists"), models: z.array(z.string()).min(1, "Select at least one model"), routing_strategy: z.string().min(1, "Strategy is required"), routing_strategy_args: z.string(), diff --git a/ui/litellm-dashboard/src/components/routing_groups/routingGroupPayload.ts b/ui/litellm-dashboard/src/components/routing_groups/routingGroupPayload.ts index ddc24938ea7..68f06356262 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/routingGroupPayload.ts +++ b/ui/litellm-dashboard/src/components/routing_groups/routingGroupPayload.ts @@ -2,7 +2,6 @@ import type { RoutingGroup } from "./types"; export const STRATEGIES_WITH_ARGS = new Set(["latency-based-routing", "usage-based-routing"]); -export const GROUP_NAME_PATTERN = /^[A-Za-z0-9._-]+$/; export const GROUP_NAME_MAX_LENGTH = 64; export interface RoutingGroupFormValues { From 1548be8235817946b7f8221a66180214721b5eaf Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 14:09:59 -0700 Subject: [PATCH 82/88] feat(guardrails): report the usage units a guardrail's cost leaves out A row's cost sums only the daily rows that carry a tracked cost, so it silently under-reports whenever some rows are NULL (pre-migration days, old pods mid-rollout, an unpriced counter). Both usage endpoints now return the per-counter units behind those NULL rows next to the cost (untrackedUsageUnits / totalUntrackedUsageUnits on the overview, untracked_usage_units on the detail), so a partial cost is never mistaken for a complete one and the reader can see exactly what it excludes Claude-Session: https://claude.ai/code/session_01EX13mWex6RaBo9PYnkAtFW --- litellm/proxy/_lazy_openapi_snapshot.json | 54 +++++++++++++++++-- litellm/proxy/guardrails/usage_endpoints.py | 52 ++++++++++++++---- .../proxy/guardrails/test_usage_endpoints.py | 37 +++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 20 ++++++- 4 files changed, 147 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 5385b4d6f7e..c399e5594f6 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -13142,6 +13142,13 @@ "title": "Type", "type": "string" }, + "untracked_usage_units": { + "additionalProperties": { + "type": "integer" + }, + "title": "Untracked Usage Units", + "type": "object" + }, "usage_units": { "additionalProperties": { "type": "integer" @@ -13197,7 +13204,8 @@ "cost", "cost_by_unit", "cost_by_team", - "cost_by_key" + "cost_by_key", + "untracked_usage_units" ], "title": "UsageDetailResponse", "type": "object" @@ -13367,6 +13375,13 @@ "title": "Totalrequests", "type": "integer" }, + "totalUntrackedUsageUnits": { + "additionalProperties": { + "type": "integer" + }, + "title": "Totaluntrackedusageunits", + "type": "object" + }, "totalUsageUnits": { "additionalProperties": { "type": "integer" @@ -13382,7 +13397,8 @@ "totalBlocked", "passRate", "totalUsageUnits", - "totalCost" + "totalCost", + "totalUntrackedUsageUnits" ], "title": "UsageOverviewResponse", "type": "object" @@ -13420,6 +13436,7 @@ "type": "null" } ], + "description": "USD billed for usageUnits over the window, summed over days with tracked cost; null when none have it", "title": "Cost" }, "failRate": { @@ -13454,6 +13471,14 @@ "title": "Type", "type": "string" }, + "untrackedUsageUnits": { + "additionalProperties": { + "type": "integer" + }, + "description": "The share of usageUnits that cost leaves out: units from days with no tracked cost, per counter", + "title": "Untrackedusageunits", + "type": "object" + }, "usageUnits": { "additionalProperties": { "type": "integer" @@ -13474,7 +13499,8 @@ "status", "trend", "usageUnits", - "cost" + "cost", + "untrackedUsageUnits" ], "title": "UsageOverviewRow", "type": "object" @@ -28881,6 +28907,13 @@ "title": "Totalrequests", "type": "integer" }, + "totalUntrackedUsageUnits": { + "additionalProperties": { + "type": "integer" + }, + "title": "Totaluntrackedusageunits", + "type": "object" + }, "totalUsageUnits": { "additionalProperties": { "type": "integer" @@ -28896,7 +28929,8 @@ "totalBlocked", "passRate", "totalUsageUnits", - "totalCost" + "totalCost", + "totalUntrackedUsageUnits" ], "title": "UsageOverviewResponse", "type": "object" @@ -28934,6 +28968,7 @@ "type": "null" } ], + "description": "USD billed for usageUnits over the window, summed over days with tracked cost; null when none have it", "title": "Cost" }, "failRate": { @@ -28968,6 +29003,14 @@ "title": "Type", "type": "string" }, + "untrackedUsageUnits": { + "additionalProperties": { + "type": "integer" + }, + "description": "The share of usageUnits that cost leaves out: units from days with no tracked cost, per counter", + "title": "Untrackedusageunits", + "type": "object" + }, "usageUnits": { "additionalProperties": { "type": "integer" @@ -28988,7 +29031,8 @@ "status", "trend", "usageUnits", - "cost" + "cost", + "untrackedUsageUnits" ], "title": "UsageOverviewRow", "type": "object" diff --git a/litellm/proxy/guardrails/usage_endpoints.py b/litellm/proxy/guardrails/usage_endpoints.py index 69516487d7c..523efe0da75 100644 --- a/litellm/proxy/guardrails/usage_endpoints.py +++ b/litellm/proxy/guardrails/usage_endpoints.py @@ -8,10 +8,10 @@ from collections.abc import Callable, Iterable, Mapping, Sequence from datetime import date, datetime, timedelta, timezone from itertools import groupby from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, overload +from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, overload from fastapi import APIRouter, Depends, Query -from pydantic import BaseModel +from pydantic import BaseModel, Field from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger @@ -42,6 +42,8 @@ router: Final = APIRouter() _EMPTY_UNITS: Final[Mapping[str, int]] = MappingProxyType({}) +_T = TypeVar("_T") + _USAGE_MAX_RANGE_DAYS: Final = 366 @@ -183,6 +185,17 @@ def _cost_by( return MappingProxyType({key: _sum_tracked_cost(group) for key, group in groupby(ordered, key=key_of)}) +def _untracked_rows( + rows: "Sequence[prisma_models.LiteLLM_DailyGuardrailUsageUnits]", +) -> "tuple[prisma_models.LiteLLM_DailyGuardrailUsageUnits, ...]": + """Rows whose cost is unknown, so their units are exactly what the tracked cost sums leave out.""" + return tuple(r for r in rows if r.cost is None) + + +def _first_match(lookup_keys: Sequence[str], mapping: Mapping[str, _T], default: _T) -> _T: + return next((mapping[k] for k in lookup_keys if k in mapping), default) + + # --- Response models --- @@ -232,8 +245,12 @@ class UsageOverviewRow(BaseModel): status: str # healthy | warning | critical trend: str # up | down | stable usageUnits: Mapping[str, int] - cost: float | None - """USD billed for usageUnits over the window, summed over days with tracked cost; null when none have it.""" + cost: float | None = Field( + description="USD billed for usageUnits over the window, summed over days with tracked cost; null when none have it" + ) + untrackedUsageUnits: Mapping[str, int] = Field( + description="The share of usageUnits that cost leaves out: units from days with no tracked cost, per counter" + ) class UsageOverviewResponse(BaseModel): @@ -244,10 +261,18 @@ class UsageOverviewResponse(BaseModel): passRate: float totalUsageUnits: Mapping[str, int] totalCost: float | None + totalUntrackedUsageUnits: Mapping[str, int] _EMPTY_OVERVIEW: Final = UsageOverviewResponse( - rows=[], chart=[], totalRequests=0, totalBlocked=0, passRate=100.0, totalUsageUnits=_EMPTY_UNITS, totalCost=None + rows=[], + chart=[], + totalRequests=0, + totalBlocked=0, + passRate=100.0, + totalUsageUnits=_EMPTY_UNITS, + totalCost=None, + totalUntrackedUsageUnits=_EMPTY_UNITS, ) @@ -278,6 +303,7 @@ class UsageDetailResponse(BaseModel): cost_by_unit: Mapping[str, float | None] cost_by_team: Mapping[str, float | None] cost_by_key: Mapping[str, float | None] + untracked_usage_units: Mapping[str, int] class UsageLogEntry(BaseModel): @@ -395,6 +421,7 @@ def _guardrail_overview_rows( prev_agg: Mapping[str, float], units_agg: Mapping[str, Mapping[str, int]], cost_agg: Mapping[str, float | None], + untracked_agg: Mapping[str, Mapping[str, int]], ) -> list[UsageOverviewRow]: rows: Final[list[UsageOverviewRow]] = [] covered_keys: Final[set[str]] = set() @@ -420,8 +447,6 @@ def _guardrail_overview_rows( prev_fail = float(prev_agg.get(k, 0.0) or 0.0) break trend = _trend_from_comparison(fail_rate, prev_fail) - row_units: Mapping[str, int] = next((units_agg[k] for k in lookup_keys if k in units_agg), _EMPTY_UNITS) - row_cost: float | None = next((cost_agg[k] for k in lookup_keys if k in cost_agg), None) rows.append( UsageOverviewRow( id=gid, @@ -434,8 +459,9 @@ def _guardrail_overview_rows( avgLatency=None, status=_status_from_fail_rate(fail_rate), trend=trend, - usageUnits=row_units, - cost=row_cost, + usageUnits=_first_match(lookup_keys, units_agg, _EMPTY_UNITS), + cost=_first_match(lookup_keys, cost_agg, None), + untrackedUsageUnits=_first_match(lookup_keys, untracked_agg, _EMPTY_UNITS), ) ) # Add rows for guardrails with metrics but not in guardrails table (e.g. MCP, config) @@ -460,6 +486,7 @@ def _guardrail_overview_rows( trend=trend, usageUnits=units_agg.get(agg_key, _EMPTY_UNITS), cost=cost_agg.get(agg_key), + untrackedUsageUnits=untracked_agg.get(agg_key, _EMPTY_UNITS), ) ) return rows @@ -491,6 +518,7 @@ def _policy_overview_rows( trend=trend, usageUnits=_EMPTY_UNITS, cost=None, + untrackedUsageUnits=_EMPTY_UNITS, ) ) return rows @@ -545,13 +573,15 @@ async def guardrails_usage_overview( agg: Final = _aggregate_daily_metrics(metrics, "guardrail_id") prev_agg: Final = _prev_fail_rates(metrics_prev, "guardrail_id") + untracked_rows: Final = _untracked_rows(units_rows) units_agg: Final = _units_by(units_rows, lambda r: r.guardrail_id) cost_agg: Final = _cost_by(units_rows, lambda r: r.guardrail_id) + untracked_agg: Final = _units_by(untracked_rows, lambda r: r.guardrail_id) chart: Final = _chart_from_metrics(metrics) total_requests: Final = sum(a["requests"] for a in agg.values()) total_blocked: Final = sum(a["blocked"] for a in agg.values()) pass_rate: Final = (100.0 * (total_requests - total_blocked) / total_requests) if total_requests else 100.0 - rows: Final = _guardrail_overview_rows(guardrails, agg, prev_agg, units_agg, cost_agg) + rows: Final = _guardrail_overview_rows(guardrails, agg, prev_agg, units_agg, cost_agg, untracked_agg) return UsageOverviewResponse( rows=rows, chart=chart, @@ -560,6 +590,7 @@ async def guardrails_usage_overview( passRate=round(pass_rate, 1), totalUsageUnits=_sum_counter_units(units_rows), totalCost=_sum_tracked_cost(units_rows), + totalUntrackedUsageUnits=_sum_counter_units(untracked_rows), ) except Exception as e: from litellm.proxy.utils import handle_exception_on_proxy @@ -677,6 +708,7 @@ async def guardrails_usage_detail( cost_by_unit=_cost_by(units_rows, _counter_name), cost_by_team=_cost_by(units_rows, lambda r: r.team_id), cost_by_key=_cost_by(units_rows, lambda r: r.api_key), + untracked_usage_units=_sum_counter_units(_untracked_rows(units_rows)), ) diff --git a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py index b8455b01e35..4a11c589810 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py @@ -314,6 +314,7 @@ async def test_overview_degrades_units_to_empty_when_units_table_is_missing(): assert (row.requestsEvaluated, row.usageUnits) == (4, {}) assert (resp.totalRequests, resp.totalBlocked, resp.totalUsageUnits) == (4, 1, {}) assert (row.cost, resp.totalCost) == (None, None) + assert (row.untrackedUsageUnits, resp.totalUntrackedUsageUnits) == ({}, {}) @pytest.mark.asyncio @@ -344,6 +345,40 @@ async def test_overview_reports_cost_per_row_and_total_summing_only_tracked_days assert resp.totalCost == pytest.approx(0.45) +@pytest.mark.asyncio +async def test_overview_reports_the_units_its_cost_leaves_out_per_row_and_total(): + """A row's cost silently under-reports whenever some of its days carry NULL, so + the response must say exactly which units (per counter) that cost excludes. + A guardrail whose rows are all priced reports none; one with only NULL rows + reports all of its units; a mix reports just the NULL rows' units.""" + prisma = _prisma( + find_many=[], + metrics=[_metric("yaml-pii", requests=4, passed=3, blocked=1)], + units=[ + _units_row("yaml-pii", usage_unit="contentPolicyUnits", units=1000, cost=0.15), + _units_row("yaml-pii", date="2026-04-24", usage_unit="contentPolicyUnits", units=5000, cost=None), + _units_row("yaml-pii", date="2026-04-24", usage_unit="topicPolicyUnits", units=40, cost=None), + _units_row("yaml-pii", usage_unit="wordPolicyUnits", units=9, cost=0.0), + _units_row("legacy-guard", usage_unit="topicPolicyUnits", units=7, cost=None), + _units_row("priced-guard", usage_unit="contentPolicyUnits", units=3, cost=0.0003), + ], + ) + handler = _config_handler( + _yaml_guardrail(guardrail_id="yaml-uuid", name="yaml-pii"), + _yaml_guardrail(guardrail_id="legacy-uuid", name="legacy-guard"), + _yaml_guardrail(guardrail_id="priced-uuid", name="priced-guard"), + ) + p1, p2 = _patches(prisma, handler) + with p1, p2: + resp = await guardrails_usage_overview(start_date=START, end_date=END, user_api_key_dict=ADMIN) + by_id = {r.id: r for r in resp.rows} + assert by_id["yaml-uuid"].usageUnits == {"contentPolicyUnits": 6000, "topicPolicyUnits": 40, "wordPolicyUnits": 9} + assert by_id["yaml-uuid"].untrackedUsageUnits == {"contentPolicyUnits": 5000, "topicPolicyUnits": 40} + assert by_id["legacy-uuid"].untrackedUsageUnits == {"topicPolicyUnits": 7} + assert by_id["priced-uuid"].untrackedUsageUnits == {} + assert resp.totalUntrackedUsageUnits == {"contentPolicyUnits": 5000, "topicPolicyUnits": 47} + + @pytest.mark.asyncio async def test_detail_breaks_cost_down_by_unit_day_team_and_key(): """Every cost breakdown keeps the same keys as its units twin so the UI can @@ -380,6 +415,7 @@ async def test_detail_breaks_cost_down_by_unit_day_team_and_key(): assert resp.cost_by_key == {"hash-1": pytest.approx(0.15), "hash-2": pytest.approx(0.03)} assert resp.cost_by_team.keys() == resp.usage_units_by_team.keys() assert resp.cost_by_key.keys() == resp.usage_units_by_key.keys() + assert resp.untracked_usage_units == {"topicPolicyUnits": 10} @pytest.mark.asyncio @@ -400,6 +436,7 @@ async def test_detail_degrades_units_to_empty_when_units_table_is_missing(): {}, ) assert (resp.cost, resp.cost_by_unit, resp.cost_by_team, resp.cost_by_key) == (None, {}, {}, {}) + assert resp.untracked_usage_units == {} # ---- logs ------------------------------------------------------------------- diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a3d8b22a672..b21bb523aa5 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -37825,6 +37825,10 @@ export interface components { trend: string; /** Type */ type: string; + /** Untracked Usage Units */ + untracked_usage_units: { + [key: string]: number; + }; /** Usage Units */ usage_units: { [key: string]: number; @@ -37890,6 +37894,10 @@ export interface components { totalCost: number | null; /** Totalrequests */ totalRequests: number; + /** Totaluntrackedusageunits */ + totalUntrackedUsageUnits: { + [key: string]: number; + }; /** Totalusageunits */ totalUsageUnits: { [key: string]: number; @@ -37901,7 +37909,10 @@ export interface components { avgLatency: number | null; /** Avgscore */ avgScore: number | null; - /** Cost */ + /** + * Cost + * @description USD billed for usageUnits over the window, summed over days with tracked cost; null when none have it + */ cost: number | null; /** Failrate */ failRate: number; @@ -37919,6 +37930,13 @@ export interface components { trend: string; /** Type */ type: string; + /** + * Untrackedusageunits + * @description The share of usageUnits that cost leaves out: units from days with no tracked cost, per counter + */ + untrackedUsageUnits: { + [key: string]: number; + }; /** Usageunits */ usageUnits: { [key: string]: number; From 2f7ee39545ad66c527c5827b4f58d5cc9c1dcfdf Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 14:11:31 -0700 Subject: [PATCH 83/88] fix(jwt): invalidate JWT key mapping cache on /key/regenerate /key/regenerate carries the JWT-to-key mapping to the new token via FK cascade, but the jwt_key_mapping cache entry kept resolving the old (now invalid) token for up to virtual_key_mapping_cache_ttl. Snapshot the key's mapping cache keys before the token update and evict them with evict_and_broadcast so every worker drops the stale entry. Also share the cache-key format through jwt_key_mapping_cache_key and upgrade the /jwt/key/mapping CRUD endpoints from local-only deletes to evict_and_broadcast, closing the same cross-worker staleness there. --- litellm/proxy/auth/auth_checks.py | 19 ++++ litellm/proxy/auth/user_api_key_auth.py | 3 +- .../jwt_key_mapping_endpoints.py | 22 ++-- .../key_management_endpoints.py | 11 ++ .../test_key_management_endpoints.py | 103 ++++++++++++++++++ 5 files changed, 147 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 2b328b455dc..f83f0303deb 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -142,6 +142,8 @@ class _PrismaDictableRow(Protocol): class _PrismaJWTKeyMappingRow(Protocol): token: str + jwt_claim_name: str + jwt_claim_value: str class _PrismaModelDumpRow(Protocol): @@ -3466,6 +3468,23 @@ async def _fetch_key_object_from_db_with_reconnect( raise +def jwt_key_mapping_cache_key(jwt_claim_name: str, jwt_claim_value: str) -> str: + """Cache key under which ``_resolve_jwt_to_virtual_key`` stores a JWT-claim-to-key mapping.""" + return f"jwt_key_mapping:{jwt_claim_name}:{jwt_claim_value}" + + +@log_db_metrics +async def get_jwt_key_mapping_cache_keys_for_token( + hashed_token: str, + prisma_client: PrismaClient, +) -> tuple[str, ...]: + """Cache keys of every JWT claim mapped to the given virtual key.""" + mappings: Final = await _jwt_key_mapping_table(JWTKeyMappingRepository(prisma_client)).find_many( + where={"token": hashed_token} + ) + return tuple(jwt_key_mapping_cache_key(m.jwt_claim_name, m.jwt_claim_value) for m in mappings) + + @log_db_metrics async def get_jwt_key_mapping_object( jwt_claim_name: str, diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 5fb6dad0cd7..93293db24c6 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -58,6 +58,7 @@ from litellm.proxy.auth.auth_checks import ( get_team_object, get_user_object, is_valid_fallback_model, + jwt_key_mapping_cache_key, resolve_and_validate_end_user_id, ) from litellm.proxy.auth.auth_exception_handler import UserAPIKeyAuthExceptionHandler @@ -970,7 +971,7 @@ async def _resolve_jwt_to_virtual_key( ) return None - cache_key: Final = f"jwt_key_mapping:{virtual_key_claim_field}:{claim_value}" + cache_key: Final = jwt_key_mapping_cache_key(virtual_key_claim_field, str(claim_value)) cached_mapping: Final = await user_api_key_cache.async_get_cache(cache_key) if cached_mapping == _JWT_PROXY_ADMIN_SENTINEL: diff --git a/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py b/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py index ccfd5338ec4..4f6468e911f 100644 --- a/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py +++ b/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py @@ -13,7 +13,9 @@ from litellm.proxy._types import ( UserAPIKeyAuth, hash_token, ) +from litellm.proxy.auth.auth_checks import jwt_key_mapping_cache_key from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view from litellm.repositories.table_repositories import JWTKeyMappingRepository @@ -118,9 +120,8 @@ async def create_jwt_key_mapping( new_mapping: Final = await _mapping_table(prisma_client).create(data=create_data) - # Invalidate cache - cache_key: Final = f"jwt_key_mapping:{data.jwt_claim_name}:{data.jwt_claim_value}" - await user_api_key_cache.async_delete_cache(cache_key) + cache_key: Final = jwt_key_mapping_cache_key(data.jwt_claim_name, data.jwt_claim_value) + await evict_and_broadcast(cache_keys=(cache_key,), user_api_key_cache=user_api_key_cache) return _to_response(new_mapping) except HTTPException: @@ -169,17 +170,18 @@ async def update_jwt_key_mapping( if old_mapping is None: raise HTTPException(status_code=404, detail="Mapping not found") - cache_key = f"jwt_key_mapping:{old_mapping.jwt_claim_name}:{old_mapping.jwt_claim_value}" - await user_api_key_cache.async_delete_cache(cache_key) + old_cache_key: Final = jwt_key_mapping_cache_key(old_mapping.jwt_claim_name, old_mapping.jwt_claim_value) + await evict_and_broadcast(cache_keys=(old_cache_key,), user_api_key_cache=user_api_key_cache) updated_mapping: Final = await _mapping_table(prisma_client).update(where={"id": data.id}, data=update_data) if updated_mapping is None: raise HTTPException(status_code=404, detail="Mapping not found") - # Invalidate new cache key if claim fields changed - cache_key = f"jwt_key_mapping:{updated_mapping.jwt_claim_name}:{updated_mapping.jwt_claim_value}" - await user_api_key_cache.async_delete_cache(cache_key) + new_cache_key: Final = jwt_key_mapping_cache_key( + updated_mapping.jwt_claim_name, updated_mapping.jwt_claim_value + ) + await evict_and_broadcast(cache_keys=(new_cache_key,), user_api_key_cache=user_api_key_cache) return _to_response(updated_mapping) except HTTPException: @@ -219,8 +221,8 @@ async def delete_jwt_key_mapping( if old_mapping is None: raise HTTPException(status_code=404, detail="Mapping not found") - cache_key: Final = f"jwt_key_mapping:{old_mapping.jwt_claim_name}:{old_mapping.jwt_claim_value}" - await user_api_key_cache.async_delete_cache(cache_key) + cache_key: Final = jwt_key_mapping_cache_key(old_mapping.jwt_claim_name, old_mapping.jwt_claim_value) + await evict_and_broadcast(cache_keys=(cache_key,), user_api_key_cache=user_api_key_cache) await _mapping_table(prisma_client).delete(where={"id": data.id}) return {"status": "success"} diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 324d380b85b..ebcfab090b5 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -54,6 +54,7 @@ from litellm.proxy._types import Litellm_EntityType, LiteLLM_VerificationToken, from litellm.proxy.auth.auth_checks import ( _delete_cache_key_object, can_team_access_model, + get_jwt_key_mapping_cache_keys_for_token, get_org_object, get_project_object, get_team_object, @@ -65,6 +66,7 @@ from litellm.proxy.auth.auth_utils import ( ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( + evict_and_broadcast, publish_auth_cache_invalidation, ) from litellm.proxy.common_utils.callback_config_validation import logging_metadata_config_error @@ -4975,6 +4977,13 @@ async def _execute_virtual_key_regeneration( update_data.update(non_default_values) jsonified_update_data: Final[Mapping[str, object]] = prisma_client.jsonify_object(data=update_data) + # Snapshot before the token update: the FK cascade rewrites mapping rows to the new hash, + # but their cached jwt_key_mapping entries still point at the old token (LIT-5379). + jwt_mapping_cache_keys: Final = await get_jwt_key_mapping_cache_keys_for_token( + hashed_token=hashed_api_key, + prisma_client=prisma_client, + ) + # If grace period set, insert deprecated key so old key remains valid await _insert_deprecated_key( prisma_client=prisma_client, @@ -5000,6 +5009,8 @@ async def _execute_virtual_key_regeneration( proxy_logging_obj=proxy_logging_obj, ) + await evict_and_broadcast(cache_keys=jwt_mapping_cache_keys, user_api_key_cache=user_api_key_cache) + # After credential invalidation, so a failure here can never keep the old key alive. await sync_key_regeneration_access_group_membership( prisma_client=prisma_client, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 0e4af9f75a5..293e966f051 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -11912,6 +11912,109 @@ async def test_execute_virtual_key_regeneration_allows_within_limit_duration(mon assert mock_prisma_client.db.litellm_verificationtoken.update.await_count == 1 +@pytest.mark.asyncio +async def test_regenerate_evicts_jwt_key_mapping_cache_so_next_jwt_call_gets_new_token(): + """ + LIT-5379: /key/regenerate rewrites the JWT mapping row to the new token (FK + cascade) but left the jwt_key_mapping cache entry pointing at the old hash, + so JWT calls kept resolving the dead token until the cache TTL expired. + Regenerate must evict the entry locally, broadcast the eviction to other + workers, and the very next JWT resolve must return the rotated token. + """ + from litellm.caching.caching import DualCache + from litellm.proxy._types import LiteLLM_JWTAuth + from litellm.proxy.auth.auth_method import AuthMethod + from litellm.proxy.auth.resolvers.models import CredentialRef + from litellm.proxy.auth.resolvers.store import IdentityStore + from litellm.proxy.auth.user_api_key_auth import _resolve_jwt_to_virtual_key + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _execute_virtual_key_regeneration, + ) + + stale_cache_key = "jwt_key_mapping:sub:user1" + existing_key = _make_regenerate_existing_key() + mock_prisma_client = _make_regenerate_mock_prisma() + mock_prisma_client.db.litellm_jwtkeymapping.find_many = AsyncMock( + return_value=[MagicMock(jwt_claim_name="sub", jwt_claim_value="user1")] + ) + mock_prisma_client.db.litellm_jwtkeymapping.find_first = AsyncMock( + return_value=MagicMock(token="new-hashed-token") + ) + user_api_key_cache = DualCache() + await user_api_key_cache.async_set_cache(key=stale_cache_key, value="abc123") + + publish_mock = AsyncMock() + with ( + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.get_new_token", + new_callable=AsyncMock, + return_value="sk-newtoken1234ab12", + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._insert_deprecated_key", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_rotated_hook", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.common_utils.auth_cache_invalidation_pubsub.publish_auth_cache_invalidation", + publish_mock, + ), + ): + await _execute_virtual_key_regeneration( + prisma_client=mock_prisma_client, + key_in_db=existing_key, + hashed_api_key="abc123", + key="abc123", + data=None, + user_api_key_dict=_make_regenerate_user_api_key_dict(), + litellm_changed_by=None, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=MagicMock(), + ) + + assert await user_api_key_cache.async_get_cache(stale_cache_key) is None + publish_mock.assert_any_await(cache_key=stale_cache_key) + mock_prisma_client.db.litellm_jwtkeymapping.find_many.assert_awaited_once_with(where={"token": "abc123"}) + + rotated_key = UserAPIKeyAuth(token="new-hashed-token", user_id="user-1") + rotated_principal = IdentityStore._principal_from_key( + rotated_key, + auth_method=AuthMethod.API_KEY, + credential_ref=CredentialRef(token_id="new-hashed-token"), + ) + + async def fake_resolve(hashed_token): + assert hashed_token == "new-hashed-token", f"JWT resolved stale token {hashed_token!r} after regenerate" + return rotated_principal + + jwt_handler = MagicMock() + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( + virtual_key_claim_field="sub", virtual_key_mapping_cache_ttl=300 + ) + with patch( + "litellm.proxy.auth.resolvers.store.IdentityStore.resolve", + new_callable=AsyncMock, + side_effect=fake_resolve, + ): + resolved = await _resolve_jwt_to_virtual_key( + jwt_claims={"sub": "user1"}, + jwt_handler=jwt_handler, + prisma_client=mock_prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + assert isinstance(resolved, UserAPIKeyAuth) + assert resolved.token == "new-hashed-token" + + @pytest.mark.asyncio async def test_execute_virtual_key_regeneration_rejects_over_limit_max_budget(monkeypatch): """Regenerate must reject max_budget exceeding upperbound — proves the fix covers non-duration fields.""" From 52b746e8eab8af08c35e4b109152848c723ebba5 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 14:18:09 -0700 Subject: [PATCH 84/88] test(key): annotate regenerate JWT mapping test patches for TQ008 --- .../test_key_management_endpoints.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 293e966f051..47571497f74 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -11945,24 +11945,24 @@ async def test_regenerate_evicts_jwt_key_mapping_cache_so_next_jwt_call_gets_new publish_mock = AsyncMock() with ( - patch( + patch( # test-quality-ok: deterministic token; same pattern as sibling regenerate tests "litellm.proxy.management_endpoints.key_management_endpoints.get_new_token", new_callable=AsyncMock, return_value="sk-newtoken1234ab12", ), - patch( + patch( # test-quality-ok: grace-period path not under test; same pattern as sibling regenerate tests "litellm.proxy.management_endpoints.key_management_endpoints._insert_deprecated_key", new_callable=AsyncMock, ), - patch( + patch( # test-quality-ok: key-object eviction is separate from the mapping eviction under test "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", new_callable=AsyncMock, ), - patch( + patch( # test-quality-ok: background rotation hook is irrelevant to cache eviction "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_rotated_hook", new_callable=AsyncMock, ), - patch( + patch( # test-quality-ok: captures the cross-worker broadcast without a redis instance "litellm.proxy.common_utils.auth_cache_invalidation_pubsub.publish_auth_cache_invalidation", publish_mock, ), @@ -11998,7 +11998,7 @@ async def test_regenerate_evicts_jwt_key_mapping_cache_so_next_jwt_call_gets_new jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( virtual_key_claim_field="sub", virtual_key_mapping_cache_ttl=300 ) - with patch( + with patch( # test-quality-ok: DB-backed resolve; fake asserts it receives the rotated hash "litellm.proxy.auth.resolvers.store.IdentityStore.resolve", new_callable=AsyncMock, side_effect=fake_resolve, From 2849aee57dd6b78ce285df827cf204d21c007a59 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 4 Sep 2026 14:18:41 -0700 Subject: [PATCH 85/88] fix(health): probe test_connection with the credential the request names (#39801) * fix(health): probe test_connection with the credential the request names /health/test_connection matches the request's model string against the configured deployments and merges the match's litellm_params underneath the request. A request that named a stored credential but no key of its own still satisfied the "request sets no connection fields" test, so it inherited the matched deployment's api_key and api_base, and load_credentials_from_list then skipped the named credential because api_key was already set. A wildcard route covering the model is enough to match, so the Add Model page's Test Connect probed with an unrelated deployment's key while echoing back the credential that was selected. Naming a credential the configuration does not name now withholds the configuration's credential fields, the same set already withheld from a request that supplies its own endpoint. Naming no credential still inherits them, as documented. * test(health): drop test docstrings that restate their own names * test(health): assert the credential probe on the wire, not on the call args The connection-test regressions patched litellm.ahealth_check and read the params handed to it. Driving the endpoint through the app with respx faking the upstream instead lets the real credential resolution run, so the tests assert the key and host that actually go out, which is what the bug was about. It also drops three of the five patched proxy internals; the two that are left are proxy-global wiring with no injection seam, the same ones the image_edit connection test already has to reach for. * chore(ui): regenerate schema.d.ts for the test_connection docs change --- .../health_endpoints/_health_endpoints.py | 46 +++-- .../health_endpoints/test_health_endpoints.py | 167 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 3 + 3 files changed, 203 insertions(+), 13 deletions(-) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 65d0ec8c0dc..1785a2f0992 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -115,6 +115,29 @@ _CONFIG_CONNECTION_FIELDS: Final[frozenset[str]] = frozenset( ) +def _request_inherits_config_credentials( + config_params: Mapping[str, object], + request_params: Mapping[str, object], + allow_client_side_credentials: bool, +) -> bool: + """Whether the configuration's credentials are this request's to be probed with. + + The configuration reached here by matching the request's model string, which + also matches wildcard routes and unrelated deployments that merely serve the + same model, so a request naming a stored credential of its own has already + said where its credentials come from and does not borrow that one's. A blank + name is no name: ``load_credentials_from_list`` resolves nothing from it, so + it must not cost the request the credentials it would otherwise be probed + with. + """ + requested_credential: Final = request_params.get("litellm_credential_name") + if requested_credential and requested_credential != config_params.get("litellm_credential_name"): + return False + if allow_client_side_credentials: + return True + return not any(param in request_params for param in _BANNED_REQUEST_BODY_PARAMS) + + def _config_base_for_health_check( config_params: Mapping[str, object], request_params: Mapping[str, object], @@ -122,25 +145,19 @@ def _config_base_for_health_check( ) -> dict[str, object]: """Return the configured parameters to merge under a connection-test request. - A request that sets its own connection fields describes a connection of its - own, so the configuration's credentials are not carried into it: they belong - to the endpoint the configuration names. Anything the request does not set - still comes from the configuration, which is what lets a request name a - configured model and test it as configured. + A request that sets its own connection fields, or names its own stored + credential, describes a connection of its own, so the configuration's + credentials are not carried into it: they belong to the endpoint the + configuration names. Anything the request does not set still comes from the + configuration, which is what lets a request name a configured model and test + it as configured. ``litellm_credential_name`` is dropped alongside the literal credential fields: it names a stored credential that ``load_credentials_from_list`` resolves into the same secrets further down the call, so leaving it in place would reintroduce them by reference. - - ``general_settings.allow_client_side_credentials`` is the existing proxy-wide - opt-in for callers supplying their own connection parameters. Where an admin - has enabled it, a request may pair its own endpoint with the configured - credentials, as it could before. """ - if allow_client_side_credentials: - return dict(config_params) - if not any(param in request_params for param in _BANNED_REQUEST_BODY_PARAMS): + if _request_inherits_config_credentials(config_params, request_params, allow_client_side_credentials): return dict(config_params) return {key: value for key, value in config_params.items() if key not in _CONFIG_CONNECTION_FIELDS} @@ -1959,6 +1976,9 @@ async def test_model_connection( Note: - If the model is configured in proxy_config.yaml, credentials (api_key, api_base, etc.) will be automatically loaded from the config (with resolved environment variables). + - A request naming a stored credential (`litellm_credential_name`) that the configuration + does not name is probed with that credential instead, and inherits no credentials + from the configuration its model string happened to match. - You can override specific params by including them in the request. - You can use `os.environ/VARIABLE_NAME` syntax to reference environment variables, which will be resolved automatically (same as in proxy_config.yaml). diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index 0e90c107865..624d2f00817 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -15,6 +15,7 @@ from prisma.errors import ClientNotConnectedError, HTTPClientClosedError, Prisma import litellm import litellm.proxy.health_endpoints._health_endpoints as _health_endpoints_module from litellm.litellm_core_utils.health_check_helpers import TEST_IMAGE_BASE64 +from litellm.models.credentials import CredentialItem from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.health_endpoints._health_endpoints import ( @@ -2675,6 +2676,172 @@ class TestConfigBaseForHealthCheck: assert base["litellm_credential_name"] == "OpenAI-prod" assert base["api_key"] == "sk-configured" + def test_request_naming_another_credential_does_not_inherit_config_credentials(self): + base = self._base(self.CONFIG, {"model": "openai/gpt-4o", "litellm_credential_name": "Another-cred"}) + assert "api_key" not in base + assert "api_base" not in base + assert "vertex_credentials" not in base + assert base["rpm"] == 100 + + def test_blank_credential_name_names_no_credential(self): + base = self._base(self.CONFIG, {"model": "openai/gpt-4o", "litellm_credential_name": ""}) + assert base["api_key"] == "sk-configured" + + def test_opt_in_does_not_put_config_credentials_over_a_named_credential(self): + base = self._base( + self.CONFIG, + {"model": "openai/gpt-4o", "litellm_credential_name": "Another-cred"}, + allow_client_side_credentials=True, + ) + assert "api_key" not in base + + +class TestTestConnectionUsesTheNamedCredential: + CREDENTIAL_KEY = "sk-credential-key" + OTHER_DEPLOYMENT_KEY = "sk-other-deployment-key" + OTHER_DEPLOYMENT_BASE = "https://other-deployment.example/v1" + REQUEST = { + "model": "xai/grok-4", + "custom_llm_provider": "xai", + "litellm_credential_name": "my-xai-cred", + } + COMPLETION = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1700000000, + "model": "grok-4", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "ok"}}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + + @staticmethod + def _credential(**values: str) -> CredentialItem: + return CredentialItem(credential_name="my-xai-cred", credential_info={}, credential_values=values) + + @staticmethod + def _wildcard_deployment(**litellm_params: str) -> dict: + return { + "model_name": "xai/*", + "litellm_params": {"model": "xai/*", **litellm_params}, + "model_info": {"id": "unrelated-wildcard-deployment"}, + } + + def _probe( + self, + monkeypatch, + deployment: dict, + request_litellm_params: dict, + deployment_by_id: object | None = None, + request_model_info: dict | None = None, + ) -> httpx.Request: + """Run /health/test_connection and hand back the upstream request it made.""" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + app = FastAPI() + app.include_router(_health_endpoints_module.router) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + + router = MagicMock() + router.get_model_list.return_value = [deployment] + router.get_deployment.return_value = deployment_by_id + + with ( + patch( # test-quality-ok: the endpoint reads the proxy-global DB client and 500s when it is None; it has no injection seam + "litellm.proxy.proxy_server.prisma_client", MagicMock() + ), + patch( # test-quality-ok: the deployment the probe is matched against is a proxy global; it has no injection seam + "litellm.proxy.proxy_server.llm_router", router + ), + respx.mock(assert_all_called=True) as respx_mock, + ): + respx_mock.post(path__regex=r".*/chat/completions").respond(json=self.COMPLETION) + response = TestClient(app).post( + "/health/test_connection", + json={ + "mode": "chat", + "litellm_params": request_litellm_params, + "model_info": request_model_info or {"mode": "chat"}, + }, + ) + probe = respx_mock.calls.last.request + + assert response.status_code == 200, response.text + assert response.json()["status"] == "success", response.text + return probe + + def test_named_credentials_key_is_sent_not_the_matched_deployments_key(self, monkeypatch): + monkeypatch.setattr(litellm, "credential_list", [self._credential(api_key=self.CREDENTIAL_KEY)]) + + probe = self._probe( + monkeypatch, + self._wildcard_deployment(api_key=self.OTHER_DEPLOYMENT_KEY), + self.REQUEST, + ) + + assert probe.headers["authorization"] == f"Bearer {self.CREDENTIAL_KEY}" + + def test_named_credentials_api_base_is_used_not_the_matched_deployments(self, monkeypatch): + monkeypatch.setattr( + litellm, + "credential_list", + [self._credential(api_key=self.CREDENTIAL_KEY, api_base="https://credential.example/v1")], + ) + + probe = self._probe( + monkeypatch, + self._wildcard_deployment(api_base=self.OTHER_DEPLOYMENT_BASE), + self.REQUEST, + ) + + assert probe.url.host == "credential.example" + + def test_named_credential_without_an_api_base_leaves_the_provider_default(self, monkeypatch): + monkeypatch.setattr(litellm, "credential_list", [self._credential(api_key=self.CREDENTIAL_KEY)]) + + probe = self._probe( + monkeypatch, + self._wildcard_deployment(api_base=self.OTHER_DEPLOYMENT_BASE), + self.REQUEST, + ) + + assert probe.url.host == "api.x.ai" + + def test_configured_model_named_without_a_credential_still_inherits_its_config(self, monkeypatch): + probe = self._probe( + monkeypatch, + self._wildcard_deployment(api_key=self.OTHER_DEPLOYMENT_KEY, api_base=self.OTHER_DEPLOYMENT_BASE), + {"model": "xai/grok-4", "custom_llm_provider": "xai"}, + ) + + assert probe.headers["authorization"] == f"Bearer {self.OTHER_DEPLOYMENT_KEY}" + assert probe.url.host == "other-deployment.example" + + def test_deployment_probed_by_id_keeps_the_endpoint_it_is_configured_with(self, monkeypatch): + """The model detail page always echoes back the credential the deployment already uses.""" + from litellm.types.router import Deployment, LiteLLM_Params + + monkeypatch.setattr(litellm, "credential_list", [self._credential(api_key=self.CREDENTIAL_KEY)]) + + probe = self._probe( + monkeypatch, + self._wildcard_deployment(api_key=self.OTHER_DEPLOYMENT_KEY, api_base=self.OTHER_DEPLOYMENT_BASE), + self.REQUEST, + deployment_by_id=Deployment( + model_name="grok-4", + litellm_params=LiteLLM_Params( + model="xai/grok-4", + api_base="https://configured.example/v1", + litellm_credential_name="my-xai-cred", + ), + model_info={"id": "configured-deployment"}, + ), + request_model_info={"id": "configured-deployment", "mode": "chat"}, + ) + + assert probe.url.host == "configured.example" + assert probe.headers["authorization"] == f"Bearer {self.CREDENTIAL_KEY}" + class TestNoRedisWarning: """`show_no_redis_warning` drives the Admin UI's default-on "no Redis" banner.""" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index f4cb88bbae1..3a889aa63e9 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -7037,6 +7037,9 @@ export interface paths { * Note: * - If the model is configured in proxy_config.yaml, credentials (api_key, api_base, etc.) * will be automatically loaded from the config (with resolved environment variables). + * - A request naming a stored credential (`litellm_credential_name`) that the configuration + * does not name is probed with that credential instead, and inherits no credentials + * from the configuration its model string happened to match. * - You can override specific params by including them in the request. * - You can use `os.environ/VARIABLE_NAME` syntax to reference environment variables, * which will be resolved automatically (same as in proxy_config.yaml). From 49cac6fc5b59d10b017d4686c39dd601ae2d0af5 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 14:35:42 -0700 Subject: [PATCH 86/88] fix(jwt): evict mapping cache after DB write in /jwt/key/mapping update and delete Evicting before the mutation commits left a race: a concurrent JWT request could re-cache the old mapping between the eviction and the commit, keeping a deleted or renamed claim authorized until the cache TTL expired. Flagged by review on PR #39808. --- .../jwt_key_mapping_endpoints.py | 16 ++-- .../proxy_unit_tests/test_jwt_key_mapping.py | 83 +++++++++++++++++++ 2 files changed, 93 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py b/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py index 4f6468e911f..694930a543c 100644 --- a/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py +++ b/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py @@ -170,18 +170,20 @@ async def update_jwt_key_mapping( if old_mapping is None: raise HTTPException(status_code=404, detail="Mapping not found") - old_cache_key: Final = jwt_key_mapping_cache_key(old_mapping.jwt_claim_name, old_mapping.jwt_claim_value) - await evict_and_broadcast(cache_keys=(old_cache_key,), user_api_key_cache=user_api_key_cache) - updated_mapping: Final = await _mapping_table(prisma_client).update(where={"id": data.id}, data=update_data) if updated_mapping is None: raise HTTPException(status_code=404, detail="Mapping not found") + # Evict only after the write commits: a concurrent request between an + # early eviction and the commit would re-cache the old mapping and keep + # it authorized until TTL. + old_cache_key: Final = jwt_key_mapping_cache_key(old_mapping.jwt_claim_name, old_mapping.jwt_claim_value) new_cache_key: Final = jwt_key_mapping_cache_key( updated_mapping.jwt_claim_name, updated_mapping.jwt_claim_value ) - await evict_and_broadcast(cache_keys=(new_cache_key,), user_api_key_cache=user_api_key_cache) + cache_keys: Final = (old_cache_key,) if old_cache_key == new_cache_key else (old_cache_key, new_cache_key) + await evict_and_broadcast(cache_keys=cache_keys, user_api_key_cache=user_api_key_cache) return _to_response(updated_mapping) except HTTPException: @@ -221,10 +223,12 @@ async def delete_jwt_key_mapping( if old_mapping is None: raise HTTPException(status_code=404, detail="Mapping not found") + await _mapping_table(prisma_client).delete(where={"id": data.id}) + + # Evict only after the row is gone, else a concurrent request can + # re-cache the deleted mapping and keep it authorized until TTL. cache_key: Final = jwt_key_mapping_cache_key(old_mapping.jwt_claim_name, old_mapping.jwt_claim_value) await evict_and_broadcast(cache_keys=(cache_key,), user_api_key_cache=user_api_key_cache) - - await _mapping_table(prisma_client).delete(where={"id": data.id}) return {"status": "success"} except HTTPException: raise diff --git a/tests/proxy_unit_tests/test_jwt_key_mapping.py b/tests/proxy_unit_tests/test_jwt_key_mapping.py index 4b50f83e9eb..e8db5d1cf7f 100644 --- a/tests/proxy_unit_tests/test_jwt_key_mapping.py +++ b/tests/proxy_unit_tests/test_jwt_key_mapping.py @@ -1333,3 +1333,86 @@ def test_jwt_client_id_field_does_not_raise_on_duplicate(): virtual_key_claim_field="new_field", ) assert auth.virtual_key_claim_field == "new_field" + + +# ────────────────────────────────────────────── +# Tests: cache eviction must happen AFTER the DB write commits +# ────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_delete_evicts_cache_after_row_is_gone(): + """A JWT request racing the delete must not keep the removed mapping authorized. + + The DB delete simulates a concurrent request re-caching the mapping mid-write. + If the endpoint evicts before the delete commits, that repopulated entry + survives until TTL and the deleted mapping stays usable. + """ + from litellm.proxy._types import DeleteJWTKeyMappingRequest + from litellm.proxy.auth.auth_checks import jwt_key_mapping_cache_key + + cache_key = jwt_key_mapping_cache_key("email", "user@example.com") + user_api_key_cache = DualCache() + await user_api_key_cache.async_set_cache(key=cache_key, value="hashed_token") + + mock_prisma = _mock_prisma() + mock_prisma.db.litellm_jwtkeymapping.find_unique.return_value = _mock_mapping() + + async def concurrent_reader_repopulates(**kwargs): + await user_api_key_cache.async_set_cache(key=cache_key, value="hashed_token") + return _mock_mapping() + + mock_prisma.db.litellm_jwtkeymapping.delete.side_effect = concurrent_reader_repopulates + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), # test-quality-ok: proxy_server module global is the endpoint's only injection point + patch("litellm.proxy.proxy_server.user_api_key_cache", user_api_key_cache), # test-quality-ok: proxy_server module global is the endpoint's only injection point + ): + result = await delete_jwt_key_mapping( + data=DeleteJWTKeyMappingRequest(id="mapping-1"), + user_api_key_dict=_make_admin_auth(), + ) + + assert result == {"status": "success"} + assert await user_api_key_cache.async_get_cache(cache_key) is None + + +@pytest.mark.asyncio +async def test_update_evicts_old_and_new_cache_keys_after_write(): + """Renaming a mapping's claim must leave neither claim serving stale cache. + + The DB update simulates a concurrent request re-caching the OLD mapping + mid-write. Both the old claim's entry (would restore the pre-rename token) + and the new claim's __NO_MAPPING__ sentinel (would 403 the renamed claim) + must be gone once the endpoint returns. + """ + from litellm.proxy._types import UpdateJWTKeyMappingRequest + from litellm.proxy.auth.auth_checks import jwt_key_mapping_cache_key + + old_cache_key = jwt_key_mapping_cache_key("email", "user@example.com") + new_cache_key = jwt_key_mapping_cache_key("email", "renamed@example.com") + user_api_key_cache = DualCache() + await user_api_key_cache.async_set_cache(key=old_cache_key, value="hashed_token") + await user_api_key_cache.async_set_cache(key=new_cache_key, value="__NO_MAPPING__") + + mock_prisma = _mock_prisma() + mock_prisma.db.litellm_jwtkeymapping.find_unique.return_value = _mock_mapping() + + async def concurrent_reader_repopulates(**kwargs): + await user_api_key_cache.async_set_cache(key=old_cache_key, value="hashed_token") + return _mock_mapping(claim_value="renamed@example.com") + + mock_prisma.db.litellm_jwtkeymapping.update.side_effect = concurrent_reader_repopulates + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), # test-quality-ok: proxy_server module global is the endpoint's only injection point + patch("litellm.proxy.proxy_server.user_api_key_cache", user_api_key_cache), # test-quality-ok: proxy_server module global is the endpoint's only injection point + ): + result = await update_jwt_key_mapping( + data=UpdateJWTKeyMappingRequest(id="mapping-1", jwt_claim_value="renamed@example.com"), + user_api_key_dict=_make_admin_auth(), + ) + + assert result.jwt_claim_value == "renamed@example.com" + assert await user_api_key_cache.async_get_cache(old_cache_key) is None + assert await user_api_key_cache.async_get_cache(new_cache_key) is None From 6c81a5c4235d62850427f8f922dbb63fe96130cd Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 14:38:08 -0700 Subject: [PATCH 87/88] feat(guardrails): store untracked units on the rollup row instead of nulling cost A row that received both priced and unpriced increments used to collapse to cost NULL, throwing away the priced subtotal and making every unit on it read as untracked. The rollup now carries a second column, untracked_units, that the aggregator increments for units with no known price while cost keeps accruing for the rest, so cost covers exactly units - untracked_units. Rows written before the migration keep cost NULL and still read as untracked in full The endpoints read untracked units off the column (or the whole row for a legacy NULL) rather than from a NULL filter, and the policies overview now fills totalUntrackedUsageUnits, which the previous commit missed Claude-Session: https://claude.ai/code/session_01EX13mWex6RaBo9PYnkAtFW --- .../migration.sql | 1 + .../litellm_proxy_extras/schema.prisma | 3 +- litellm/proxy/_lazy_openapi_snapshot.json | 8 +- litellm/proxy/guardrails/usage_endpoints.py | 70 ++++++++-------- litellm/proxy/guardrails/usage_tracking.py | 26 ++++-- litellm/proxy/schema.prisma | 3 +- schema.prisma | 3 +- .../proxy/guardrails/test_usage_endpoints.py | 61 +++++++++++--- .../proxy/guardrails/test_usage_tracking.py | 84 +++++++++++-------- ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +- 10 files changed, 168 insertions(+), 95 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000001_add_guardrail_usage_units_cost/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000001_add_guardrail_usage_units_cost/migration.sql index 27a86a0b09a..a89b7c4c6f8 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000001_add_guardrail_usage_units_cost/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000001_add_guardrail_usage_units_cost/migration.sql @@ -1,2 +1,3 @@ -- AlterTable ALTER TABLE "LiteLLM_DailyGuardrailUsageUnits" ADD COLUMN IF NOT EXISTS "cost" DOUBLE PRECISION; +ALTER TABLE "LiteLLM_DailyGuardrailUsageUnits" ADD COLUMN IF NOT EXISTS "untracked_units" BIGINT NOT NULL DEFAULT 0; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 3134d7dde0e..28ed49fd0be 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1123,7 +1123,8 @@ model LiteLLM_DailyGuardrailUsageUnits { api_key String // hashed virtual key; empty string when unknown usage_unit String // provider counter name, e.g. Bedrock's contentPolicyUnits units BigInt @default(0) - cost Float? // USD billed for these units; null when any contributing increment was unpriced + cost Float? // USD for the priced share of units; null only on rows written before this column existed + untracked_units BigInt @default(0) // units recorded with no known price, the share cost leaves out created_at DateTime @default(now()) updated_at DateTime @updatedAt diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index c399e5594f6..fff4bb9cd6f 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -13436,7 +13436,7 @@ "type": "null" } ], - "description": "USD billed for usageUnits over the window, summed over days with tracked cost; null when none have it", + "description": "USD for the priced share of usageUnits over the window; null when no unit was priced", "title": "Cost" }, "failRate": { @@ -13475,7 +13475,7 @@ "additionalProperties": { "type": "integer" }, - "description": "The share of usageUnits that cost leaves out: units from days with no tracked cost, per counter", + "description": "The share of usageUnits that cost leaves out: units recorded with no known price, per counter", "title": "Untrackedusageunits", "type": "object" }, @@ -28968,7 +28968,7 @@ "type": "null" } ], - "description": "USD billed for usageUnits over the window, summed over days with tracked cost; null when none have it", + "description": "USD for the priced share of usageUnits over the window; null when no unit was priced", "title": "Cost" }, "failRate": { @@ -29007,7 +29007,7 @@ "additionalProperties": { "type": "integer" }, - "description": "The share of usageUnits that cost leaves out: units from days with no tracked cost, per counter", + "description": "The share of usageUnits that cost leaves out: units recorded with no known price, per counter", "title": "Untrackedusageunits", "type": "object" }, diff --git a/litellm/proxy/guardrails/usage_endpoints.py b/litellm/proxy/guardrails/usage_endpoints.py index 523efe0da75..0390a2b5013 100644 --- a/litellm/proxy/guardrails/usage_endpoints.py +++ b/litellm/proxy/guardrails/usage_endpoints.py @@ -156,6 +156,16 @@ def _counter_name(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> str: return row.usage_unit +def _row_untracked_units(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> int: + """A row written before the cost column carries NULL cost and is untracked in full.""" + return int(row.units) if row.cost is None else int(row.untracked_units) + + +def _row_tracked_cost(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> float | None: + """The row's cost when it prices at least one unit; None when every unit is untracked.""" + return None if row.cost is None or _row_untracked_units(row) >= int(row.units) else row.cost + + def _sum_counter_units(rows: "Iterable[prisma_models.LiteLLM_DailyGuardrailUsageUnits]") -> Mapping[str, int]: ordered: Final = sorted(rows, key=_counter_name) return MappingProxyType( @@ -163,33 +173,27 @@ def _sum_counter_units(rows: "Iterable[prisma_models.LiteLLM_DailyGuardrailUsage ) -def _units_by( - rows: "Sequence[prisma_models.LiteLLM_DailyGuardrailUsageUnits]", - key_of: "Callable[[prisma_models.LiteLLM_DailyGuardrailUsageUnits], str]", -) -> Mapping[str, Mapping[str, int]]: - ordered: Final = sorted(rows, key=key_of) - return MappingProxyType({key: _sum_counter_units(group) for key, group in groupby(ordered, key=key_of)}) +def _sum_untracked_units(rows: "Iterable[prisma_models.LiteLLM_DailyGuardrailUsageUnits]") -> Mapping[str, int]: + ordered: Final = sorted(rows, key=_counter_name) + per_counter: Final = tuple( + (name, sum(map(_row_untracked_units, group))) for name, group in groupby(ordered, key=_counter_name) + ) + return MappingProxyType({name: units for name, units in per_counter if units}) def _sum_tracked_cost(rows: "Iterable[prisma_models.LiteLLM_DailyGuardrailUsageUnits]") -> float | None: - """Sum over rows with a tracked cost; None when no row has one (pre-migration or unpriced).""" - tracked: Final = tuple(r.cost for r in rows if r.cost is not None) + """Sum over rows that price at least one unit; None when no row does.""" + tracked: Final = tuple(cost for cost in map(_row_tracked_cost, rows) if cost is not None) return sum(tracked) if tracked else None -def _cost_by( +def _by( rows: "Sequence[prisma_models.LiteLLM_DailyGuardrailUsageUnits]", key_of: "Callable[[prisma_models.LiteLLM_DailyGuardrailUsageUnits], str]", -) -> Mapping[str, float | None]: + reduce: "Callable[[Iterable[prisma_models.LiteLLM_DailyGuardrailUsageUnits]], _T]", +) -> Mapping[str, _T]: ordered: Final = sorted(rows, key=key_of) - return MappingProxyType({key: _sum_tracked_cost(group) for key, group in groupby(ordered, key=key_of)}) - - -def _untracked_rows( - rows: "Sequence[prisma_models.LiteLLM_DailyGuardrailUsageUnits]", -) -> "tuple[prisma_models.LiteLLM_DailyGuardrailUsageUnits, ...]": - """Rows whose cost is unknown, so their units are exactly what the tracked cost sums leave out.""" - return tuple(r for r in rows if r.cost is None) + return MappingProxyType({key: reduce(group) for key, group in groupby(ordered, key=key_of)}) def _first_match(lookup_keys: Sequence[str], mapping: Mapping[str, _T], default: _T) -> _T: @@ -246,10 +250,10 @@ class UsageOverviewRow(BaseModel): trend: str # up | down | stable usageUnits: Mapping[str, int] cost: float | None = Field( - description="USD billed for usageUnits over the window, summed over days with tracked cost; null when none have it" + description="USD for the priced share of usageUnits over the window; null when no unit was priced" ) untrackedUsageUnits: Mapping[str, int] = Field( - description="The share of usageUnits that cost leaves out: units from days with no tracked cost, per counter" + description="The share of usageUnits that cost leaves out: units recorded with no known price, per counter" ) @@ -573,10 +577,9 @@ async def guardrails_usage_overview( agg: Final = _aggregate_daily_metrics(metrics, "guardrail_id") prev_agg: Final = _prev_fail_rates(metrics_prev, "guardrail_id") - untracked_rows: Final = _untracked_rows(units_rows) - units_agg: Final = _units_by(units_rows, lambda r: r.guardrail_id) - cost_agg: Final = _cost_by(units_rows, lambda r: r.guardrail_id) - untracked_agg: Final = _units_by(untracked_rows, lambda r: r.guardrail_id) + units_agg: Final = _by(units_rows, lambda r: r.guardrail_id, _sum_counter_units) + cost_agg: Final = _by(units_rows, lambda r: r.guardrail_id, _sum_tracked_cost) + untracked_agg: Final = _by(units_rows, lambda r: r.guardrail_id, _sum_untracked_units) chart: Final = _chart_from_metrics(metrics) total_requests: Final = sum(a["requests"] for a in agg.values()) total_blocked: Final = sum(a["blocked"] for a in agg.values()) @@ -590,7 +593,7 @@ async def guardrails_usage_overview( passRate=round(pass_rate, 1), totalUsageUnits=_sum_counter_units(units_rows), totalCost=_sum_tracked_cost(units_rows), - totalUntrackedUsageUnits=_sum_counter_units(untracked_rows), + totalUntrackedUsageUnits=_sum_untracked_units(units_rows), ) except Exception as e: from litellm.proxy.utils import handle_exception_on_proxy @@ -681,8 +684,8 @@ async def guardrails_usage_detail( litellm_params: Final = _to_dict(_get_guardrail_field(guardrail, "litellm_params")) guardrail_info: Final = _to_dict(_get_guardrail_field(guardrail, "guardrail_info")) _guardrail_name: Final = _get_guardrail_field(guardrail, "guardrail_name") - daily_unit_sums: Final = sorted(_units_by(units_rows, lambda r: r.date).items()) - daily_cost: Final = _cost_by(units_rows, lambda r: r.date) + daily_unit_sums: Final = sorted(_by(units_rows, lambda r: r.date, _sum_counter_units).items()) + daily_cost: Final = _by(units_rows, lambda r: r.date, _sum_tracked_cost) units_daily: Final = tuple( UsageUnitsDailyPoint(date=d, units=units, cost=daily_cost.get(d)) for d, units in daily_unit_sums ) @@ -702,13 +705,13 @@ async def guardrails_usage_detail( time_series=time_series, usage_units=_sum_counter_units(units_rows), usage_units_daily=units_daily, - usage_units_by_team=_units_by(units_rows, lambda r: r.team_id), - usage_units_by_key=_units_by(units_rows, lambda r: r.api_key), + usage_units_by_team=_by(units_rows, lambda r: r.team_id, _sum_counter_units), + usage_units_by_key=_by(units_rows, lambda r: r.api_key, _sum_counter_units), cost=_sum_tracked_cost(units_rows), - cost_by_unit=_cost_by(units_rows, _counter_name), - cost_by_team=_cost_by(units_rows, lambda r: r.team_id), - cost_by_key=_cost_by(units_rows, lambda r: r.api_key), - untracked_usage_units=_sum_counter_units(_untracked_rows(units_rows)), + cost_by_unit=_by(units_rows, _counter_name, _sum_tracked_cost), + cost_by_team=_by(units_rows, lambda r: r.team_id, _sum_tracked_cost), + cost_by_key=_by(units_rows, lambda r: r.api_key, _sum_tracked_cost), + untracked_usage_units=_sum_untracked_units(units_rows), ) @@ -961,6 +964,7 @@ async def policies_usage_overview( passRate=round(pass_rate, 1), totalUsageUnits=_EMPTY_UNITS, totalCost=None, + totalUntrackedUsageUnits=_EMPTY_UNITS, ) except Exception as e: from litellm.proxy.utils import handle_exception_on_proxy diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index 41cad232efe..cb6aec14f8c 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -47,7 +47,16 @@ class _UsageUnitKey(NamedTuple): class _UsageUnitIncrement(NamedTuple): units: int - cost: float | None + cost: float + """USD for the priced share of units.""" + untracked_units: int + """Units recorded with no known price, the share cost leaves out.""" + + +def _usage_unit_increment(units: int, cost: float | None) -> _UsageUnitIncrement: + if cost is None: + return _UsageUnitIncrement(units=units, cost=0.0, untracked_units=units) + return _UsageUnitIncrement(units=units, cost=cost, untracked_units=0) class _MetricsKey(NamedTuple): @@ -79,7 +88,7 @@ class PendingRollups: _PENDING_ROLLUPS: Final = PendingRollups() _NO_COUNTERS: Final[Mapping[str, int]] = MappingProxyType({}) -_NO_INCREMENT: Final = _UsageUnitIncrement(units=0, cost=0.0) +_NO_INCREMENT: Final = _UsageUnitIncrement(units=0, cost=0.0, untracked_units=0) def _merged_keys(base: Mapping[_RowKey, object], extra: Mapping[_RowKey, object]) -> tuple[_RowKey, ...]: @@ -87,12 +96,11 @@ def _merged_keys(base: Mapping[_RowKey, object], extra: Mapping[_RowKey, object] def _summed_increments(increments: Iterable[_UsageUnitIncrement]) -> _UsageUnitIncrement: - """Units add; cost adds too unless any increment was unpriced, which makes the sum unknown.""" materialized: Final = tuple(increments) - costs: Final = tuple(i.cost for i in materialized) return _UsageUnitIncrement( units=sum(i.units for i in materialized), - cost=None if any(c is None for c in costs) else sum(c for c in costs if c is not None), + cost=sum(i.cost for i in materialized), + untracked_units=sum(i.untracked_units for i in materialized), ) @@ -251,7 +259,7 @@ def _iter_usage_unit_increments( if isinstance(units, int) and not isinstance(units, bool) and units > 0: key = _UsageUnitKey(guardrail_id, date_key, team_id, api_key, str(unit_name)) cost = cost_by_unit.get(str(unit_name)) if cost_by_unit is not None else None - yield key, _UsageUnitIncrement(units=units, cost=cost) + yield key, _usage_unit_increment(units=units, cost=cost) def _sum_usage_unit_increments( @@ -277,6 +285,7 @@ async def _upsert_usage_unit_row( "usage_unit": key.usage_unit, "units": increment.units, "cost": increment.cost, + "untracked_units": increment.untracked_units, } where: Final[_UsageUnitWhereUnique] = { "guardrail_id_date_team_id_api_key_usage_unit": { @@ -287,12 +296,13 @@ async def _upsert_usage_unit_row( "usage_unit": key.usage_unit, } } - # NULL + x stays NULL in SQL, so an unknown cost stays unknown; writing NULL outright makes it so + # A row written before the cost column has NULL cost, and NULL + x stays NULL, so it keeps reading as unknown data: Final[prisma_types.LiteLLM_DailyGuardrailUsageUnitsUpsertInput] = { "create": row, "update": { "units": {"increment": increment.units}, - "cost": {"increment": increment.cost} if increment.cost is not None else None, + "cost": {"increment": increment.cost}, + "untracked_units": {"increment": increment.untracked_units}, }, } await DailyGuardrailUsageUnitsRepository(prisma_client).table.upsert(where=where, data=data) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 3134d7dde0e..28ed49fd0be 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1123,7 +1123,8 @@ model LiteLLM_DailyGuardrailUsageUnits { api_key String // hashed virtual key; empty string when unknown usage_unit String // provider counter name, e.g. Bedrock's contentPolicyUnits units BigInt @default(0) - cost Float? // USD billed for these units; null when any contributing increment was unpriced + cost Float? // USD for the priced share of units; null only on rows written before this column existed + untracked_units BigInt @default(0) // units recorded with no known price, the share cost leaves out created_at DateTime @default(now()) updated_at DateTime @updatedAt diff --git a/schema.prisma b/schema.prisma index 3134d7dde0e..28ed49fd0be 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1123,7 +1123,8 @@ model LiteLLM_DailyGuardrailUsageUnits { api_key String // hashed virtual key; empty string when unknown usage_unit String // provider counter name, e.g. Bedrock's contentPolicyUnits units BigInt @default(0) - cost Float? // USD billed for these units; null when any contributing increment was unpriced + cost Float? // USD for the priced share of units; null only on rows written before this column existed + untracked_units BigInt @default(0) // units recorded with no known price, the share cost leaves out created_at DateTime @default(now()) updated_at DateTime @updatedAt diff --git a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py index 4a11c589810..ebb2be6edc2 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py @@ -86,7 +86,9 @@ def _units_row( usage_unit: str = "contentPolicyUnits", units: int = 1, cost: float | None = None, + untracked_units: int = 0, ) -> Any: + """cost=None is a row written before the cost column existed (untracked in full).""" r = MagicMock() r.guardrail_id = guardrail_id r.date = date @@ -95,6 +97,7 @@ def _units_row( r.usage_unit = usage_unit r.units = units r.cost = cost + r.untracked_units = untracked_units return r @@ -320,8 +323,9 @@ async def test_overview_degrades_units_to_empty_when_units_table_is_missing(): @pytest.mark.asyncio async def test_overview_reports_cost_per_row_and_total_summing_only_tracked_days(): """LIT-5652: cost rides the units rollup. Rows written before the cost column - (or by an unpriced hook) carry NULL and must drop out of the sum rather than - read as $0, and a guardrail with only NULL rows reports None, not 0.0.""" + carry NULL and rows whose every unit was unpriced carry 0.0 with + untracked_units == units; both must drop out of the sum rather than read as + $0, and a guardrail with only such rows reports None, not 0.0.""" prisma = _prisma( find_many=[], metrics=[_metric("yaml-pii", requests=4, passed=3, blocked=1)], @@ -329,6 +333,9 @@ async def test_overview_reports_cost_per_row_and_total_summing_only_tracked_days _units_row("yaml-pii", usage_unit="contentPolicyUnits", units=1000, cost=0.15), _units_row("yaml-pii", team_id="team-a", usage_unit="contentPolicyUnits", units=2000, cost=0.3), _units_row("yaml-pii", date="2026-04-24", usage_unit="contentPolicyUnits", units=5000, cost=None), + _units_row( + "yaml-pii", date="2026-04-23", usage_unit="topicPolicyUnits", units=9, cost=0.0, untracked_units=9 + ), _units_row("legacy-guard", usage_unit="topicPolicyUnits", units=7, cost=None), ], ) @@ -347,17 +354,21 @@ async def test_overview_reports_cost_per_row_and_total_summing_only_tracked_days @pytest.mark.asyncio async def test_overview_reports_the_units_its_cost_leaves_out_per_row_and_total(): - """A row's cost silently under-reports whenever some of its days carry NULL, so - the response must say exactly which units (per counter) that cost excludes. - A guardrail whose rows are all priced reports none; one with only NULL rows - reports all of its units; a mix reports just the NULL rows' units.""" + """A row's cost covers only the units that had a price, so the response must + say exactly which units (per counter) that cost excludes: the row's own + untracked_units, or all of its units when it predates the cost column. A + guardrail whose rows are all priced reports none, one whose rows are all + unpriced reports all of its units, and a mixed row keeps its priced subtotal + while reporting just the unpriced share.""" prisma = _prisma( find_many=[], metrics=[_metric("yaml-pii", requests=4, passed=3, blocked=1)], units=[ - _units_row("yaml-pii", usage_unit="contentPolicyUnits", units=1000, cost=0.15), + _units_row("yaml-pii", usage_unit="contentPolicyUnits", units=1000, cost=0.15, untracked_units=200), _units_row("yaml-pii", date="2026-04-24", usage_unit="contentPolicyUnits", units=5000, cost=None), - _units_row("yaml-pii", date="2026-04-24", usage_unit="topicPolicyUnits", units=40, cost=None), + _units_row( + "yaml-pii", date="2026-04-24", usage_unit="topicPolicyUnits", units=40, cost=0.0, untracked_units=40 + ), _units_row("yaml-pii", usage_unit="wordPolicyUnits", units=9, cost=0.0), _units_row("legacy-guard", usage_unit="topicPolicyUnits", units=7, cost=None), _units_row("priced-guard", usage_unit="contentPolicyUnits", units=3, cost=0.0003), @@ -373,10 +384,11 @@ async def test_overview_reports_the_units_its_cost_leaves_out_per_row_and_total( resp = await guardrails_usage_overview(start_date=START, end_date=END, user_api_key_dict=ADMIN) by_id = {r.id: r for r in resp.rows} assert by_id["yaml-uuid"].usageUnits == {"contentPolicyUnits": 6000, "topicPolicyUnits": 40, "wordPolicyUnits": 9} - assert by_id["yaml-uuid"].untrackedUsageUnits == {"contentPolicyUnits": 5000, "topicPolicyUnits": 40} + assert by_id["yaml-uuid"].cost == pytest.approx(0.15) + assert by_id["yaml-uuid"].untrackedUsageUnits == {"contentPolicyUnits": 5200, "topicPolicyUnits": 40} assert by_id["legacy-uuid"].untrackedUsageUnits == {"topicPolicyUnits": 7} assert by_id["priced-uuid"].untrackedUsageUnits == {} - assert resp.totalUntrackedUsageUnits == {"contentPolicyUnits": 5000, "topicPolicyUnits": 47} + assert resp.totalUntrackedUsageUnits == {"contentPolicyUnits": 5200, "topicPolicyUnits": 47} @pytest.mark.asyncio @@ -387,7 +399,9 @@ async def test_detail_breaks_cost_down_by_unit_day_team_and_key(): find_unique=None, units=[ _units_row("yaml-pii", date="2026-04-25", team_id="team-a", api_key="hash-1", units=1000, cost=0.15), - _units_row("yaml-pii", date="2026-04-25", team_id="", api_key="hash-2", units=200, cost=0.03), + _units_row( + "yaml-pii", date="2026-04-25", team_id="", api_key="hash-2", units=200, cost=0.03, untracked_units=50 + ), _units_row( "yaml-pii", date="2026-04-24", @@ -415,7 +429,7 @@ async def test_detail_breaks_cost_down_by_unit_day_team_and_key(): assert resp.cost_by_key == {"hash-1": pytest.approx(0.15), "hash-2": pytest.approx(0.03)} assert resp.cost_by_team.keys() == resp.usage_units_by_team.keys() assert resp.cost_by_key.keys() == resp.usage_units_by_key.keys() - assert resp.untracked_usage_units == {"topicPolicyUnits": 10} + assert resp.untracked_usage_units == {"contentPolicyUnits": 50, "topicPolicyUnits": 10} @pytest.mark.asyncio @@ -518,6 +532,29 @@ async def test_detail_rejects_reversed_dates(): assert exc.value.status_code == 400 +@pytest.mark.asyncio +async def test_policies_overview_returns_a_full_row_and_totals(): + """Regression: the policies overview shares the guardrail response model, so + every field added there (usage units, cost, untracked units) must be filled + here too or the endpoint 500s on model validation.""" + policy = MagicMock(spec=["policy_id", "policy_name"]) + policy.policy_id = "pol-1" + policy.policy_name = "block-pii" + metric = _metric("pol-1", requests=10, passed=8, blocked=2) + metric.policy_id = "pol-1" + prisma = _prisma() + prisma.db.litellm_policytable.find_many = AsyncMock(return_value=[policy]) + prisma.db.litellm_dailypolicymetrics.find_many = AsyncMock(return_value=[metric]) + p1, p2 = _patches(prisma, _config_handler()) + with p1, p2: + resp = await policies_usage_overview(start_date=START, end_date=END, user_api_key_dict=ADMIN) + row = next(r for r in resp.rows if r.id == "pol-1") + assert (row.name, row.type, row.requestsEvaluated, row.failRate) == ("block-pii", "Policy", 10, 20.0) + assert (row.usageUnits, row.cost, row.untrackedUsageUnits) == ({}, None, {}) + assert (resp.totalRequests, resp.totalBlocked, resp.passRate) == (10, 2, 80.0) + assert (resp.totalUsageUnits, resp.totalCost, resp.totalUntrackedUsageUnits) == ({}, None, {}) + + @pytest.mark.asyncio async def test_policies_overview_rejects_range_over_max_days(): prisma = _prisma() diff --git a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py index 347c65cf819..ae360b281cb 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py @@ -64,16 +64,17 @@ def _units_upserts(prisma: MagicMock) -> dict[tuple, int]: return out -def _cost_upserts(prisma: MagicMock) -> dict[str, tuple[float | None, object]]: - """usage_unit -> (cost written on create, cost clause sent on update).""" +def _cost_upserts(prisma: MagicMock) -> dict[str, tuple[float, int]]: + """usage_unit -> (cost, untracked_units) written on create; the update path must increment by the same.""" calls = prisma.db.litellm_dailyguardrailusageunits.upsert.call_args_list - return { - c.kwargs["data"]["create"]["usage_unit"]: ( - c.kwargs["data"]["create"]["cost"], - c.kwargs["data"]["update"]["cost"], - ) - for c in calls - } + out: dict[str, tuple[float, int]] = {} + for c in calls: + create = c.kwargs["data"]["create"] + update = c.kwargs["data"]["update"] + assert update["cost"] == {"increment": create["cost"]} + assert update["untracked_units"] == {"increment": create["untracked_units"]} + out[create["usage_unit"]] = (create["cost"], create["untracked_units"]) + return out @pytest.mark.asyncio @@ -200,7 +201,7 @@ async def test_retry_exhausted_rows_are_requeued_and_land_on_the_next_flush(): ) assert dict(pending.units) == { - ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "topicPolicyUnits"): (2, None) + ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "topicPolicyUnits"): (2, 0.0, 2) } recovered = _prisma() @@ -368,15 +369,15 @@ async def test_cost_rolled_up_per_counter_alongside_units(): ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "wordPolicyUnits"): 60, } costs = _cost_upserts(prisma) - assert costs["contentPolicyUnits"][0] == pytest.approx(0.45) - assert costs["contentPolicyUnits"][1] == {"increment": pytest.approx(0.45)} - assert costs["wordPolicyUnits"] == (0.0, {"increment": 0.0}) + assert costs["contentPolicyUnits"] == (pytest.approx(0.45), 0) + assert costs["wordPolicyUnits"] == (0.0, 0) @pytest.mark.asyncio -async def test_counter_the_hook_could_not_price_is_stored_unknown_not_free(): - """A counter the cost map does not list arrives stamped as None. Its row must - carry NULL, while the priced counter on the same request keeps its cost.""" +async def test_counter_the_hook_could_not_price_is_stored_as_untracked_units_not_free(): + """A counter the cost map does not list arrives stamped as None. Its units + must land in untracked_units with no cost, so the row never reads as free, + while the priced counter on the same request keeps its cost.""" prisma = _prisma() logs = [ _payload( @@ -393,20 +394,21 @@ async def test_counter_the_hook_could_not_price_is_stored_unknown_not_free(): ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "someFutureCounter"): 3, } costs = _cost_upserts(prisma) - assert costs["contentPolicyUnits"] == (pytest.approx(0.15), {"increment": pytest.approx(0.15)}) - assert costs["someFutureCounter"] == (None, None) + assert costs["contentPolicyUnits"] == (pytest.approx(0.15), 0) + assert costs["someFutureCounter"] == (0.0, 3) @pytest.mark.asyncio -async def test_unpriced_increment_makes_the_rows_cost_unknown_not_partial(): - """A payload with usage but no per-counter cost (a hook without pricing, a - pre-upgrade proxy in a mixed fleet) must poison that row's cost to NULL on - both create and update. Keeping the priced part would understate the day - while looking exact.""" +async def test_mixed_priced_and_unpriced_increments_keep_the_subtotal_and_count_the_rest_untracked(): + """Priced and unpriced increments on the same row (a hook without pricing, + a pre-upgrade proxy in a mixed fleet) must keep the priced subtotal and + count exactly the unpriced units as untracked. Nulling the cost would throw + away a known number; keeping it alone would look exact while understating.""" prisma = _prisma() logs = [ _payload("r1", usage={"contentPolicyUnits": 1000}, cost_by_unit={"contentPolicyUnits": 0.15}), - _payload("r2", usage={"contentPolicyUnits": 1000}), + _payload("r2", usage={"contentPolicyUnits": 700}), + _payload("r3", usage={"contentPolicyUnits": 300}, cost_by_unit={"contentPolicyUnits": None}), ] await process_spend_logs_guardrail_usage(prisma, logs) @@ -414,7 +416,7 @@ async def test_unpriced_increment_makes_the_rows_cost_unknown_not_partial(): assert _units_upserts(prisma) == { ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "contentPolicyUnits"): 2000, } - assert _cost_upserts(prisma) == {"contentPolicyUnits": (None, None)} + assert _cost_upserts(prisma) == {"contentPolicyUnits": (pytest.approx(0.15), 1000)} @pytest.mark.asyncio @@ -438,16 +440,17 @@ async def test_report_only_and_forged_costs_are_not_rolled_up_but_units_are(): ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "topicPolicyUnits"): 10, } assert _cost_upserts(prisma) == { - "text_records": (None, None), - "contentPolicyUnits": (None, None), - "topicPolicyUnits": (None, None), + "text_records": (0.0, 3), + "contentPolicyUnits": (0.0, 10), + "topicPolicyUnits": (0.0, 10), } @pytest.mark.asyncio async def test_requeued_cost_is_added_to_the_next_flush(): - """Cost must survive the connection-error requeue the same way units do, or - a DB blip would silently drop dollars while keeping the units they bought.""" + """Cost and untracked units must survive the connection-error requeue the + same way units do, or a DB blip would silently drop dollars (or the record + that some units had no price) while keeping the units themselves.""" pending = PendingRollups() down = _prisma() down.db.litellm_dailyguardrailmetrics.upsert.side_effect = httpx.ConnectError("db down") @@ -456,19 +459,34 @@ async def test_requeued_cost_is_added_to_the_next_flush(): await process_spend_logs_guardrail_usage( down, - [_payload("r1", usage={"contentPolicyUnits": 1000}, cost_by_unit={"contentPolicyUnits": 0.15})], + [ + _payload( + "r1", + usage={"contentPolicyUnits": 1000, "someFutureCounter": 3}, + cost_by_unit={"contentPolicyUnits": 0.15, "someFutureCounter": None}, + ) + ], sleep=sleep, pending=pending, ) recovered = _prisma() await process_spend_logs_guardrail_usage( recovered, - [_payload("r2", usage={"contentPolicyUnits": 2000}, cost_by_unit={"contentPolicyUnits": 0.3})], + [ + _payload( + "r2", + usage={"contentPolicyUnits": 2000, "someFutureCounter": 4}, + cost_by_unit={"contentPolicyUnits": 0.3, "someFutureCounter": None}, + ) + ], sleep=sleep, pending=pending, ) assert _units_upserts(recovered) == { ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "contentPolicyUnits"): 3000, + ("bedrock-guard", "2026-08-17", "team-a", "hashed-key-1", "someFutureCounter"): 7, } - assert _cost_upserts(recovered)["contentPolicyUnits"][0] == pytest.approx(0.45) + costs = _cost_upserts(recovered) + assert costs["contentPolicyUnits"] == (pytest.approx(0.45), 0) + assert costs["someFutureCounter"] == (0.0, 7) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index b21bb523aa5..ee5edf2d98c 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -37911,7 +37911,7 @@ export interface components { avgScore: number | null; /** * Cost - * @description USD billed for usageUnits over the window, summed over days with tracked cost; null when none have it + * @description USD for the priced share of usageUnits over the window; null when no unit was priced */ cost: number | null; /** Failrate */ @@ -37932,7 +37932,7 @@ export interface components { type: string; /** * Untrackedusageunits - * @description The share of usageUnits that cost leaves out: units from days with no tracked cost, per counter + * @description The share of usageUnits that cost leaves out: units recorded with no known price, per counter */ untrackedUsageUnits: { [key: string]: number; From 6dff3a5f7280d6bcfa3e050548f446b55f0886ed Mon Sep 17 00:00:00 2001 From: tin-berri Date: Fri, 4 Sep 2026 15:11:50 -0700 Subject: [PATCH 88/88] fix(complexity_router): fall back to a live peer when the decided tier model is fully cooled down (#39675) A complexity tier can name several model groups, but the pool pick and the session-pin replay both returned a group without consulting deployment health, so a group whose every deployment was in cooldown was still routed to and the request died at the router's zero-deployment check while a healthy peer sat in the same tier. Gate the decided response at the pre-routing hook's exits, the seam the modality gate already occupies, so every arm that can place a request is covered by one owner: a fresh classification, a replayed or escalated pin, a plan-mode floor, a context-window escalation, an adaptive pick, and whatever arm is added next. Peers come from the decided tier only. Climbing to a higher tier costs more than the classifier asked for and is left to a follow-up. The gate fails open on every uncertainty: an unreadable cooldown view, a decision carrying no tier, a group the router knows no deployments for, or a tier whose peers are all cooling. --- .../complexity_router/complexity_router.py | 199 +++++- litellm/types/utils.py | 4 + .../router_strategy/test_complexity_router.py | 575 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 4 files changed, 762 insertions(+), 18 deletions(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 7dbb2ddc544..1a6e451730e 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -35,7 +35,10 @@ from litellm.constants import ( SESSION_ID_GENERATED_METADATA_KEY, ) from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs +from litellm.litellm_core_utils.core_helpers import ( + _get_parent_otel_span_from_kwargs, + get_metadata_variable_name_from_kwargs, +) from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal_call_metadata from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload @@ -765,6 +768,11 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo seconds against a TTL of an hour that every later turn refreshes. Its cause is whatever the fallback path reports, so the circuit signal is what marks the decision, and leaving it unpinned lets the session classify again as soon as the breaker closes. + + A health failover describes the fleet's state right now, not the session's traffic, and it can + displace decisions that were themselves unpinnable (a housekeeping call, a modality escalation). + Pinning it would hold the session on the substitute long after the displaced group recovers; the + gate re-fires per request, so leaving it unpinned costs nothing but the classifier call. """ return decision is None or ( decision.get("cause") @@ -774,6 +782,7 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo "housekeeping", "modality_escalation", "modality_pin_override", + "health_failover", ) and not decision.get("context_escalated") and _CLASSIFIER_CIRCUIT_OPEN_SIGNAL not in (decision.get("signals") or ()) @@ -2650,6 +2659,150 @@ class ComplexityRouter(CustomLogger): and self._matched_plan_mode_signal(request_kwargs, resolved_messages) is None ) + async def _model_group_can_serve( + self, + model_name: str, + messages: list[dict[str, Any]] | None, # mutable-ok: forwarded verbatim to the router's own probe + input: str | list | None, # mutable-ok: mirrors the owner's own input parameter, which this forwards verbatim + request_kwargs: dict, # mutable-ok: same shape the hook receives + ) -> bool: + """Whether the router would find a deployment for this group ON THIS REQUEST. + + Asks the same owner the routing path itself will ask, with the same prompt arguments it + will pass, so every filter that decides a deployment's eligibility applies here exactly + as it applies downstream: cooldowns, admin pause, team scoping, model access groups, tag + routing, routing plugins, RPM limits, and the context-window pre-call check. Re-deriving + any subset of that list is how a substitute gets chosen that the pipeline then rejects, + and dropping `input` would silently skip the window check on the Responses API surface, + where the prompt never arrives as messages. + + Probed on a COPY of request_kwargs because the owner pops routing bookkeeping off the + dict it is handed (`_target_order`, `_excluded_deployment_ids`), and this is a + speculative question about a model that may never be picked. + + Every way the owner says "nothing here can serve this" is a negative verdict: no healthy + deployment for the group at all (BadRequestError, which ContextWindowExceededError + subclasses), every deployment filtered out (RouterRateLimitError), and every deployment + over its RPM (RouterRateLimitErrorBasic). Anything else is unknown rather than negative, + so it reads as capacity: absent information must never decide the verdict. + """ + from litellm.exceptions import BadRequestError + from litellm.types.router import RouterRateLimitError, RouterRateLimitErrorBasic + + probe_kwargs: Final = dict(request_kwargs) # mutable-ok: the owner pops routing keys off the dict it is handed + try: + deployments: Final = await self.litellm_router_instance.async_get_healthy_deployments( + model=model_name, + request_kwargs=probe_kwargs, + messages=messages, + input=input, + parent_otel_span=_get_parent_otel_span_from_kwargs(request_kwargs), + ) + except (RouterRateLimitError, RouterRateLimitErrorBasic, BadRequestError): + return False + except Exception as exc: # noqa: BLE001 # a speculative eligibility read must fail open on unknown faults + verbose_router_logger.debug( + "ComplexityRouter: eligibility probe for %s failed, treating the group as live: %s", model_name, exc + ) + return True + return bool(deployments) + + async def _gate_response_health( + self, + response: PreRoutingHookResponse, + messages: list[dict[str, Any]] | None, # mutable-ok: forwarded verbatim to the list-typed re-pick + input: str | list | None, # mutable-ok: mirrors the owner's own input parameter, which this forwards verbatim + resolved_messages: Sequence[Mapping[str, object]] | None, + request_kwargs: dict, # mutable-ok: same shape the hook receives + ) -> PreRoutingHookResponse: + """Replace a decided model group that has no serving capacity with a live peer in the same tier. + + Applied to the decided response at the hook's exits, so every arm that can place a request + is covered by one owner: a fresh classification, a replayed or escalated session pin, a + plan-mode floor, a context-window escalation, an adaptive pick, and whatever arm is added + next. Peers come from the DECIDED tier only; climbing to another tier is deliberately not + done here, since a higher tier costs more than the classifier asked for. + + Serving capacity is one question asked of one owner (`_model_group_can_serve`), so the + substitute is only ever a group the pipeline would actually accept for this request. The + pick then runs through `_pick_model_for_tier`, so routing plugins decide the substitute + exactly as they decided the original. + + Fails open everywhere it cannot be sure: an unreadable eligibility view, a decision + carrying no tier (default_model), or a tier whose every peer is unusable too. It fails + CLOSED on a plugin that empties the pool, leaving the original decision to fail rather + than serving a model the plugin excluded. + """ + decision: Final = response.routing_decision + decided_tier: Final = decision.get("tier") if decision is not None else None + if decision is None or not isinstance(decided_tier, str): + return response + peers: Final = tuple(self._tier_pools().get(decided_tier, ())) + if len(peers) < 2: + return response + if await self._model_group_can_serve(response.model, messages, input, request_kwargs): + return response + eligible: Final = ( + self._modality_eligible_models() + if self.config.modality_routing and resolved_messages and request_contains_image_content(resolved_messages) + else None + ) + candidates: Final = tuple( + peer for peer in peers if peer != response.model and (eligible is None or peer in eligible) + ) + if not candidates: + return response + servable: Final = await asyncio.gather( + *(self._model_group_can_serve(peer, messages, input, request_kwargs) for peer in candidates) + ) + live: Final = tuple(peer for peer, can_serve in zip(candidates, servable) if can_serve) + if not live: + return response + repick_messages: Final = ( + list(resolved_messages) if resolved_messages else None # mutable-ok: the pick's param is list-typed + ) + try: + new_model: Final = await self._pick_model_for_tier( + decided_tier if self.config.has_custom_tiers else ComplexityTier(decided_tier), + messages, + repick_messages, # pyright: ignore[reportArgumentType] # hook-resolved message dicts; the pick only reads them + request_kwargs, + allowed_models=live, + ) + except ValueError as exc: + verbose_router_logger.debug( + "ComplexityRouter: health failover found no candidate the routing plugins allow: %s", exc + ) + return response + self._restamp_adaptive_choice(request_kwargs, response.model, new_model) + verbose_router_logger.info( + "ComplexityRouter: routing decision cause=health_failover, routed_model=%s, displaced=%s", + new_model, + response.model, + ) + new_decision: Final = self._build_routing_decision( + routed_model=new_model, + cause="health_failover", + tier=decision.get("tier"), + score=decision.get("score"), + signals=(*(decision.get("signals") or ()), f"health_displaced:{response.model}"), + matched_keyword=decision.get("matched_keyword"), + escalation_keyword=decision.get("escalation_keyword"), + escalated=bool(decision.get("escalated", False)), + classifier_model=decision.get("classifier_model"), + classifier_cost=decision.get("classifier_cost"), + conversation_continuing=bool(decision.get("conversation_continuing", True)), + tier_litellm_params=self._litellm_params_for_model(decided_tier, new_model), + context_escalation_original_tier=decision.get("context_escalation_original_tier"), + ) + return response.model_copy( + update={ # mutable-ok: model_copy types update as a plain dict + "model": new_model, + "litellm_params": self._litellm_params_for_model(decided_tier, new_model), + "routing_decision": new_decision, + } + ) + def _placed_default_model(self) -> str: """The default_model behind a usable-default verdict; the raise is the type-level proof, not a reachable path.""" @@ -3047,24 +3200,30 @@ class ComplexityRouter(CustomLogger): session_tier_litellm_params: Final = self._litellm_params_for_model(routed_pin_tier, routed_model) has_original_messages: Final = messages is not None and len(messages) > 0 return self._with_session_deployment_affinity( - await self._gate_response_modality( - PreRoutingHookResponse( - model=routed_model, - messages=messages if has_original_messages else None, - litellm_params=session_tier_litellm_params, - routing_decision=self._build_routing_decision( - routed_model=routed_model, - cause=cause, - tier=routed_pin_tier, - matched_keyword=pin_plan_sentinel if plan_floored else None, - escalation_keyword=pin_escalation_keyword, - escalated=escalated, - conversation_continuing=conversation_continuing, - tier_litellm_params=session_tier_litellm_params, - context_escalation_original_tier=pin_context_original_tier, + await self._gate_response_health( + await self._gate_response_modality( + PreRoutingHookResponse( + model=routed_model, + messages=messages if has_original_messages else None, + litellm_params=session_tier_litellm_params, + routing_decision=self._build_routing_decision( + routed_model=routed_model, + cause=cause, + tier=routed_pin_tier, + matched_keyword=pin_plan_sentinel if plan_floored else None, + escalation_keyword=pin_escalation_keyword, + escalated=escalated, + conversation_continuing=conversation_continuing, + tier_litellm_params=session_tier_litellm_params, + context_escalation_original_tier=pin_context_original_tier, + ), ), + messages, + resolved_messages, + request_kwargs, ), messages, + input, resolved_messages, request_kwargs, ) @@ -3080,7 +3239,13 @@ class ComplexityRouter(CustomLogger): resolved_messages=resolved_messages, ) response: Final = ( - await self._gate_response_modality(routed_response, messages, resolved_messages, request_kwargs) + await self._gate_response_health( + await self._gate_response_modality(routed_response, messages, resolved_messages, request_kwargs), + messages, + input, + resolved_messages, + request_kwargs, + ) if routed_response is not None else None ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c1d90694f14..26a132cec2d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2886,6 +2886,10 @@ RoutingDecisionCause = Literal[ # carries an image the pinned model cannot accept. The stored pin is untouched, so the next # text turn replays it. Distinct from "modality_escalation", which never displaces a pin. "modality_pin_override", + # Every deployment behind the decided model group was in cooldown, so a healthy peer in the + # same tier served instead. The displaced group rides in signals. Reported even on a kept + # session pin, since the pinned model did not serve the request. + "health_failover", "session_affinity_pin", "session_affinity_escalation", # classification_mode 'user_turn': the request is an agent loop's continuation turn (no new diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 3ce4b8be6f6..57ee74f04ed 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -11571,3 +11571,578 @@ class TestModalityRouting: model="m", request_kwargs={"metadata": {"session_id": "s1"}}, messages=self.IMAGE_MESSAGE ) assert cache.async_set_cache.await_args.kwargs["value"] == {"model": "text-cheap", "tier": "SIMPLE"} + + +class TestTierHealthFailover: + """A tier whose decided model group is entirely in cooldown falls back to a live peer.""" + + SIMPLE_MESSAGE = [{"role": "user", "content": "Hello!"}] + TIERS = {"SIMPLE": ["dead-a", "live-b"], "MEDIUM": "mid", "COMPLEX": "big", "REASONING": "top"} + + @staticmethod + def _router( + mock_router_instance, + config, + ids_by_model, + cooling=(), + blocked=(), + excluded=(), + raises_for=None, + health_error=None, + ): + """ids_by_model: model group -> deployment ids the router knows. + + The fake mirrors the real async_get_healthy_deployments contract, including how it says + no: BadRequestError for a group with no deployment at all, RouterRateLimitError when every + deployment is filtered out (cooling, admin-paused, or excluded by a request-scoped policy + such as tags, team scoping or access groups), a per-model exception via raises_for (the + RPM verdict), and an unrelated failure via health_error. It records what it was handed so + tests can prove the probe passes a kwargs copy and forwards the prompt arguments. + """ + import litellm as litellm_module + + from litellm.types.router import RouterRateLimitError + + probed_kwargs = [] + probed_prompts = [] + + async def get_healthy_deployments( + model, request_kwargs, messages=None, input=None, parent_otel_span=None, **kwargs + ): + probed_kwargs.append(request_kwargs) + probed_prompts.append((messages, input)) + if health_error is not None: + raise health_error + if raises_for and model in raises_for: + raise raises_for[model] + if not ids_by_model.get(model): + raise litellm_module.BadRequestError( + message=f"You passed in model={model}. There are no healthy deployments.", + model=model, + llm_provider="", + ) + filtered = (*cooling, *blocked, *excluded) + healthy = [ + {"model_name": model, "model_info": {"id": i}} for i in ids_by_model[model] if i not in filtered + ] + if not healthy: + raise RouterRateLimitError( + model=model, cooldown_time=60.0, enable_pre_call_checks=False, cooldown_list=[] + ) + return healthy + + mock_router_instance.async_get_healthy_deployments = get_healthy_deployments + mock_router_instance.probed_kwargs = probed_kwargs + mock_router_instance.probed_prompts = probed_prompts + mock_router_instance.cache = DualCache() + return ComplexityRouter( + model_name="health-test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=config, + ) + + async def _pinned_hook(self, router, session_id="sess-1", messages=None): + """Drive the hook twice so the second call replays a pin, which makes the decided + model deterministic instead of a coin flip over the tier pool.""" + kwargs = {"metadata": {"session_id": session_id}} + await router.async_pre_routing_hook(model="m", request_kwargs=kwargs, messages=messages or self.SIMPLE_MESSAGE) + return await router.async_pre_routing_hook( + model="m", request_kwargs=kwargs, messages=messages or self.SIMPLE_MESSAGE + ) + + @pytest.mark.asyncio + async def test_dead_pinned_group_fails_over_to_live_peer_and_reports_the_displacement(self, mock_router_instance): + """The core regression: a session pinned to a group whose every deployment is cooling + serves from the live peer, and the row says so rather than naming the pinned model.""" + router = self._router( + mock_router_instance, + {"tiers": dict(self.TIERS), "session_affinity": True}, + {"dead-a": ["id-a1", "id-a2"], "live-b": ["id-b1"]}, + cooling=("id-a1", "id-a2"), + ) + # Seed the pin onto the dead group directly so the replay path is exercised. + key = router._get_session_affinity_cache_key("sess-dead", {}) + await router.litellm_router_instance.cache.async_set_cache( + key=key, value={"model": "dead-a", "tier": "SIMPLE"}, ttl=600 + ) + result = await router.async_pre_routing_hook( + model="m", request_kwargs={"metadata": {"session_id": "sess-dead"}}, messages=self.SIMPLE_MESSAGE + ) + assert result.model == "live-b" + assert result.routing_decision["cause"] == "health_failover" + assert "health_displaced:dead-a" in result.routing_decision["signals"] + assert result.routing_decision["tier"] == "SIMPLE" + + @pytest.mark.asyncio + async def test_fresh_classification_never_serves_a_fully_cooled_group(self, mock_router_instance): + """The pool pick is a uniform draw, so the invariant is asserted over repeated turns: + no turn may land on the dead group while a live peer sits in the same tier.""" + router = self._router( + mock_router_instance, + {"tiers": dict(self.TIERS)}, + {"dead-a": ["id-a1"], "live-b": ["id-b1"]}, + cooling=("id-a1",), + ) + results = [ + await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self.SIMPLE_MESSAGE) + for _ in range(20) + ] + assert {r.model for r in results} == {"live-b"} + assert all(r.routing_decision["cause"] in ("heuristic_scorer", "health_failover") for r in results) + assert any(r.routing_decision["cause"] == "health_failover" for r in results) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "ids_by_model, cooling, health_error, tiers, reason", + [ + ({"dead-a": ["id-a1"], "live-b": ["id-b1"]}, (), None, None, "nothing_cooling"), + ({"dead-a": ["id-a1"], "live-b": ["id-b1"]}, ("id-a1", "id-b1"), None, None, "every_peer_dead"), + ( + {"dead-a": ["id-a1"], "live-b": ["id-b1"]}, + ("id-a1",), + RuntimeError("redis down"), + None, + "health_view_unreadable", + ), + ( + {"only": ["id-1"]}, + ("id-1",), + None, + {"SIMPLE": "only", "MEDIUM": "mid", "COMPLEX": "big", "REASONING": "top"}, + "single_model_tier_has_no_peer", + ), + ], + ) + async def test_gate_fails_open_and_leaves_the_decision_untouched( + self, mock_router_instance, ids_by_model, cooling, health_error, tiers, reason + ): + """Every uncertainty leaves the decided model in place, so the request fails exactly + as it does today rather than being rerouted on a guess.""" + router = self._router( + mock_router_instance, + {"tiers": dict(tiers or self.TIERS), "session_affinity": True}, + ids_by_model, + cooling=cooling, + health_error=health_error, + ) + pinned = "only" if tiers else "dead-a" + key = router._get_session_affinity_cache_key("sess-open", {}) + await router.litellm_router_instance.cache.async_set_cache( + key=key, value={"model": pinned, "tier": "SIMPLE"}, ttl=600 + ) + result = await router.async_pre_routing_hook( + model="m", request_kwargs={"metadata": {"session_id": "sess-open"}}, messages=self.SIMPLE_MESSAGE + ) + assert result.model == pinned, reason + assert result.routing_decision["cause"] == "session_affinity_pin", reason + + @pytest.mark.asyncio + async def test_a_failed_over_turn_is_never_pinned(self, mock_router_instance): + """A failover describes the fleet's state, not the session's traffic, so it must not + become the pin: the substitute would outlive the outage that caused it. + + Asserted over many sessions because the underlying pool pick is a uniform draw. + """ + router = self._router( + mock_router_instance, + {"tiers": dict(self.TIERS), "session_affinity": True}, + {"dead-a": ["id-a1"], "live-b": ["id-b1"]}, + cooling=("id-a1",), + ) + + async def pin_after_session(turn: int): + session_id = f"sess-write-{turn}" + await router.async_pre_routing_hook( + model="m", + request_kwargs={"metadata": {"session_id": session_id}}, + messages=self.SIMPLE_MESSAGE, + ) + return await router.litellm_router_instance.cache.async_get_cache( + key=router._get_session_affinity_cache_key(session_id, {}) + ) + + stored = [await pin_after_session(turn) for turn in range(20)] + assert all(entry in (None, {"model": "live-b", "tier": "SIMPLE"}) for entry in stored) + assert any(entry is None for entry in stored), "a failed-over turn must leave the pin unwritten" + + @pytest.mark.asyncio + async def test_an_unpinnable_displaced_cause_stays_unpinnable_after_failover(self, mock_router_instance): + """A housekeeping turn is deliberately never pinned. Rewriting its cause to health_failover + must not smuggle it past that guard and lock the session onto the cheapest tier.""" + router = self._router( + mock_router_instance, + {"tiers": dict(self.TIERS), "session_affinity": True}, + {"dead-a": ["id-a1"], "live-b": ["id-b1"]}, + cooling=("id-a1",), + ) + session_id = "sess-housekeeping" + result = await router.async_pre_routing_hook( + model="m", + request_kwargs={"metadata": {"session_id": session_id}}, + messages=[{"role": "user", "content": TITLE_ASK}], + ) + assert result.routing_decision["cause"] in ("housekeeping", "health_failover") + stored = await router.litellm_router_instance.cache.async_get_cache( + key=router._get_session_affinity_cache_key(session_id, {}) + ) + assert stored is None + + @pytest.mark.asyncio + async def test_a_peer_whose_deployments_are_admin_paused_is_not_a_failover_target(self, mock_router_instance): + """Capacity is the router's own verdict, not just cooldown: a paused peer would be + rejected downstream and the request would fail with a live third peer available.""" + router = self._router( + mock_router_instance, + { + "tiers": { + "SIMPLE": ["dead-a", "paused-b", "live-c"], + "MEDIUM": "mid", + "COMPLEX": "big", + "REASONING": "top", + }, + "session_affinity": True, + }, + {"dead-a": ["id-a1"], "paused-b": ["id-b1"], "live-c": ["id-c1"]}, + cooling=("id-a1",), + blocked=("id-b1",), + ) + key = router._get_session_affinity_cache_key("sess-paused", {}) + await router.litellm_router_instance.cache.async_set_cache( + key=key, value={"model": "dead-a", "tier": "SIMPLE"}, ttl=600 + ) + results = [ + await router.async_pre_routing_hook( + model="m", request_kwargs={"metadata": {"session_id": "sess-paused"}}, messages=self.SIMPLE_MESSAGE + ) + for _ in range(20) + ] + assert {r.model for r in results} == {"live-c"} + + @pytest.mark.asyncio + async def test_failover_fails_closed_when_a_routing_plugin_excludes_every_peer(self, mock_router_instance): + """A plugin's exclusion is policy, so a peer it removed must not be served just because + the plugin's own choice went into cooldown.""" + + class ExcludeEverythingButDead: + async def run(self, context): + context.candidate_models = [m for m in context.candidate_models if m == "dead-a"] + return context + + router = self._router( + mock_router_instance, + {"tiers": dict(self.TIERS), "plugins": [ExcludeEverythingButDead()]}, + {"dead-a": ["id-a1"], "live-b": ["id-b1"]}, + cooling=("id-a1",), + ) + result = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self.SIMPLE_MESSAGE) + assert result.model == "dead-a" + assert result.routing_decision["cause"] != "health_failover" + + @pytest.mark.asyncio + async def test_failover_moves_the_adaptive_chosen_model_marker(self, mock_router_instance): + """The adaptive feedback loop scores the marker, so leaving it on the displaced group + would credit a model that never ran.""" + router = self._router( + mock_router_instance, + {"tiers": dict(self.TIERS), "session_affinity": True}, + {"dead-a": ["id-a1"], "live-b": ["id-b1"]}, + cooling=("id-a1",), + ) + key = router._get_session_affinity_cache_key("sess-adaptive", {}) + await router.litellm_router_instance.cache.async_set_cache( + key=key, value={"model": "dead-a", "tier": "SIMPLE"}, ttl=600 + ) + request_kwargs = {"metadata": {"session_id": "sess-adaptive", "adaptive_router_chosen_model": "dead-a"}} + result = await router.async_pre_routing_hook( + model="m", request_kwargs=request_kwargs, messages=self.SIMPLE_MESSAGE + ) + assert result.model == "live-b" + assert request_kwargs["metadata"]["adaptive_router_chosen_model"] == "live-b" + + @pytest.mark.asyncio + async def test_health_failover_never_undoes_the_modality_gate(self, mock_router_instance): + """An image turn whose only live peer cannot take images keeps the vision model the + modality gate chose: serving a cooling vision model beats a hard 400.""" + vision_by_model = {"dead-vision": True, "live-text": False} + + def get_model_list(model_name=None): + if model_name not in vision_by_model: + return [] + return [ + { + "model_name": model_name, + "litellm_params": {"model": f"openai/unmapped-{model_name}"}, + "model_info": {"supports_vision": vision_by_model[model_name]}, + } + ] + + mock_router_instance.get_model_list = get_model_list + router = self._router( + mock_router_instance, + { + "tiers": { + "SIMPLE": ["dead-vision", "live-text"], + "MEDIUM": "mid", + "COMPLEX": "big", + "REASONING": "top", + }, + "session_affinity": True, + "modality_routing": True, + }, + {"dead-vision": ["id-v1"], "live-text": ["id-t1"]}, + cooling=("id-v1",), + ) + key = router._get_session_affinity_cache_key("sess-image", {}) + await router.litellm_router_instance.cache.async_set_cache( + key=key, value={"model": "dead-vision", "tier": "SIMPLE"}, ttl=600 + ) + image_message = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What color is this?"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGk="}}, + ], + } + ] + result = await router.async_pre_routing_hook( + model="m", request_kwargs={"metadata": {"session_id": "sess-image"}}, messages=image_message + ) + assert result.model == "dead-vision" + + @pytest.mark.asyncio + async def test_failover_will_not_pick_a_peer_that_cannot_hold_the_prompt(self): + """The context-window filter is a pre-call check inside the eligibility owner, so this + drives the REAL owner on a real Router and injects only the cooldown. A substitute the + prompt overflows must never be chosen while a peer that holds it exists.""" + pool = ["dead-big", "live-small", "live-big"] + router_instance = _windowed_router( + ("dead-big", "openai/gpt-4o-mini", 200000), + ("live-small", "openai/gpt-3.5-turbo", 16385), + ("live-big", "openai/gpt-4o-mini", 200000), + ) + router_instance.enable_pre_call_checks = True + dead_ids = {d["model_info"]["id"] for d in router_instance.model_list if d["model_name"] == "dead-big"} + + async def active_cooldowns(model_ids, parent_otel_span): + return [(i, {"exception_received": "boom"}) for i in model_ids if i in dead_ids] + + router_instance.cooldown_cache.async_get_active_cooldowns = active_cooldowns + router_instance.cache = DualCache() + router = ComplexityRouter( + model_name="health-window-router", + litellm_router_instance=router_instance, + complexity_router_config={ + "tiers": {name: list(pool) for name in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING")}, + "session_affinity": True, + "enable_context_window_escalation": True, + }, + ) + key = router._get_session_affinity_cache_key("sess-window", {}) + await router.litellm_router_instance.cache.async_set_cache( + key=key, value={"model": "dead-big", "tier": "SIMPLE"}, ttl=600 + ) + results = [ + await router.async_pre_routing_hook( + model="m", + request_kwargs={"metadata": {"session_id": "sess-window"}}, + messages=list(_OVERSIZED_TURNS), + ) + for _ in range(20) + ] + assert "live-small" not in {r.model for r in results} + assert {r.model for r in results} == {"live-big"} + + @pytest.mark.asyncio + async def test_a_decision_with_no_tier_is_left_alone(self, mock_router_instance): + """default_model placements carry no tier, so there is no pool to draw a peer from. + The gate leaves them exactly as they are rather than inventing a tier.""" + router = self._router( + mock_router_instance, + { + "tiers": dict(self.TIERS), + "default_model": "fallback-model", + "classifier_type": "llm", + "classifier_llm_config": {"model": "gpt-4o-mini"}, + "classifier_fallback": "default_model", + }, + {"fallback-model": ["id-f1"], "dead-a": ["id-a1"], "live-b": ["id-b1"]}, + cooling=("id-f1", "id-a1"), + ) + mock_router_instance.acompletion = AsyncMock(side_effect=RuntimeError("classifier down")) + result = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self.SIMPLE_MESSAGE) + assert result.model == "fallback-model" + assert result.routing_decision.get("tier") is None + assert result.routing_decision["cause"] != "health_failover" + + @pytest.mark.asyncio + async def test_a_tier_entry_the_router_cannot_serve_fails_over_instead_of_erroring(self, mock_router_instance): + """A tier naming a model this proxy has no deployment for is unservable, and the + eligibility owner says so, so the peer serves rather than the request 429ing.""" + router = self._router( + mock_router_instance, + {"tiers": dict(self.TIERS), "session_affinity": True}, + {"live-b": ["id-b1"]}, + ) + key = router._get_session_affinity_cache_key("sess-unknown", {}) + await router.litellm_router_instance.cache.async_set_cache( + key=key, value={"model": "dead-a", "tier": "SIMPLE"}, ttl=600 + ) + result = await router.async_pre_routing_hook( + model="m", request_kwargs={"metadata": {"session_id": "sess-unknown"}}, messages=self.SIMPLE_MESSAGE + ) + assert result.model == "live-b" + assert result.routing_decision["cause"] == "health_failover" + + @pytest.mark.asyncio + async def test_a_peer_excluded_by_a_request_scoped_policy_is_not_a_failover_target(self, mock_router_instance): + """Tag, team and access-group filters are request-scoped and live inside the eligibility + owner. A peer they exclude would be rejected downstream, so it must not be chosen.""" + router = self._router( + mock_router_instance, + { + "tiers": { + "SIMPLE": ["dead-a", "tagged-out-b", "live-c"], + "MEDIUM": "mid", + "COMPLEX": "big", + "REASONING": "top", + }, + "session_affinity": True, + }, + {"dead-a": ["id-a1"], "tagged-out-b": ["id-b1"], "live-c": ["id-c1"]}, + cooling=("id-a1",), + excluded=("id-b1",), + ) + key = router._get_session_affinity_cache_key("sess-tagged", {}) + await router.litellm_router_instance.cache.async_set_cache( + key=key, value={"model": "dead-a", "tier": "SIMPLE"}, ttl=600 + ) + results = [ + await router.async_pre_routing_hook( + model="m", request_kwargs={"metadata": {"session_id": "sess-tagged"}}, messages=self.SIMPLE_MESSAGE + ) + for _ in range(20) + ] + assert {r.model for r in results} == {"live-c"} + + @pytest.mark.asyncio + async def test_the_eligibility_probe_never_mutates_the_caller_request_kwargs(self, mock_router_instance): + """The owner pops routing bookkeeping off the dict it is handed, so a probe that passed + the real kwargs would strip them before the request is ever placed.""" + router = self._router( + mock_router_instance, + {"tiers": dict(self.TIERS), "session_affinity": True}, + {"dead-a": ["id-a1"], "live-b": ["id-b1"]}, + cooling=("id-a1",), + ) + key = router._get_session_affinity_cache_key("sess-kwargs", {}) + await router.litellm_router_instance.cache.async_set_cache( + key=key, value={"model": "dead-a", "tier": "SIMPLE"}, ttl=600 + ) + request_kwargs = { + "metadata": {"session_id": "sess-kwargs"}, + "_target_order": 1, + "_excluded_deployment_ids": ["id-x"], + } + result = await router.async_pre_routing_hook( + model="m", request_kwargs=request_kwargs, messages=self.SIMPLE_MESSAGE + ) + assert result.model == "live-b" + assert request_kwargs["_target_order"] == 1 + assert request_kwargs["_excluded_deployment_ids"] == ["id-x"] + assert all(probed is not request_kwargs for probed in router.litellm_router_instance.probed_kwargs) + + @pytest.mark.asyncio + async def test_a_peer_whose_every_deployment_is_over_its_rpm_is_not_a_failover_target( + self, mock_router_instance + ): + """RPM exhaustion is its own verdict from the owner (RouterRateLimitErrorBasic). A peer + in that state would be rejected downstream, so it cannot be the substitute.""" + from litellm.types.router import RouterRateLimitErrorBasic + + router = self._router( + mock_router_instance, + { + "tiers": { + "SIMPLE": ["dead-a", "rpm-full-b", "live-c"], + "MEDIUM": "mid", + "COMPLEX": "big", + "REASONING": "top", + }, + "session_affinity": True, + }, + {"dead-a": ["id-a1"], "rpm-full-b": ["id-b1"], "live-c": ["id-c1"]}, + cooling=("id-a1",), + raises_for={"rpm-full-b": RouterRateLimitErrorBasic(model="rpm-full-b")}, + ) + key = router._get_session_affinity_cache_key("sess-rpm", {}) + await router.litellm_router_instance.cache.async_set_cache( + key=key, value={"model": "dead-a", "tier": "SIMPLE"}, ttl=600 + ) + results = [ + await router.async_pre_routing_hook( + model="m", request_kwargs={"metadata": {"session_id": "sess-rpm"}}, messages=self.SIMPLE_MESSAGE + ) + for _ in range(20) + ] + assert {r.model for r in results} == {"live-c"} + + @pytest.mark.asyncio + async def test_the_probe_forwards_input_so_window_checks_run_on_input_only_surfaces( + self, mock_router_instance + ): + """The Responses API carries its prompt as `input`, never as messages. The owner only + runs its context-window pre-call check when one of them is present, so dropping `input` + would silently skip window filtering on that whole surface.""" + router = self._router( + mock_router_instance, + {"tiers": dict(self.TIERS), "session_affinity": True}, + {"dead-a": ["id-a1"], "live-b": ["id-b1"]}, + cooling=("id-a1",), + ) + key = router._get_session_affinity_cache_key("sess-input", {}) + await router.litellm_router_instance.cache.async_set_cache( + key=key, value={"model": "dead-a", "tier": "SIMPLE"}, ttl=600 + ) + result = await router.async_pre_routing_hook( + model="m", + request_kwargs={"metadata": {"session_id": "sess-input"}}, + input="summarize this document for me", + ) + assert result.model == "live-b" + assert any( + probed_input == "summarize this document for me" + for _, probed_input in router.litellm_router_instance.probed_prompts + ), "the eligibility probe must forward `input` to the owner" + + @pytest.mark.asyncio + async def test_a_group_the_router_has_no_deployment_for_is_not_a_failover_target( + self, mock_router_instance + ): + """The owner answers an unconfigured group with BadRequestError. Reading that as live + would both skip failover off it and let it be chosen as a substitute.""" + router = self._router( + mock_router_instance, + { + "tiers": { + "SIMPLE": ["dead-a", "unconfigured-b", "live-c"], + "MEDIUM": "mid", + "COMPLEX": "big", + "REASONING": "top", + }, + "session_affinity": True, + }, + {"dead-a": ["id-a1"], "live-c": ["id-c1"]}, + cooling=("id-a1",), + ) + key = router._get_session_affinity_cache_key("sess-missing", {}) + await router.litellm_router_instance.cache.async_set_cache( + key=key, value={"model": "dead-a", "tier": "SIMPLE"}, ttl=600 + ) + results = [ + await router.async_pre_routing_hook( + model="m", request_kwargs={"metadata": {"session_id": "sess-missing"}}, messages=self.SIMPLE_MESSAGE + ) + for _ in range(20) + ] + assert {r.model for r in results} == {"live-c"} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 5f3cca49644..fe1cc2772b4 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -36045,7 +36045,7 @@ export interface components { * Cause * @enum {string} */ - cause?: "heuristic_scorer" | "heuristic_v2" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "hybrid_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "modality_escalation" | "modality_pin_override" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit"; + cause?: "heuristic_scorer" | "heuristic_v2" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "hybrid_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "modality_escalation" | "modality_pin_override" | "health_failover" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit"; /** Classifier Cost */ classifier_cost?: number; /** Classifier Model */