diff --git a/db_scripts/partition_spend_logs.sql b/db_scripts/partition_spend_logs.sql
index 08fcbddb6f8..4e4a93539d7 100644
--- a/db_scripts/partition_spend_logs.sql
+++ b/db_scripts/partition_spend_logs.sql
@@ -10,6 +10,11 @@
-- partitioned, so existing installs are unaffected until you run this.
--
-- IMPORTANT
+-- * After partitioning, `prisma db push` (including the proxy's
+-- --use_prisma_db_push startup mode) is NOT supported: it tries to rewrite
+-- the primary key back to ("request_id"), which Postgres rejects on a
+-- partitioned table. The proxy detects this and exits with guidance.
+-- Use the default startup path (`prisma migrate deploy`) instead.
-- * Test on a staging copy first and take a backup.
-- * Postgres cannot convert a populated table to partitioned in place, so this
-- renames the old table aside and creates a fresh partitioned table.
diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml
index b7a62e52cf9..3653aba67ef 100644
--- a/enterprise/pyproject.toml
+++ b/enterprise/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
-version = "0.1.60"
+version = "0.1.61"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
-version = "0.1.60"
+version = "0.1.61"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",
diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py
index 5118865e43a..b27221c9beb 100644
--- a/litellm-proxy-extras/litellm_proxy_extras/utils.py
+++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py
@@ -40,6 +40,65 @@ def _get_prisma_env() -> dict:
_MIGRATION_TS_RE = re.compile(r"^(\d{14})_")
+_SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE)
+_SPEND_LOGS_ARTIFACT_DROP_RE = re.compile(
+ r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE
+)
+_SPEND_LOGS_PK_CLAUSE_RE = re.compile(
+ r'^(?:DROP\s+CONSTRAINT\s+"[^"]*_pkey"'
+ r'|ADD\s+(?:CONSTRAINT\s+"[^"]*"\s+)?PRIMARY\s+KEY\s*\([^)]*\))$',
+ re.IGNORECASE,
+)
+
+PARTITIONED_SPEND_LOGS_PUSH_ERROR = (
+ "LiteLLM_SpendLogs is a partitioned table (see db_scripts/partition_spend_logs.sql), "
+ "so its primary key must include the partition key (\"startTime\"). `prisma db push` "
+ "reconciles the database against schema.prisma, which declares the unpartitioned "
+ "primary key (\"request_id\"), and Postgres rejects that rewrite with: unique "
+ "constraint on partitioned table must include all partitioning columns. Start the "
+ "proxy without --use_prisma_db_push so it uses `prisma migrate deploy`, which only "
+ "applies shipped migrations and leaves the partitioned primary key alone."
+)
+
+
+def _without_sql_comments(statement: str) -> str:
+ return "\n".join(
+ line
+ for line in statement.splitlines()
+ if line.strip() and not line.strip().startswith("--")
+ ).strip()
+
+
+def _without_spend_logs_pk_clauses(statement: str) -> Optional[str]:
+ prefix_match = _SPEND_LOGS_ALTER_RE.match(statement)
+ if not prefix_match:
+ return statement
+ kept = tuple(
+ clause.strip()
+ for clause in statement[prefix_match.end():].split(",\n")
+ if not _SPEND_LOGS_PK_CLAUSE_RE.match(clause.strip())
+ )
+ if not kept:
+ return None
+ return statement[: prefix_match.end()] + ",\n".join(kept)
+
+
+def filter_partitioned_spend_logs_diff(diff_sql: str) -> str:
+ """Drop statements from a `prisma migrate diff` script that fight the
+ SpendLogs partitioning runbook (db_scripts/partition_spend_logs.sql): the
+ primary-key rewrite on "LiteLLM_SpendLogs", which Postgres rejects on a
+ partitioned table, and drops of runbook artifacts such as
+ "LiteLLM_SpendLogs_legacy"."""
+ kept = tuple(
+ filtered
+ for statement in diff_sql.split(";")
+ for bare in (_without_sql_comments(statement),)
+ if bare and not _SPEND_LOGS_ARTIFACT_DROP_RE.match(bare)
+ for filtered in (_without_spend_logs_pk_clauses(bare),)
+ if filtered is not None
+ )
+ return "".join(f"{statement};\n\n" for statement in kept)
+
def _migration_timestamp(name: str) -> int:
"""Extract the leading `YYYYMMDDHHMMSS` timestamp from a migration name.
@@ -355,7 +414,24 @@ class ProxyExtrasDBManager:
return
logger.info(f"Migration diff created at {diff_sql_path}")
+ if ProxyExtrasDBManager.spend_logs_is_partitioned():
+ filtered_sql = filter_partitioned_spend_logs_diff(
+ diff_sql_path.read_text()
+ )
+ diff_sql_path.write_text(filtered_sql)
+ logger.info(
+ "LiteLLM_SpendLogs is partitioned; removed its primary-key "
+ "rewrite and partitioning artifacts from the drift script"
+ )
+ if not filtered_sql.strip():
+ logger.info("Drift script is empty after filtering; nothing to apply")
+ if not mark_all_applied:
+ return
+ ProxyExtrasDBManager._mark_migrations_applied(migrations_dir)
+ return
+
# 2. Run prisma db execute to apply the migration
+ applied_ok = False
try:
logger.info("Running prisma db execute to apply the migration diff...")
result = subprocess.run(
@@ -376,6 +452,7 @@ class ProxyExtrasDBManager:
)
logger.info(f"prisma db execute stdout: {result.stdout}")
logger.info("✅ Migration diff applied successfully")
+ applied_ok = True
except subprocess.CalledProcessError as e:
logger.warning(f"Failed to apply migration diff: {e.stderr}")
except subprocess.TimeoutExpired:
@@ -384,6 +461,16 @@ class ProxyExtrasDBManager:
# 3. Mark all migrations as applied
if not mark_all_applied:
return
+ if not applied_ok:
+ logger.warning(
+ "Drift script failed to apply; NOT marking migrations as "
+ "applied so a later migration run can retry them"
+ )
+ return
+ ProxyExtrasDBManager._mark_migrations_applied(migrations_dir)
+
+ @staticmethod
+ def _mark_migrations_applied(migrations_dir: str):
migration_names = ProxyExtrasDBManager._get_migration_names(migrations_dir)
logger.info(f"Resolving {len(migration_names)} migrations")
for migration_name in migration_names:
@@ -410,6 +497,55 @@ class ProxyExtrasDBManager:
f"Failed to resolve migration {migration_name}: {e.stderr}"
)
+ @staticmethod
+ def spend_logs_is_partitioned() -> bool:
+ """True when the connected database's LiteLLM_SpendLogs is a
+ partitioned table in Prisma's target schema (the `schema` URL param,
+ falling back to Prisma's default target, public), i.e. the operator
+ ran db_scripts/partition_spend_logs.sql. Returns False when psycopg is
+ unavailable or the database cannot be reached, preserving the
+ pre-existing behavior in those cases."""
+ database_url = os.getenv("DATABASE_URL")
+ if not database_url:
+ return False
+
+ try:
+ import psycopg
+ except ImportError:
+ return False
+
+ cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
+ try:
+ with psycopg.connect(
+ cleaned_url, connect_timeout=10, autocommit=True
+ ) as conn:
+ row = conn.execute(
+ "SELECT 1 "
+ "FROM pg_partitioned_table pt "
+ "JOIN pg_class c ON c.oid = pt.partrelid "
+ "JOIN pg_namespace n ON n.oid = c.relnamespace "
+ "WHERE c.relname = 'LiteLLM_SpendLogs' "
+ " AND n.nspname = %s",
+ (
+ ProxyExtrasDBManager._prisma_schema_param(database_url)
+ or "public",
+ ),
+ ).fetchone()
+ except (psycopg.OperationalError, psycopg.DatabaseError):
+ return False
+ return row is not None
+
+ @staticmethod
+ def _prisma_schema_param(url: str) -> Optional[str]:
+ """The `schema` query param Prisma uses to pick its target schema,
+ or None when the URL does not set one."""
+ from urllib.parse import urlparse, parse_qsl
+
+ return next(
+ (v for k, v in parse_qsl(urlparse(url).query) if k == "schema"),
+ None,
+ )
+
@staticmethod
def _strip_prisma_query_params(url: str) -> str:
"""Remove Prisma-specific query params (connection_limit, pool_timeout,
@@ -528,7 +664,8 @@ class ProxyExtrasDBManager:
migrations_dir = ProxyExtrasDBManager._get_prisma_dir()
if not use_migrate:
- # Preserve `prisma db push` path unchanged.
+ if ProxyExtrasDBManager.spend_logs_is_partitioned():
+ raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
original_dir = os.getcwd()
os.chdir(migrations_dir)
try:
@@ -972,6 +1109,8 @@ class ProxyExtrasDBManager:
)
raise
else:
+ if ProxyExtrasDBManager.spend_logs_is_partitioned():
+ raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
# Use prisma db push with increased timeout
subprocess.run(
[_get_prisma_command(), "db", "push", "--accept-data-loss"],
diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml
index 98a3d8d535e..0ef5cd1e856 100644
--- a/litellm-proxy-extras/pyproject.toml
+++ b/litellm-proxy-extras/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
-version = "0.4.89"
+version = "0.4.90"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
-version = "0.4.89"
+version = "0.4.90"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",
diff --git a/litellm/__init__.py b/litellm/__init__.py
index eebd2dad91e..ec2960c196e 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -445,6 +445,7 @@ max_ui_session_budget: Optional[float] = (
1.0 # USD budget for each dashboard login session (playground, test connection)
)
internal_user_budget_duration: Optional[str] = None
+budget_rollover: bool = False # carry spend beyond max_budget into the next window instead of zeroing it
tag_budget_config: Optional[Dict[str, "BudgetConfig"]] = None
max_end_user_budget: Optional[float] = None
max_end_user_budget_id: Optional[str] = None
diff --git a/litellm/constants.py b/litellm/constants.py
index b2f59bc667c..ed89474600e 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -1652,6 +1652,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
"enable_anthropic_prompt_caching",
"anthropic_prompt_caching_ttl",
"max_ui_session_budget",
+ "budget_rollover",
]
SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"]
DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60))
diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py
index 244e58eddf3..101dbc6538d 100644
--- a/litellm/integrations/otel/emitter.py
+++ b/litellm/integrations/otel/emitter.py
@@ -146,7 +146,7 @@ class SpanEmitter:
For callers that own and manage their own span lifecycle. ``tracer``
overrides the bound tracer for this span only, used for per-request
multi-tenant credential routing. ``links`` records related-but-not-parent
- spans (e.g. the transport span of an MCP message, per MCP semconv).
+ spans (e.g. the trace context an MCP client propagated in ``params._meta``).
"""
return (tracer or self._tracer).start_span(
name,
@@ -196,8 +196,8 @@ class SpanEmitter:
Return the span, or ``None`` if it was deduplicated away. ``tracer``
overrides the bound tracer for this span, used for per-request routing.
- ``links`` records related-but-not-parent spans (the transport span of an
- MCP message).
+ ``links`` records related-but-not-parent spans (e.g. the trace context an
+ MCP client propagated in ``params._meta``).
"""
# LLM-call and MCP tool-call spans carry a dedup key (their request's
# call id), so a sync+async double-firing coalesces. ``isinstance`` narrows
diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py
index 4359b222d06..d2a32ef73b6 100644
--- a/litellm/integrations/otel/logger.py
+++ b/litellm/integrations/otel/logger.py
@@ -390,10 +390,10 @@ class OpenTelemetryV2(CustomLogger):
MCP tool calls reach the success/failure callbacks like any other request
(with ``call_type`` ``call_mcp_tool``), but they are not LLM calls and have
- no ``pre_call`` carrier — so they get their own CLIENT span here. Per the MCP
- semconv it parents to the trace context the client propagated in
- ``params._meta`` (or starts a new root) and links the transport span, rather
- than nesting under the HTTP/session span. Returns whether it handled the
+ no ``pre_call`` carrier — so they get their own CLIENT span here. It nests
+ under the transport span of the request carrying this message, and trace
+ context the client propagated in ``params._meta`` is recorded as a span
+ link (see ``resolve_mcp_span_context``). Returns whether it handled the
event, so the caller skips the LLM-call path. The whole span is emitted at
once (there is no boundary to open it at), deduped on the call id.
"""
@@ -436,9 +436,9 @@ class OpenTelemetryV2(CustomLogger):
Like a tool call, listing reaches the success/failure callbacks (here with
``call_type`` ``list_mcp_tools``) with no ``pre_call`` carrier, so it gets its
- own CLIENT span. Per the MCP semconv it parents to the ``params._meta`` trace
- context (or starts a new root) and links the transport span, rather than
- nesting under the HTTP/session span. Returns whether it handled the event so
+ own CLIENT span, nested under the transport span of the request carrying
+ this message with any ``params._meta`` trace context recorded as a span
+ link (see ``resolve_mcp_span_context``). Returns whether it handled the event so
the caller skips the LLM-call path.
"""
raw_payload: Final = kwargs.get("standard_logging_object")
diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py
index 08318f78b7c..35fc50a2a83 100644
--- a/litellm/integrations/otel/model/spans.py
+++ b/litellm/integrations/otel/model/spans.py
@@ -10,6 +10,8 @@ Canonical hierarchy::
│ └── DB_CALL (CLIENT) # its key/user/team lookups nest here
├── GUARDRAIL (INTERNAL) # request-lifecycle hook, sibling of LLM_CALL
├── LLM_CALL (CLIENT)
+ ├── MCP_TOOL_CALL (CLIENT) # nests under the POST carrying the message
+ ├── MCP_LIST_TOOLS (CLIENT) # (client-propagated context is a span link)
└── DB_CALL (CLIENT) # e.g. the spend-log write
Guardrails parent to PROXY_REQUEST, not LLM_CALL: pre/during/post-call guardrail
@@ -18,14 +20,14 @@ before the LLM call even starts), so a guardrail is a sibling of the LLM call,
not a child of it. The emitter parents every span to the ambient OTel context
(the active server span), which matches this.
-MCP spans (``MCP_TOOL_CALL``, ``MCP_LIST_TOOLS``) have two shapes, chosen at emit
-time by :func:`resolve_mcp_span_context`. When the client propagates trace context
-in ``params._meta`` MCP and the HTTP transport are independent contexts per the
-OTel GenAI MCP semconv, so the span parents to that propagated context and records
-the ``PROXY_REQUEST`` transport span as a span *link*, never a parent — the shape
-this registry's ``parent=None, links=PROXY_REQUEST`` entry encodes. When nothing is
-propagated (the common case) the span nests under the transport span of the request
-carrying that message, so the tool call stays in one trace.
+MCP spans (``MCP_TOOL_CALL``, ``MCP_LIST_TOOLS``) are parented at emit time by
+:func:`resolve_mcp_span_context`: they nest under the ``PROXY_REQUEST`` transport
+span of the request carrying that message, so the tool call stays in one trace.
+Trace context the client propagated in ``params._meta`` (SEP-414) is recorded as
+a span *link*, never the parent — a remote parent would root the span in a trace
+whose root never reaches the gateway's tracing backend. Links always target that
+remote client context, never a registry role, so ``SpanSpec`` declares no link
+field; the concrete transport parent is resolved per message at emit time.
Not every service call becomes a span — :func:`span_role_for_service` decides:
@@ -85,25 +87,19 @@ class SpanSpec:
role: SpanRole
kind: LiteLLMSpanKind
parent: SpanRole | None
- links: SpanRole | None = None
SPAN_REGISTRY: Final[dict[SpanRole, SpanSpec]] = {
SpanRole.PROXY_REQUEST: SpanSpec(SpanRole.PROXY_REQUEST, LiteLLMSpanKind.SERVER, parent=None),
SpanRole.LLM_CALL: SpanSpec(SpanRole.LLM_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
# The proxy is an MCP client to the upstream server, so MCP spans are CLIENT
- # spans. With trace context propagated in ``params._meta``, MCP and the HTTP
- # transport are independent contexts (OTel GenAI MCP semconv): the span parents
- # to the propagated context and records the PROXY_REQUEST transport span as a
- # span *link*, never a parent — the shape ``parent=None, links=PROXY_REQUEST``
- # encodes. With nothing propagated, ``resolve_mcp_span_context`` nests the span
- # under that message's transport span instead, keeping the call in one trace.
- SpanRole.MCP_TOOL_CALL: SpanSpec(
- SpanRole.MCP_TOOL_CALL, LiteLLMSpanKind.CLIENT, parent=None, links=SpanRole.PROXY_REQUEST
- ),
- SpanRole.MCP_LIST_TOOLS: SpanSpec(
- SpanRole.MCP_LIST_TOOLS, LiteLLMSpanKind.CLIENT, parent=None, links=SpanRole.PROXY_REQUEST
- ),
+ # spans. ``resolve_mcp_span_context`` nests them under the PROXY_REQUEST
+ # transport span of the request carrying that message (resolved per message at
+ # emit time), keeping the call in one trace. Trace context the client
+ # propagated in ``params._meta`` becomes a span *link* to that remote context,
+ # which is not a registry role, so ``SpanSpec`` has no link field.
+ SpanRole.MCP_TOOL_CALL: SpanSpec(SpanRole.MCP_TOOL_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
+ SpanRole.MCP_LIST_TOOLS: SpanSpec(SpanRole.MCP_LIST_TOOLS, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
SpanRole.GUARDRAIL: SpanSpec(SpanRole.GUARDRAIL, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST),
SpanRole.DB_CALL: SpanSpec(SpanRole.DB_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
SpanRole.SERVICE: SpanSpec(SpanRole.SERVICE, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST),
@@ -209,8 +205,8 @@ def service_span_name(data: "ServiceSpanData") -> str:
def root_roles() -> list[SpanRole]:
- """Roles with no in-process parent. They start a new trace unless they adopt a
- remote parent (e.g. an MCP span joining the client's propagated context)."""
+ """Roles with no in-process parent, i.e. they start a new trace (only the
+ instrumentor-owned ``PROXY_REQUEST`` server span today)."""
return [role for role, spec in SPAN_REGISTRY.items() if spec.parent is None]
@@ -227,8 +223,6 @@ def validate_registry(
raise ValueError(f"SPAN_REGISTRY[{role}] has mismatched role {spec.role}")
if spec.parent is not None and spec.parent not in reg:
raise ValueError(f"span role {role} declares unknown parent {spec.parent}")
- if spec.links is not None and spec.links not in reg:
- raise ValueError(f"span role {role} declares unknown link target {spec.links}")
missing: Final = [role for role in SpanRole if role not in reg]
if missing:
raise ValueError(f"SPAN_REGISTRY is missing roles: {missing}")
diff --git a/litellm/integrations/otel/plumbing/context.py b/litellm/integrations/otel/plumbing/context.py
index 19b36c0b967..159a84b121f 100644
--- a/litellm/integrations/otel/plumbing/context.py
+++ b/litellm/integrations/otel/plumbing/context.py
@@ -57,8 +57,8 @@ def request_root_span() -> "Span | None":
# The W3C trace-context carrier (``traceparent``/``tracestate``/``baggage``) the
# MCP client propagated in the current request's ``params._meta``. The MCP gateway
-# sets it per message so the MCP span can parent to the client's span rather than
-# to the transport. A ``ContextVar`` because, like the root-span anchor, it must
+# sets it per message so the MCP span can record the client's span as a span
+# link. A ``ContextVar`` because, like the root-span anchor, it must
# ride the request task and be readable by the inline success-logging callback.
_mcp_message_trace_carrier: Final["ContextVar[Mapping[str, str] | None]"] = ContextVar(
"litellm_otel_mcp_message_trace_carrier", default=None
@@ -148,10 +148,10 @@ def _mcp_transport_span_context() -> "SpanContext | None":
Prefers the transport the gateway published for this specific message; falls
back to the ambient request anchor for paths that emit an MCP span on the
- request task itself (the REST MCP endpoints, the SDK). Parenting and linking
- only need the immutable context, and unlike ``mcp_message_transport_span`` they
- stay correct against a transport that has already finished, so this does not
- require the span to still be recording.
+ request task itself (the REST MCP endpoints). Parenting needs only the
+ immutable context, and unlike ``mcp_message_transport_span`` it stays correct
+ against a transport that has already finished, so this does not require the
+ span to still be recording.
"""
published: Final = _mcp_message_transport_span.get()
if published is not None:
@@ -222,25 +222,31 @@ def resolve_mcp_span_context(
) -> "tuple[Context, tuple[Link, ...]]":
"""Parent context + links for an MCP message span.
+ The span always nests under the transport span of the request carrying this
+ message, so a tool call and the ``POST`` that carried it stay in one trace.
+ The transport comes from :func:`_mcp_transport_span_context`, which is the
+ *current message's* POST rather than whatever request happened to open the
+ session, so a long-lived session does not glue every message under its first
+ request.
+
When the client propagates W3C trace context in the request's ``params._meta``
- (SEP-414), MCP and the underlying transport are independent lifecycles — one
- streamable-HTTP session multiplexes many messages, and the client's own span is
- the truthful parent. So, per the OTel GenAI MCP semconv:
+ (SEP-414), that remote context is recorded as a span *link*, never the parent.
+ The OTel GenAI MCP semconv prefers the inverse (remote parent, transport link),
+ but the gateway's tracing backend only ever receives the gateway's half of such
+ a trace: parenting into the client's trace id roots the span in a trace whose
+ root span never reaches the backend, so the span is unreachable from the trace
+ view and the transport transaction shows a dangling link (observed with
+ clients that propagate synthetic trace ids). Anchoring to the gateway's own
+ request and linking the client's context keeps every trace renderable while
+ preserving the client-side correlation.
- * parent to the trace context the client propagated (a *remote* parent), and
- * record the transport span as a *link*, never the parent.
-
- Almost no client implements SEP-414 yet, so in practice nothing is propagated.
- Rooting the span there splits a single tool call into two disconnected traces
- joined only by a link, which is how it surfaces in APM: the ``POST`` transaction
- and the ``tools/call`` span share no trace. With no remote parent to honor,
- parent to the transport span of the request carrying this message instead, so
- the call stays in one trace; no link is added since the transport is now the
- real parent. The transport comes from :func:`_mcp_transport_span_context`, which
- is the *current message's* POST rather than whatever request happened to open
- the session, so a long-lived session does not glue every message under its
- first request. With neither a remote parent nor a transport the returned context
- carries no span and the span legitimately starts its own root trace.
+ With no transport at all the span starts its own root trace, still carrying
+ the link — the client context is only ever a link, so this event keeps one
+ shape everywhere. Both returned contexts are built on an explicitly empty
+ base, so ambient (stale session) state can never leak in, and the span
+ inherits the transport's sampling decision exactly like every other
+ request-level span — a client's sampled flag neither forces nor suppresses
+ recording.
Only trace context (``traceparent``/``tracestate``) is extracted, never the
client's W3C Baggage: ``params._meta`` is caller-controlled, and the otel
@@ -251,13 +257,12 @@ def resolve_mcp_span_context(
never fall through to the ambient (stale session) span.
"""
source: Final = carrier if carrier is not None else _mcp_message_trace_carrier.get()
- parent: Final = _PROPAGATOR.extract(dict(source or {}), context=Context())
+ propagated: Final = get_current_span(_PROPAGATOR.extract(dict(source or {}), context=Context()))
+ links: Final = (Link(propagated.get_span_context()),) if is_recordable_span(propagated) else ()
transport: Final = _mcp_transport_span_context()
- if is_recordable_span(get_current_span(parent)):
- return parent, (Link(transport),) if transport is not None else ()
- if transport is not None:
- return context_from_span(NonRecordingSpan(transport)), ()
- return parent, ()
+ if transport is None:
+ return Context(), links
+ return context_from_span(NonRecordingSpan(transport), context=Context()), links
def is_recordable_span(obj: object) -> bool:
diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py
index 2da63554b75..9125ed6e70a 100644
--- a/litellm/litellm_core_utils/realtime_streaming.py
+++ b/litellm/litellm_core_utils/realtime_streaming.py
@@ -330,6 +330,24 @@ class RealTimeStreaming:
except (AttributeError, TypeError):
pass
+ def _flush_unbilled_transcription_usage(self) -> None:
+ if self.provider_config is None:
+ return
+ usage: Final = self.provider_config.unbilled_usage_on_session_close(self.model)
+ if usage is None:
+ return
+ flush_event: Final = (
+ cast( # cast-ok: usage-only partial event, the same shape _capture_transcription_usage logs
+ OpenAIRealtimeEvents,
+ {
+ "type": "conversation.item.input_audio_transcription.completed",
+ "usage": usage,
+ },
+ )
+ )
+ self.store_message(flush_event)
+ self._capture_transcription_usage(flush_event)
+
def _collect_tool_calls_from_response_done(self, event_obj: dict | OpenAIRealtimeEvents) -> None:
"""Extract function_call items from response.done events for spend logging."""
try:
@@ -1069,6 +1087,7 @@ class RealTimeStreaming:
except Exception as e:
verbose_logger.exception("Error in backend to client send messages: %s", e)
finally:
+ self._flush_unbilled_transcription_usage()
await self.log_messages()
@staticmethod
diff --git a/litellm/llms/base_llm/realtime/transformation.py b/litellm/llms/base_llm/realtime/transformation.py
index 26c189504df..cfcde7c6e9e 100644
--- a/litellm/llms/base_llm/realtime/transformation.py
+++ b/litellm/llms/base_llm/realtime/transformation.py
@@ -5,6 +5,7 @@ import httpx
from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents
from litellm.types.realtime import (
+ RealtimeInputAudioTranscriptionUsage,
RealtimeResponseTransformInput,
RealtimeResponseTypedDict,
)
@@ -70,6 +71,9 @@ class BaseRealtimeConfig(ABC):
def session_configuration_request(self, model: str) -> str | None: # message sent to setup the realtime session
return None
+ def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
+ return None
+
def transform_session_created_event(
self,
model: str,
diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py
index a3b6381306e..367619db37d 100644
--- a/litellm/llms/gemini/realtime/transformation.py
+++ b/litellm/llms/gemini/realtime/transformation.py
@@ -1191,6 +1191,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
}
return usage
+ def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
+ return self._consume_input_transcription_usage_estimate(model)
+
def transform_realtime_response(
self,
message: str | bytes,
diff --git a/litellm/main.py b/litellm/main.py
index 8ee102f5d07..583c5b3f92a 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -1219,6 +1219,7 @@ def _register_custom_pricing_for_request(
shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry),
},
persist_across_reloads=False,
+ warning_display_name=shared_key,
)
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index fc10b61b7cd..b21e8a60599 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -4691,7 +4691,7 @@
"supports_web_search": false
},
"azure/gpt-4.1-nano": {
- "deprecation_date": "2026-10-14",
+ "deprecation_date": "2027-04-14",
"cache_read_input_token_cost": 2.5e-08,
"input_cost_per_token": 1e-07,
"input_cost_per_token_batches": 5e-08,
@@ -4724,7 +4724,7 @@
"supports_vision": true
},
"azure/gpt-4.1-nano-2025-04-14": {
- "deprecation_date": "2026-10-14",
+ "deprecation_date": "2027-04-14",
"cache_read_input_token_cost": 2.5e-08,
"input_cost_per_token": 1e-07,
"input_cost_per_token_batches": 5e-08,
@@ -15167,6 +15167,34 @@
"output_dbu_cost_per_token": 7.143e-06,
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
},
+ "databricks/databricks-glm-5-2": {
+ "cache_creation_input_token_cost": 1.4e-06,
+ "cache_read_input_token_cost": 2.5998e-07,
+ "input_cost_per_token": 1.4e-06,
+ "input_dbu_cost_per_token": 2e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 4.39999e-06,
+ "output_dbu_cost_per_token": 6.2857e-05,
+ "source": "https://www.databricks.com/product/pricing/foundation-model-serving",
+ "supported_modalities": [
+ "text"
+ ],
+ "supported_output_modalities": [
+ "text"
+ ],
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
"databricks/databricks-gpt-5": {
"cache_creation_input_token_cost": 1.24999e-06,
"cache_read_input_token_cost": 1.2502e-07,
@@ -15434,6 +15462,35 @@
"output_vector_size": 1024,
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
},
+ "databricks/databricks-kimi-k3": {
+ "cache_creation_input_token_cost": 2.99999e-06,
+ "cache_read_input_token_cost": 3.0002e-07,
+ "input_cost_per_token": 2.99999e-06,
+ "input_dbu_cost_per_token": 4.2857e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1048576,
+ "max_tokens": 1048576,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.500002e-05,
+ "output_dbu_cost_per_token": 0.000214286,
+ "source": "https://www.databricks.com/product/pricing/foundation-model-serving",
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "text"
+ ],
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
"databricks/databricks-llama-2-70b-chat": {
"cache_creation_input_token_cost": 5.0001e-07,
"cache_read_input_token_cost": 5.0001e-07,
@@ -16112,12 +16169,13 @@
"max_tokens": 4096,
"max_input_tokens": 4096,
"max_output_tokens": 4096,
- "input_cost_per_token": 8e-08,
- "output_cost_per_token": 9e-08,
+ "input_cost_per_token": 4e-07,
+ "output_cost_per_token": 4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/NousResearch/Hermes-3-Llama-3.1-405B": {
"max_tokens": 131072,
@@ -16134,11 +16192,12 @@
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 3e-07,
- "output_cost_per_token": 3e-07,
+ "input_cost_per_token": 7e-07,
+ "output_cost_per_token": 7e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
- "supports_tool_choice": false
+ "supports_tool_choice": false,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Qwen/QwQ-32B": {
"max_tokens": 131072,
@@ -16155,12 +16214,13 @@
"max_tokens": 32768,
"max_input_tokens": 32768,
"max_output_tokens": 32768,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3.9e-07,
+ "input_cost_per_token": 3.6e-07,
+ "output_cost_per_token": 4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Qwen/Qwen2.5-7B-Instruct": {
"max_tokens": 32768,
@@ -16188,12 +16248,13 @@
"max_tokens": 40960,
"max_input_tokens": 40960,
"max_output_tokens": 40960,
- "input_cost_per_token": 6e-08,
+ "input_cost_per_token": 1.2e-07,
"output_cost_per_token": 2.4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Qwen/Qwen3-235B-A22B": {
"max_tokens": 40960,
@@ -16211,11 +16272,12 @@
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"input_cost_per_token": 9e-08,
- "output_cost_per_token": 6e-07,
+ "output_cost_per_token": 5.5e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Qwen/Qwen3-235B-A22B-Thinking-2507": {
"max_tokens": 262144,
@@ -16232,23 +16294,25 @@
"max_tokens": 40960,
"max_input_tokens": 40960,
"max_output_tokens": 40960,
- "input_cost_per_token": 8e-08,
- "output_cost_per_token": 2.9e-07,
+ "input_cost_per_token": 1.2e-07,
+ "output_cost_per_token": 5e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Qwen/Qwen3-32B": {
"max_tokens": 40960,
"max_input_tokens": 40960,
"max_output_tokens": 40960,
- "input_cost_per_token": 1e-07,
+ "input_cost_per_token": 8e-08,
"output_cost_per_token": 2.8e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct": {
"max_tokens": 262144,
@@ -16265,23 +16329,27 @@
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
- "input_cost_per_token": 2.9e-07,
- "output_cost_per_token": 1.2e-06,
+ "input_cost_per_token": 3e-07,
+ "output_cost_per_token": 1e-06,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "cache_read_input_token_cost": 1e-07,
+ "supports_prompt_caching": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Qwen/Qwen3-Next-80B-A3B-Instruct": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
- "input_cost_per_token": 1.4e-07,
- "output_cost_per_token": 1.4e-06,
+ "input_cost_per_token": 9e-08,
+ "output_cost_per_token": 1.1e-06,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Qwen/Qwen3-Next-80B-A3B-Thinking": {
"max_tokens": 262144,
@@ -16308,11 +16376,12 @@
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 6.5e-07,
- "output_cost_per_token": 7.5e-07,
+ "input_cost_per_token": 8.5e-07,
+ "output_cost_per_token": 8.5e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
- "supports_tool_choice": false
+ "supports_tool_choice": false,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Sao10K/L3.3-70B-Euryale-v2.3": {
"max_tokens": 131072,
@@ -16438,36 +16507,41 @@
"max_tokens": 163840,
"max_input_tokens": 163840,
"max_output_tokens": 163840,
- "input_cost_per_token": 3.8e-07,
+ "input_cost_per_token": 3.2e-07,
"output_cost_per_token": 8.9e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/deepseek-ai/DeepSeek-V3-0324": {
"max_tokens": 163840,
"max_input_tokens": 163840,
"max_output_tokens": 163840,
- "input_cost_per_token": 2.5e-07,
- "output_cost_per_token": 8.8e-07,
+ "input_cost_per_token": 2.4e-07,
+ "output_cost_per_token": 9e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "cache_read_input_token_cost": 1.35e-07,
+ "supports_prompt_caching": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/deepseek-ai/DeepSeek-V3.1": {
"max_tokens": 163840,
"max_input_tokens": 163840,
"max_output_tokens": 163840,
- "input_cost_per_token": 2.7e-07,
- "output_cost_per_token": 1e-06,
+ "input_cost_per_token": 2.5e-07,
+ "output_cost_per_token": 9.5e-07,
"cache_read_input_token_cost": 2.16e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
"supports_reasoning": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/deepseek-ai/DeepSeek-V3.1-Terminus": {
"max_tokens": 163840,
@@ -16521,33 +16595,36 @@
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"input_cost_per_token": 5e-08,
- "output_cost_per_token": 1e-07,
+ "output_cost_per_token": 1.5e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/google/gemma-3-27b-it": {
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 9e-08,
+ "input_cost_per_token": 8e-08,
"output_cost_per_token": 1.6e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/google/gemma-3-4b-it": {
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 4e-08,
- "output_cost_per_token": 8e-08,
+ "input_cost_per_token": 5e-08,
+ "output_cost_per_token": 1e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/meta-llama/Llama-3.2-11B-Vision-Instruct": {
"max_tokens": 131072,
@@ -16585,34 +16662,37 @@
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 1.3e-07,
- "output_cost_per_token": 3.9e-07,
+ "input_cost_per_token": 1e-07,
+ "output_cost_per_token": 3.2e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_function_calling": true,
- "supports_tool_choice": true
+ "supports_tool_choice": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": {
"max_tokens": 1048576,
"max_input_tokens": 1048576,
"max_output_tokens": 1048576,
- "input_cost_per_token": 1.5e-07,
- "output_cost_per_token": 6e-07,
+ "input_cost_per_token": 2e-07,
+ "output_cost_per_token": 8e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/meta-llama/Llama-4-Scout-17B-16E-Instruct": {
"max_tokens": 327680,
"max_input_tokens": 327680,
"max_output_tokens": 327680,
- "input_cost_per_token": 8e-08,
+ "input_cost_per_token": 1e-07,
"output_cost_per_token": 3e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/meta-llama/Llama-Guard-3-8B": {
"max_tokens": 131072,
@@ -16660,12 +16740,13 @@
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 1e-07,
- "output_cost_per_token": 2.8e-07,
+ "input_cost_per_token": 4e-07,
+ "output_cost_per_token": 4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct": {
"max_tokens": 131072,
@@ -16683,11 +16764,12 @@
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"input_cost_per_token": 2e-08,
- "output_cost_per_token": 3e-08,
+ "output_cost_per_token": 4e-08,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/microsoft/WizardLM-2-8x22B": {
"max_tokens": 65536,
@@ -16714,12 +16796,13 @@
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 2e-08,
- "output_cost_per_token": 4e-08,
+ "input_cost_per_token": 1.9e-08,
+ "output_cost_per_token": 3e-08,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/mistralai/Mistral-Small-24B-Instruct-2501": {
"max_tokens": 32768,
@@ -16801,14 +16884,16 @@
},
"deepinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning": {
"max_input_tokens": 262144,
- "input_cost_per_token": 5e-08,
+ "input_cost_per_token": 8e-08,
"output_cost_per_token": 2e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
- "source": "https://deepinfra.com/nvidia/NVIDIA-Nemotron-3.5-Lightning",
+ "source": "https://deepinfra.com/pricing",
"supports_tool_choice": true,
"supports_function_calling": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "cache_read_input_token_cost": 4e-08,
+ "supports_prompt_caching": true
},
"deepinfra/nvidia/NVIDIA-Nemotron-Nano-9B-v2": {
"max_tokens": 131072,
@@ -16825,23 +16910,25 @@
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 5e-08,
- "output_cost_per_token": 4.5e-07,
+ "input_cost_per_token": 3.7e-08,
+ "output_cost_per_token": 1.7e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/openai/gpt-oss-20b": {
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 4e-08,
- "output_cost_per_token": 1.5e-07,
+ "input_cost_per_token": 3e-08,
+ "output_cost_per_token": 1.4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/zai-org/GLM-4.5": {
"max_tokens": 131072,
@@ -18636,6 +18723,22 @@
"supports_tool_choice": true,
"supports_vision": false
},
+ "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813": {
+ "cache_read_input_token_cost": 4.4e-08,
+ "input_cost_per_token": 1.32e-06,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 3.96e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
"fireworks_ai/accounts/fireworks/models/firefunction-v2": {
"input_cost_per_token": 9e-07,
"litellm_provider": "fireworks_ai",
@@ -20342,7 +20445,7 @@
"mode": "chat",
"output_cost_per_reasoning_token": 2.5e-06,
"output_cost_per_token": 2.5e-06,
- "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/",
+ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
@@ -20522,7 +20625,7 @@
"mode": "chat",
"output_cost_per_reasoning_token": 4e-07,
"output_cost_per_token": 4e-07,
- "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview",
+ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
@@ -22114,7 +22217,7 @@
"output_cost_per_reasoning_token": 2.5e-06,
"output_cost_per_token": 2.5e-06,
"rpm": 15,
- "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/",
+ "source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
@@ -22162,7 +22265,7 @@
"output_cost_per_reasoning_token": 2.5e-06,
"output_cost_per_token": 2.5e-06,
"rpm": 15,
- "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/",
+ "source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
@@ -22209,7 +22312,7 @@
"output_cost_per_reasoning_token": 4e-07,
"output_cost_per_token": 4e-07,
"rpm": 15,
- "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/",
+ "source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
@@ -22257,7 +22360,7 @@
"output_cost_per_reasoning_token": 4e-07,
"output_cost_per_token": 4e-07,
"rpm": 15,
- "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-lite",
+ "source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
@@ -22876,36 +22979,6 @@
"supports_vision": true,
"tpm": 800000
},
- "gemini/gemini-omni-1.1-flash": {
- "input_cost_per_audio_token": 1.5e-06,
- "input_cost_per_token": 1.5e-06,
- "litellm_provider": "gemini",
- "mode": "chat",
- "output_cost_per_reasoning_token": 9e-06,
- "output_cost_per_token": 9e-06,
- "output_cost_per_video_token": 1.75e-05,
- "rpm": 2000,
- "source": "https://ai.google.dev/gemini-api/docs/pricing",
- "supported_endpoints": [
- "/v1/chat/completions"
- ],
- "supported_modalities": [
- "text",
- "image",
- "audio",
- "video"
- ],
- "supported_output_modalities": [
- "text",
- "video"
- ],
- "supports_audio_input": true,
- "supports_reasoning": true,
- "supports_system_messages": true,
- "supports_video_input": true,
- "supports_vision": true,
- "tpm": 800000
- },
"gemini/gemini-3.1-pro-preview": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 2e-07,
@@ -30871,6 +30944,152 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
+ "mistral/ministral-14b-2512": {
+ "input_cost_per_token": 2e-07,
+ "litellm_provider": "mistral",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 2e-07,
+ "source": "https://docs.mistral.ai/models/ministral-3-14b-25-12",
+ "supports_assistant_prefill": true,
+ "supports_function_calling": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "mistral/ministral-14b-latest": {
+ "input_cost_per_token": 2e-07,
+ "litellm_provider": "mistral",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 2e-07,
+ "source": "https://docs.mistral.ai/models/ministral-3-14b-25-12",
+ "supports_assistant_prefill": true,
+ "supports_function_calling": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "mistral/ministral-3b-2512": {
+ "input_cost_per_token": 1e-07,
+ "litellm_provider": "mistral",
+ "max_input_tokens": 131072,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 1e-07,
+ "source": "https://docs.mistral.ai/models/ministral-3-3b-25-12",
+ "supports_assistant_prefill": true,
+ "supports_function_calling": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "mistral/ministral-3b-latest": {
+ "input_cost_per_token": 1e-07,
+ "litellm_provider": "mistral",
+ "max_input_tokens": 131072,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 1e-07,
+ "source": "https://docs.mistral.ai/models/ministral-3-3b-25-12",
+ "supports_assistant_prefill": true,
+ "supports_function_calling": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "mistral/mistral-embed-2312": {
+ "input_cost_per_token": 1e-07,
+ "litellm_provider": "mistral",
+ "max_input_tokens": 8192,
+ "max_tokens": 8192,
+ "mode": "embedding",
+ "source": "https://docs.mistral.ai/models/mistral-embed-23-12"
+ },
+ "mistral/mistral-medium-3": {
+ "input_cost_per_token": 1.5e-06,
+ "litellm_provider": "mistral",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 7.5e-06,
+ "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
+ "supports_assistant_prefill": true,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "mistral/voxtral-mini-transcribe-realtime-latest": {
+ "input_cost_per_second": 0.0001,
+ "litellm_provider": "mistral",
+ "mode": "audio_transcription",
+ "source": "https://docs.mistral.ai/models/model-cards/voxtral-mini-transcribe-realtime-26-02",
+ "supported_endpoints": [
+ "/v1/audio/transcriptions"
+ ],
+ "supported_modalities": [
+ "audio"
+ ],
+ "supported_output_modalities": [
+ "text"
+ ],
+ "supports_audio_input": true
+ },
+ "mistral/voxtral-mini-tts-latest": {
+ "litellm_provider": "mistral",
+ "mode": "audio_speech",
+ "output_cost_per_character": 1.6e-05,
+ "source": "https://docs.mistral.ai/models/model-cards/voxtral-tts-26-03",
+ "supported_endpoints": [
+ "/v1/audio/speech"
+ ],
+ "supported_modalities": [
+ "text"
+ ],
+ "supported_output_modalities": [
+ "audio"
+ ],
+ "supports_audio_output": true
+ },
+ "mistral/voxtral-small-2507": {
+ "input_cost_per_second": 6.666666666666667e-05,
+ "input_cost_per_token": 1e-07,
+ "litellm_provider": "mistral",
+ "max_input_tokens": 32768,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 4e-07,
+ "source": "https://docs.mistral.ai/models/voxtral-small-25-07",
+ "supports_audio_input": true,
+ "supports_function_calling": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true
+ },
+ "mistral/voxtral-small-latest": {
+ "input_cost_per_second": 6.666666666666667e-05,
+ "input_cost_per_token": 1e-07,
+ "litellm_provider": "mistral",
+ "max_input_tokens": 32768,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 4e-07,
+ "source": "https://docs.mistral.ai/models/voxtral-small-25-07",
+ "supports_audio_input": true,
+ "supports_function_calling": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true
+ },
"mistral/zai-glm-5-2": {
"cache_read_input_token_cost": 1.4e-07,
"input_cost_per_token": 1.4e-06,
@@ -38183,7 +38402,7 @@
"max_output_tokens": 20480,
"max_tokens": 20480,
"metadata": {
- "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro"
+ "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813"
},
"mode": "chat",
"output_cost_per_token": 7e-06,
@@ -38212,7 +38431,7 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"metadata": {
- "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro"
+ "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813"
},
"mode": "chat",
"output_cost_per_token": 1.25e-06,
@@ -38227,7 +38446,7 @@
"litellm_provider": "together_ai",
"max_tokens": 16384,
"metadata": {
- "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro"
+ "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813"
},
"mode": "chat",
"output_cost_per_token": 1.7e-06,
@@ -38518,6 +38737,7 @@
"supports_tool_choice": true
},
"together_ai/Qwen/Qwen3.5-397B-A17B": {
+ "cache_read_input_token_cost": 3.5e-07,
"deprecation_date": "2026-06-29",
"input_cost_per_token": 6e-07,
"litellm_provider": "together_ai",
@@ -38527,10 +38747,12 @@
"source": "https://www.together.ai/models/Qwen/Qwen3.5-397B-A17B",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/MiniMaxAI/MiniMax-M3": {
+ "cache_read_input_token_cost": 6e-08,
"input_cost_per_token": 3e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 524288,
@@ -38541,6 +38763,7 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@@ -38584,6 +38807,7 @@
"supports_reasoning": true
},
"together_ai/Qwen/Qwen3.7-Max": {
+ "cache_read_input_token_cost": 1.3e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1000000,
@@ -38591,7 +38815,8 @@
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 3.75e-06,
- "source": "https://docs.together.ai/docs/serverless-models"
+ "source": "https://docs.together.ai/docs/serverless-models",
+ "supports_prompt_caching": true
},
"together_ai/Qwen/Qwen3.7-Plus": {
"input_cost_per_token": 3.2e-07,
@@ -38604,6 +38829,7 @@
"source": "https://docs.together.ai/docs/serverless-models"
},
"together_ai/Qwen/Qwen3.8-2.4T-A95B": {
+ "cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 2.5e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1010000,
@@ -38611,7 +38837,8 @@
"max_tokens": 1010000,
"mode": "chat",
"output_cost_per_token": 6.25e-06,
- "source": "https://docs.together.ai/docs/serverless-models"
+ "source": "https://docs.together.ai/docs/serverless-models",
+ "supports_prompt_caching": true
},
"together_ai/arize-ai/qwen-2-1.5b-instruct": {
"input_cost_per_token": 1e-07,
@@ -38624,6 +38851,7 @@
"source": "https://docs.together.ai/docs/serverless-models"
},
"together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": {
+ "cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 1.4e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
@@ -38634,10 +38862,13 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/deepseek-ai/DeepSeek-V4-Pro": {
+ "deprecation_date": "2026-08-27",
+ "cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1.74e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 512000,
@@ -38648,11 +38879,13 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": {
+ "cache_read_input_token_cost": 1.3e-07,
"input_cost_per_token": 1.32e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
@@ -38663,10 +38896,12 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/google/gemma-3n-E4B-it": {
+ "deprecation_date": "2026-08-25",
"input_cost_per_token": 6e-08,
"litellm_provider": "together_ai",
"max_input_tokens": 32768,
@@ -38702,6 +38937,7 @@
"source": "https://docs.together.ai/docs/serverless-models"
},
"together_ai/meta-llama/Llama-Guard-4-12B": {
+ "deprecation_date": "2026-08-25",
"input_cost_per_token": 2e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
@@ -38712,6 +38948,7 @@
"source": "https://docs.together.ai/docs/serverless-models"
},
"together_ai/meta-models/Muse-Glimmer-30B": {
+ "cache_read_input_token_cost": 4e-08,
"input_cost_per_token": 3.5e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 131072,
@@ -38719,9 +38956,12 @@
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.5e-06,
- "source": "https://docs.together.ai/docs/serverless-models"
+ "source": "https://docs.together.ai/docs/serverless-models",
+ "supports_prompt_caching": true
},
"together_ai/moonshotai/Kimi-K2.7-Code": {
+ "deprecation_date": "2026-08-27",
+ "cache_read_input_token_cost": 1.9e-07,
"input_cost_per_token": 9.5e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 262144,
@@ -38732,11 +38972,13 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"together_ai/moonshotai/Kimi-K3": {
+ "cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
@@ -38747,12 +38989,15 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"together_ai/nvidia/nemotron-3-ultra-550b-a55b": {
+ "deprecation_date": "2026-08-27",
+ "cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 6e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 512288,
@@ -38763,11 +39008,13 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/pearl-ai/gemma-4-31b-it": {
+ "deprecation_date": "2026-08-27",
"input_cost_per_token": 2.8e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 262144,
@@ -38778,6 +39025,7 @@
"source": "https://docs.together.ai/docs/serverless-models"
},
"together_ai/thinkingmachines/Inkling": {
+ "cache_read_input_token_cost": 1.7e-07,
"input_cost_per_token": 1e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 524288,
@@ -38788,10 +39036,12 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/thinkingmachines/Inkling-Small": {
+ "cache_read_input_token_cost": 1e-07,
"input_cost_per_token": 5e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 524288,
@@ -38799,9 +39049,11 @@
"max_tokens": 524288,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
- "source": "https://docs.together.ai/docs/serverless-models"
+ "source": "https://docs.together.ai/docs/serverless-models",
+ "supports_prompt_caching": true
},
"together_ai/zai-org/GLM-5.2": {
+ "cache_read_input_token_cost": 2.6e-07,
"input_cost_per_token": 1.4e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1048575,
@@ -38812,6 +39064,7 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
@@ -43090,19 +43343,21 @@
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 0.015,
- "output_cost_per_token": 0.06,
+ "input_cost_per_token": 3e-08,
+ "output_cost_per_token": 1.7e-07,
"litellm_provider": "wandb",
- "mode": "chat"
+ "mode": "chat",
+ "source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/openai/gpt-oss-20b": {
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 0.005,
- "output_cost_per_token": 0.02,
+ "input_cost_per_token": 3e-08,
+ "output_cost_per_token": 1.3e-07,
"litellm_provider": "wandb",
- "mode": "chat"
+ "mode": "chat",
+ "source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/zai-org/GLM-4.5": {
"max_tokens": 131072,
@@ -43126,10 +43381,11 @@
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
- "input_cost_per_token": 0.1,
- "output_cost_per_token": 0.15,
+ "input_cost_per_token": 1e-06,
+ "output_cost_per_token": 1.5e-06,
"litellm_provider": "wandb",
- "mode": "chat"
+ "mode": "chat",
+ "source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": {
"max_tokens": 262144,
@@ -43181,19 +43437,21 @@
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
- "input_cost_per_token": 0.022,
- "output_cost_per_token": 0.022,
+ "input_cost_per_token": 2.2e-07,
+ "output_cost_per_token": 2.2e-07,
"litellm_provider": "wandb",
- "mode": "chat"
+ "mode": "chat",
+ "source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/deepseek-ai/DeepSeek-V3.1": {
"max_tokens": 128000,
- "max_input_tokens": 128000,
+ "max_input_tokens": 161000,
"max_output_tokens": 128000,
- "input_cost_per_token": 0.055,
- "output_cost_per_token": 0.165,
+ "input_cost_per_token": 5.5e-07,
+ "output_cost_per_token": 1.65e-06,
"litellm_provider": "wandb",
- "mode": "chat"
+ "mode": "chat",
+ "source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/deepseek-ai/DeepSeek-R1-0528": {
"max_tokens": 161000,
@@ -43217,10 +43475,11 @@
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
- "input_cost_per_token": 0.071,
- "output_cost_per_token": 0.071,
+ "input_cost_per_token": 7.1e-07,
+ "output_cost_per_token": 7.1e-07,
"litellm_provider": "wandb",
- "mode": "chat"
+ "mode": "chat",
+ "source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": {
"max_tokens": 64000,
@@ -43606,85 +43865,6 @@
"/v1/audio/transcriptions"
]
},
- "xai/grok-2": {
- "input_cost_per_token": 2e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "max_tokens": 131072,
- "mode": "chat",
- "output_cost_per_token": 1e-05,
- "supports_function_calling": true,
- "supports_tool_choice": true,
- "supports_web_search": true
- },
- "xai/grok-2-1212": {
- "input_cost_per_token": 2e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "max_tokens": 131072,
- "mode": "chat",
- "output_cost_per_token": 1e-05,
- "supports_function_calling": true,
- "supports_tool_choice": true,
- "supports_web_search": true
- },
- "xai/grok-2-latest": {
- "input_cost_per_token": 2e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "max_tokens": 131072,
- "mode": "chat",
- "output_cost_per_token": 1e-05,
- "supports_function_calling": true,
- "supports_tool_choice": true,
- "supports_web_search": true
- },
- "xai/grok-2-vision": {
- "input_cost_per_image": 2e-06,
- "input_cost_per_token": 2e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "max_tokens": 32768,
- "mode": "chat",
- "output_cost_per_token": 1e-05,
- "supports_function_calling": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true
- },
- "xai/grok-2-vision-1212": {
- "deprecation_date": "2026-02-28",
- "input_cost_per_image": 2e-06,
- "input_cost_per_token": 2e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "max_tokens": 32768,
- "mode": "chat",
- "output_cost_per_token": 1e-05,
- "supports_function_calling": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true
- },
- "xai/grok-2-vision-latest": {
- "input_cost_per_image": 2e-06,
- "input_cost_per_token": 2e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "max_tokens": 32768,
- "mode": "chat",
- "output_cost_per_token": 1e-05,
- "supports_function_calling": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true
- },
"xai/grok-3": {
"cache_read_input_token_cost": 7.5e-07,
"input_cost_per_token": 3e-06,
@@ -44070,7 +44250,7 @@
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
- "mode": "chat",
+ "mode": "responses",
"output_cost_per_token": 2.5e-06,
"source": "https://docs.x.ai/docs/models",
"supports_function_calling": true,
@@ -44082,7 +44262,10 @@
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_response_schema": true
+ "supports_response_schema": true,
+ "supported_endpoints": [
+ "/v1/responses"
+ ]
},
"xai/grok-4.20-beta-0309-reasoning": {
"cache_read_input_token_cost": 2e-07,
@@ -44126,69 +44309,6 @@
"supports_prompt_caching": true,
"supports_response_schema": true
},
- "xai/grok-4.20": {
- "cache_read_input_token_cost": 2e-07,
- "input_cost_per_token": 1.25e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 1000000,
- "max_output_tokens": 1000000,
- "max_tokens": 1000000,
- "mode": "chat",
- "output_cost_per_token": 2.5e-06,
- "source": "https://docs.x.ai/docs/models",
- "supports_function_calling": true,
- "supports_reasoning": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true,
- "input_cost_per_token_above_200k_tokens": 2.5e-06,
- "output_cost_per_token_above_200k_tokens": 5e-06,
- "cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_prompt_caching": true,
- "supports_response_schema": true
- },
- "xai/grok-4.20-reasoning": {
- "cache_read_input_token_cost": 2e-07,
- "input_cost_per_token": 1.25e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 1000000,
- "max_output_tokens": 1000000,
- "max_tokens": 1000000,
- "mode": "chat",
- "output_cost_per_token": 2.5e-06,
- "source": "https://docs.x.ai/docs/models",
- "supports_function_calling": true,
- "supports_reasoning": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true,
- "input_cost_per_token_above_200k_tokens": 2.5e-06,
- "output_cost_per_token_above_200k_tokens": 5e-06,
- "cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_prompt_caching": true,
- "supports_response_schema": true
- },
- "xai/grok-4.20-reasoning-latest": {
- "cache_read_input_token_cost": 2e-07,
- "input_cost_per_token": 1.25e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 1000000,
- "max_output_tokens": 1000000,
- "max_tokens": 1000000,
- "mode": "chat",
- "output_cost_per_token": 2.5e-06,
- "source": "https://docs.x.ai/docs/models",
- "supports_function_calling": true,
- "supports_reasoning": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true,
- "input_cost_per_token_above_200k_tokens": 2.5e-06,
- "output_cost_per_token_above_200k_tokens": 5e-06,
- "cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_prompt_caching": true,
- "supports_response_schema": true
- },
"xai/grok-4.20-beta-0309-non-reasoning": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1.25e-06,
@@ -44314,19 +44434,6 @@
"supports_vision": true,
"supports_web_search": true
},
- "xai/grok-beta": {
- "input_cost_per_token": 5e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "max_tokens": 131072,
- "mode": "chat",
- "output_cost_per_token": 1.5e-05,
- "supports_function_calling": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true
- },
"xai/grok-code-fast": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1e-06,
@@ -44390,139 +44497,6 @@
"supports_vision": true,
"deprecation_date": "2026-05-15"
},
- "xai/grok-imagine-image": {
- "input_cost_per_image": 0.002,
- "litellm_provider": "xai",
- "mode": "image_generation",
- "output_cost_per_image": 0.02,
- "source": "https://docs.x.ai/docs/models",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "supported_modalities": [
- "text",
- "image"
- ],
- "supported_output_modalities": [
- "image"
- ]
- },
- "xai/grok-imagine-image-2026-03-02": {
- "input_cost_per_image": 0.002,
- "litellm_provider": "xai",
- "mode": "image_generation",
- "output_cost_per_image": 0.02,
- "source": "https://docs.x.ai/docs/models",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "supported_modalities": [
- "text",
- "image"
- ],
- "supported_output_modalities": [
- "image"
- ]
- },
- "xai/grok-imagine-image-quality": {
- "input_cost_per_image": 0.01,
- "litellm_provider": "xai",
- "mode": "image_generation",
- "output_cost_per_image": 0.05,
- "source": "https://docs.x.ai/docs/models",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "supported_modalities": [
- "text",
- "image"
- ],
- "supported_output_modalities": [
- "image"
- ]
- },
- "xai/grok-imagine-image-quality-20260403": {
- "input_cost_per_image": 0.01,
- "litellm_provider": "xai",
- "mode": "image_generation",
- "output_cost_per_image": 0.05,
- "source": "https://docs.x.ai/docs/models",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "supported_modalities": [
- "text",
- "image"
- ],
- "supported_output_modalities": [
- "image"
- ]
- },
- "xai/grok-imagine-image-quality-latest": {
- "input_cost_per_image": 0.01,
- "litellm_provider": "xai",
- "mode": "image_generation",
- "output_cost_per_image": 0.05,
- "source": "https://docs.x.ai/docs/models",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "supported_modalities": [
- "text",
- "image"
- ],
- "supported_output_modalities": [
- "image"
- ]
- },
- "xai/grok-imagine-image-pro": {
- "input_cost_per_image": 0.01,
- "litellm_provider": "xai",
- "mode": "image_generation",
- "output_cost_per_image": 0.05,
- "source": "https://docs.x.ai/docs/models",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "supported_modalities": [
- "text",
- "image"
- ],
- "supported_output_modalities": [
- "image"
- ]
- },
- "xai/grok-imagine-image-2.0": {
- "input_cost_per_image": 0.01,
- "litellm_provider": "xai",
- "mode": "image_generation",
- "output_cost_per_image": 0.06,
- "source": "https://docs.x.ai/docs/models",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "supported_modalities": [
- "text",
- "image"
- ],
- "supported_output_modalities": [
- "image"
- ]
- },
- "xai/grok-vision-beta": {
- "input_cost_per_image": 5e-06,
- "input_cost_per_token": 5e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 8192,
- "max_output_tokens": 8192,
- "max_tokens": 8192,
- "mode": "chat",
- "output_cost_per_token": 1.5e-05,
- "supports_function_calling": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true
- },
"zai.glm-4.7": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock_converse",
@@ -44581,6 +44555,21 @@
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing"
},
+ "zai/glm-5.3": {
+ "cache_creation_input_token_cost": 0,
+ "cache_read_input_token_cost": 2.6e-07,
+ "input_cost_per_token": 1.4e-06,
+ "litellm_provider": "zai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 4.4e-06,
+ "source": "https://docs.z.ai/guides/overview/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true
+ },
"zai/glm-5.1": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 2.6e-07,
@@ -44786,6 +44775,7 @@
]
},
"azure/sora-2": {
+ "deprecation_date": "2026-10-15",
"litellm_provider": "azure",
"mode": "video_generation",
"output_cost_per_video_per_second": 0.1,
@@ -47369,8 +47359,8 @@
"novita/xiaomimimo/mimo-v2-flash": {
"litellm_provider": "novita",
"mode": "chat",
- "input_cost_per_token": 1e-07,
- "output_cost_per_token": 3e-07,
+ "input_cost_per_token": 1.1e-07,
+ "output_cost_per_token": 3.3e-07,
"max_input_tokens": 262144,
"max_output_tokens": 32000,
"max_tokens": 32000,
@@ -47379,8 +47369,8 @@
"supports_tool_choice": true,
"supports_system_messages": true,
"supports_response_schema": true,
- "cache_read_input_token_cost": 2e-08,
- "input_cost_per_token_cache_hit": 2e-08,
+ "cache_read_input_token_cost": 2.4e-08,
+ "input_cost_per_token_cache_hit": 2.4e-08,
"supports_reasoning": true
},
"novita/zai-org/autoglm-phone-9b-multilingual": {
@@ -47400,14 +47390,16 @@
"input_cost_per_token": 6e-07,
"output_cost_per_token": 2.5e-06,
"max_input_tokens": 262144,
- "max_output_tokens": 262144,
- "max_tokens": 262144,
+ "max_output_tokens": 100352,
+ "max_tokens": 100352,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
"supports_response_schema": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "cache_read_input_token_cost": 1.5e-07,
+ "supports_prompt_caching": true
},
"novita/minimax/minimax-m2": {
"litellm_provider": "novita",
@@ -47423,7 +47415,8 @@
"supports_system_messages": true,
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token_cache_hit": 3e-08,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "supports_response_schema": true
},
"novita/paddlepaddle/paddleocr-vl": {
"litellm_provider": "novita",
@@ -47461,7 +47454,9 @@
"max_tokens": 32768,
"supports_vision": true,
"supports_system_messages": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "supports_function_calling": true,
+ "supports_tool_choice": true
},
"novita/zai-org/glm-4.6v": {
"litellm_provider": "novita",
@@ -47526,7 +47521,8 @@
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
- "supports_response_schema": true
+ "supports_response_schema": true,
+ "supports_reasoning": true
},
"novita/qwen/qwen3-next-80b-a3b-thinking": {
"litellm_provider": "novita",
@@ -47638,8 +47634,8 @@
"input_cost_per_token": 6e-07,
"output_cost_per_token": 2.5e-06,
"max_input_tokens": 262144,
- "max_output_tokens": 262144,
- "max_tokens": 262144,
+ "max_output_tokens": 100352,
+ "max_tokens": 100352,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
@@ -47649,8 +47645,8 @@
"novita/qwen/qwen3-coder-480b-a35b-instruct": {
"litellm_provider": "novita",
"mode": "chat",
- "input_cost_per_token": 3e-07,
- "output_cost_per_token": 1.3e-06,
+ "input_cost_per_token": 3.8e-07,
+ "output_cost_per_token": 1.55e-06,
"max_input_tokens": 262144,
"max_output_tokens": 65536,
"max_tokens": 65536,
@@ -47696,8 +47692,8 @@
"input_cost_per_token": 5.7e-07,
"output_cost_per_token": 2.3e-06,
"max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "max_tokens": 131072,
+ "max_output_tokens": 100352,
+ "max_tokens": 100352,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
@@ -47710,8 +47706,8 @@
"input_cost_per_token": 2.7e-07,
"output_cost_per_token": 1.12e-06,
"max_input_tokens": 163840,
- "max_output_tokens": 163840,
- "max_tokens": 163840,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
@@ -47758,7 +47754,8 @@
"max_input_tokens": 16384,
"max_output_tokens": 16384,
"max_tokens": 16384,
- "supports_system_messages": true
+ "supports_system_messages": true,
+ "supports_response_schema": true
},
"novita/google/gemma-3-12b-it": {
"litellm_provider": "novita",
@@ -47837,13 +47834,14 @@
"mode": "chat",
"input_cost_per_token": 1.35e-07,
"output_cost_per_token": 4e-07,
- "max_input_tokens": 131072,
- "max_output_tokens": 120000,
- "max_tokens": 120000,
+ "max_input_tokens": 12288,
+ "max_output_tokens": 12288,
+ "max_tokens": 12288,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
- "supports_system_messages": true
+ "supports_system_messages": true,
+ "supports_response_schema": true
},
"novita/qwen/qwen-2.5-72b-instruct": {
"litellm_provider": "novita",
@@ -47883,7 +47881,8 @@
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "supports_response_schema": true
},
"novita/deepseek/deepseek-r1-0528": {
"litellm_provider": "novita",
@@ -47923,7 +47922,8 @@
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
- "supports_system_messages": true
+ "supports_system_messages": true,
+ "supports_response_schema": true
},
"novita/microsoft/wizardlm-2-8x22b": {
"litellm_provider": "novita",
@@ -47933,7 +47933,8 @@
"max_input_tokens": 65535,
"max_output_tokens": 8000,
"max_tokens": 8000,
- "supports_system_messages": true
+ "supports_system_messages": true,
+ "supports_response_schema": true
},
"novita/deepseek/deepseek-r1-0528-qwen3-8b": {
"litellm_provider": "novita",
@@ -47980,7 +47981,8 @@
"max_output_tokens": 20000,
"max_tokens": 20000,
"supports_system_messages": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "supports_response_schema": true
},
"novita/meta-llama/llama-4-maverick-17b-128e-instruct-fp8": {
"litellm_provider": "novita",
@@ -47991,7 +47993,8 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"supports_vision": true,
- "supports_system_messages": true
+ "supports_system_messages": true,
+ "supports_response_schema": true
},
"novita/meta-llama/llama-4-scout-17b-16e-instruct": {
"litellm_provider": "novita",
@@ -48127,7 +48130,9 @@
"max_output_tokens": 20000,
"max_tokens": 20000,
"supports_system_messages": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "supports_function_calling": true,
+ "supports_tool_choice": true
},
"novita/google/gemma-3-27b-it": {
"litellm_provider": "novita",
@@ -48165,7 +48170,8 @@
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "supports_response_schema": true
},
"novita/Sao10K/L3-8B-Stheno-v3.2": {
"litellm_provider": "novita",
@@ -48233,7 +48239,9 @@
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "cache_read_input_token_cost": 2.5e-08,
+ "supports_prompt_caching": true
},
"novita/qwen/qwen3-vl-30b-a3b-instruct": {
"litellm_provider": "novita",
@@ -48354,10 +48362,12 @@
"input_cost_per_token": 3e-08,
"output_cost_per_token": 3e-08,
"max_input_tokens": 128000,
- "max_output_tokens": 20000,
- "max_tokens": 20000,
+ "max_output_tokens": 8192,
+ "max_tokens": 8192,
"supports_system_messages": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "supports_function_calling": true,
+ "supports_tool_choice": true
},
"novita/qwen/qwen2.5-7b-instruct": {
"litellm_provider": "novita",
@@ -48365,8 +48375,8 @@
"input_cost_per_token": 7e-08,
"output_cost_per_token": 7e-08,
"max_input_tokens": 32000,
- "max_output_tokens": 32000,
- "max_tokens": 32000,
+ "max_output_tokens": 8192,
+ "max_tokens": 8192,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
@@ -49778,14 +49788,14 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-5.6-sol": {
- "input_cost_per_token": 5.5e-06,
- "input_cost_per_token_above_272k_tokens": 1.1e-05,
- "cache_creation_input_token_cost": 6.875e-06,
- "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05,
- "cache_read_input_token_cost": 5.5e-07,
- "cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
- "output_cost_per_token": 3.3e-05,
- "output_cost_per_token_above_272k_tokens": 4.95e-05,
+ "input_cost_per_token": 4.4e-06,
+ "input_cost_per_token_above_272k_tokens": 8.8e-06,
+ "cache_creation_input_token_cost": 5.5e-06,
+ "cache_creation_input_token_cost_above_272k_tokens": 1.1e-05,
+ "cache_read_input_token_cost": 4.4e-07,
+ "cache_read_input_token_cost_above_272k_tokens": 8.8e-07,
+ "output_cost_per_token": 2.2e-05,
+ "output_cost_per_token_above_272k_tokens": 3.3e-05,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
@@ -49843,6 +49853,34 @@
"supports_tool_choice": true,
"supports_vision": true
},
+ "bedrock_mantle/openai.gpt-5.6-cyber": {
+ "input_cost_per_token": 1.375e-05,
+ "cache_creation_input_token_cost": 1.71875e-05,
+ "cache_read_input_token_cost": 1.375e-06,
+ "output_cost_per_token": 8.25e-05,
+ "litellm_provider": "bedrock_mantle",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "responses",
+ "use_openai_responses_path": true,
+ "supported_endpoints": [
+ "/v1/responses"
+ ],
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "text"
+ ],
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
"bedrock_mantle/openai.gpt-5.6-luna": {
"input_cost_per_token": 2.2e-07,
"input_cost_per_token_above_272k_tokens": 4.4e-07,
@@ -49877,14 +49915,14 @@
"supports_vision": true
},
"us.openai.gpt-5.6-sol": {
- "input_cost_per_token": 5.5e-06,
- "input_cost_per_token_above_272k_tokens": 1.1e-05,
- "cache_creation_input_token_cost": 6.875e-06,
- "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05,
- "cache_read_input_token_cost": 5.5e-07,
- "cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
- "output_cost_per_token": 3.3e-05,
- "output_cost_per_token_above_272k_tokens": 4.95e-05,
+ "input_cost_per_token": 4.4e-06,
+ "input_cost_per_token_above_272k_tokens": 8.8e-06,
+ "cache_creation_input_token_cost": 5.5e-06,
+ "cache_creation_input_token_cost_above_272k_tokens": 1.1e-05,
+ "cache_read_input_token_cost": 4.4e-07,
+ "cache_read_input_token_cost_above_272k_tokens": 8.8e-07,
+ "output_cost_per_token": 2.2e-05,
+ "output_cost_per_token_above_272k_tokens": 3.3e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
@@ -49903,14 +49941,14 @@
"supports_vision": true
},
"global.openai.gpt-5.6-sol": {
- "input_cost_per_token": 5e-06,
- "input_cost_per_token_above_272k_tokens": 1e-05,
- "cache_creation_input_token_cost": 6.25e-06,
- "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
- "cache_read_input_token_cost": 5e-07,
- "cache_read_input_token_cost_above_272k_tokens": 1e-06,
- "output_cost_per_token": 3e-05,
- "output_cost_per_token_above_272k_tokens": 4.5e-05,
+ "input_cost_per_token": 4e-06,
+ "input_cost_per_token_above_272k_tokens": 8e-06,
+ "cache_creation_input_token_cost": 5e-06,
+ "cache_creation_input_token_cost_above_272k_tokens": 1e-05,
+ "cache_read_input_token_cost": 4e-07,
+ "cache_read_input_token_cost_above_272k_tokens": 8e-07,
+ "output_cost_per_token": 2e-05,
+ "output_cost_per_token_above_272k_tokens": 3e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
@@ -51185,46 +51223,6 @@
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"supports_response_schema": true
},
- "xai/grok-4.20-non-reasoning": {
- "cache_read_input_token_cost": 2e-07,
- "input_cost_per_token": 1.25e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 1000000,
- "max_output_tokens": 1000000,
- "max_tokens": 1000000,
- "mode": "chat",
- "output_cost_per_token": 2.5e-06,
- "source": "https://docs.x.ai/docs/models",
- "supports_function_calling": true,
- "supports_prompt_caching": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true,
- "input_cost_per_token_above_200k_tokens": 2.5e-06,
- "output_cost_per_token_above_200k_tokens": 5e-06,
- "cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_response_schema": true
- },
- "xai/grok-4.20-non-reasoning-latest": {
- "cache_read_input_token_cost": 2e-07,
- "input_cost_per_token": 1.25e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 1000000,
- "max_output_tokens": 1000000,
- "max_tokens": 1000000,
- "mode": "chat",
- "output_cost_per_token": 2.5e-06,
- "source": "https://docs.x.ai/docs/models",
- "supports_function_calling": true,
- "supports_prompt_caching": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true,
- "input_cost_per_token_above_200k_tokens": 2.5e-06,
- "output_cost_per_token_above_200k_tokens": 5e-06,
- "cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_response_schema": true
- },
"xai/grok-4.20-multi-agent-0309": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1.25e-06,
@@ -51232,7 +51230,7 @@
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
- "mode": "chat",
+ "mode": "responses",
"output_cost_per_token": 2.5e-06,
"source": "https://docs.x.ai/docs/models",
"supports_function_calling": true,
@@ -51244,49 +51242,10 @@
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_response_schema": true
- },
- "xai/grok-4.20-multi-agent": {
- "cache_read_input_token_cost": 2e-07,
- "input_cost_per_token": 1.25e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 1000000,
- "max_output_tokens": 1000000,
- "max_tokens": 1000000,
- "mode": "chat",
- "output_cost_per_token": 2.5e-06,
- "source": "https://docs.x.ai/docs/models",
- "supports_function_calling": true,
- "supports_prompt_caching": true,
- "supports_reasoning": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true,
- "input_cost_per_token_above_200k_tokens": 2.5e-06,
- "output_cost_per_token_above_200k_tokens": 5e-06,
- "cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_response_schema": true
- },
- "xai/grok-4.20-multi-agent-latest": {
- "cache_read_input_token_cost": 2e-07,
- "input_cost_per_token": 1.25e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 1000000,
- "max_output_tokens": 1000000,
- "max_tokens": 1000000,
- "mode": "chat",
- "output_cost_per_token": 2.5e-06,
- "source": "https://docs.x.ai/docs/models",
- "supports_function_calling": true,
- "supports_prompt_caching": true,
- "supports_reasoning": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true,
- "input_cost_per_token_above_200k_tokens": 2.5e-06,
- "output_cost_per_token_above_200k_tokens": 5e-06,
- "cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_response_schema": true
+ "supports_response_schema": true,
+ "supported_endpoints": [
+ "/v1/responses"
+ ]
},
"xai/grok-build-0.1": {
"cache_read_input_token_cost": 2e-07,
@@ -51657,7 +51616,6 @@
"litellm_provider": "gemini",
"mode": "audio_transcription",
"output_cost_per_token": 1.2e-05,
- "rpm": 10,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/audio/transcriptions"
@@ -51670,7 +51628,8 @@
"text"
],
"supports_audio_input": true,
- "tpm": 250000
+ "tpm": 800000,
+ "rpm": 2000
},
"gemini/gemini-3.5-transcribe-live": {
"input_cost_per_audio_token": 3.5e-06,
@@ -51678,7 +51637,6 @@
"litellm_provider": "gemini",
"mode": "audio_transcription",
"output_cost_per_token": 2.1e-05,
- "rpm": 10,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/realtime"
@@ -51690,7 +51648,8 @@
"text"
],
"supports_audio_input": true,
- "tpm": 250000
+ "tpm": 250000,
+ "rpm": 10
},
"perplexity/pplx-embed-context-v1-0.6b": {
"input_cost_per_token": 8e-09,
@@ -51774,14 +51733,14 @@
"supports_embedding_image_input": true
},
"fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731": {
- "cache_read_input_token_cost": 2.8e-08,
- "input_cost_per_token": 1.4e-07,
+ "cache_read_input_token_cost": 7e-09,
+ "input_cost_per_token": 2.2e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
- "output_cost_per_token": 2.8e-07,
+ "output_cost_per_token": 6.6e-07,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
@@ -52088,5 +52047,2328 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
+ },
+ "novita/zai-org/glm-5.3": {
+ "cache_read_input_token_cost": 2.6e-07,
+ "input_cost_per_token": 1.4e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4.4e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/deepseek/deepseek-v4-pro-0813": {
+ "cache_read_input_token_cost": 1.3200000000000002e-07,
+ "input_cost_per_token": 1.32e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 393216,
+ "max_tokens": 393216,
+ "mode": "chat",
+ "output_cost_per_token": 3.96e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/moonshotai/kimi-k3": {
+ "cache_read_input_token_cost": 3e-07,
+ "input_cost_per_token": 3e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 1048576,
+ "max_tokens": 1048576,
+ "mode": "chat",
+ "output_cost_per_token": 1.5e-05,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/tencent/hy3": {
+ "cache_read_input_token_cost": 3.5e-08,
+ "input_cost_per_token": 1.4e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 5.8e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/zai-org/glm-5.2": {
+ "cache_read_input_token_cost": 2.6e-07,
+ "input_cost_per_token": 1.4e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4.4e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/moonshotai/kimi-k2.7-code": {
+ "cache_read_input_token_cost": 1.9e-07,
+ "input_cost_per_token": 9.499999999999999e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 4e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/deepseek/deepseek-v4-flash-vision-exp": {
+ "cache_read_input_token_cost": 2.8e-08,
+ "input_cost_per_token": 4.4e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 393216,
+ "max_tokens": 393216,
+ "mode": "chat",
+ "output_cost_per_token": 1.32e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/deepseek/deepseek-v4-flash-0731": {
+ "cache_read_input_token_cost": 2.8e-08,
+ "input_cost_per_token": 4.4e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 393216,
+ "max_tokens": 393216,
+ "mode": "chat",
+ "output_cost_per_token": 1.32e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/mindai/macaron-v1-venti": {
+ "cache_read_input_token_cost": 3e-07,
+ "input_cost_per_token": 1.5e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4.5e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/minimax/minimax-m3": {
+ "cache_read_input_token_cost": 6e-08,
+ "input_cost_per_token": 3e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 1.2e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/deepseek/deepseek-v4-flash": {
+ "cache_read_input_token_cost": 2.8e-08,
+ "input_cost_per_token": 1.4e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 393216,
+ "max_tokens": 393216,
+ "mode": "chat",
+ "output_cost_per_token": 2.8e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/deepseek/deepseek-v4-pro": {
+ "cache_read_input_token_cost": 1.35e-07,
+ "input_cost_per_token": 1.6000000000000001e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 393216,
+ "max_tokens": 393216,
+ "mode": "chat",
+ "output_cost_per_token": 3.2000000000000003e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/inclusionai/ling-3.0-flash-fast": {
+ "cache_read_input_token_cost": 1.2e-08,
+ "input_cost_per_token": 6e-08,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 1.8e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/qwen/qwen3.8-max": {
+ "cache_read_input_token_cost": 2.5e-07,
+ "input_cost_per_token": 2e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 6e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/inclusionai/ling-3.0-flash": {
+ "cache_read_input_token_cost": 1.2e-08,
+ "input_cost_per_token": 6e-08,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 1.8e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/mindai/macaron-v1-tall": {
+ "cache_read_input_token_cost": 8e-08,
+ "input_cost_per_token": 4.5000000000000003e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 2.6e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/stepfun/step-3.7-flash": {
+ "cache_read_input_token_cost": 4e-08,
+ "input_cost_per_token": 2.0000000000000002e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 256000,
+ "max_tokens": 256000,
+ "mode": "chat",
+ "output_cost_per_token": 1.15e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/nvidia/nemotron-3-nano-30b-a3b": {
+ "input_cost_per_token": 5.0000000000000004e-08,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 2.0000000000000002e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/baidu/cobuddy": {
+ "cache_read_input_token_cost": 7e-08,
+ "input_cost_per_token": 2.8e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 131072,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 1.13e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/xiaomimimo/mimo-v2.5": {
+ "cache_read_input_token_cost": 3.4e-09,
+ "input_cost_per_token": 1.6800000000000002e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 3.3600000000000004e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/qwen/qwen3.7-max": {
+ "cache_read_input_token_cost": 2.5e-07,
+ "input_cost_per_token": 1.25e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 3.75e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/xiaomimimo/mimo-v2.5-pro": {
+ "cache_read_input_token_cost": 4.3e-09,
+ "input_cost_per_token": 5.22e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 1.044e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/qwen/qwen3.6-27b": {
+ "input_cost_per_token": 6e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 3.6000000000000003e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/moonshotai/kimi-k2.6": {
+ "cache_read_input_token_cost": 1.6e-07,
+ "input_cost_per_token": 8.000000000000001e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 3.4e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/zai-org/glm-5.1": {
+ "cache_read_input_token_cost": 2.6e-07,
+ "input_cost_per_token": 1.38e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 204800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4.4e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/minimax/minimax-m2.7-highspeed": {
+ "cache_read_input_token_cost": 6e-08,
+ "input_cost_per_token": 6e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 204800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 2.4e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/zai-org/glm-5v-turbo": {
+ "cache_read_input_token_cost": 2.4e-07,
+ "input_cost_per_token": 1.2e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 204800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/google/gemma-4-26b-a4b-it": {
+ "input_cost_per_token": 1.3e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4.0000000000000003e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/google/gemma-4-31b-it": {
+ "input_cost_per_token": 1.4e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4.0000000000000003e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/zai-org/glm-5-turbo": {
+ "cache_read_input_token_cost": 2.4e-07,
+ "input_cost_per_token": 1.2e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 202800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/minimax/minimax-m2.7": {
+ "cache_read_input_token_cost": 6e-08,
+ "input_cost_per_token": 3e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 204800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 1.2e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/minimax/minimax-m2.5-highspeed": {
+ "cache_read_input_token_cost": 3e-08,
+ "input_cost_per_token": 6e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 204800,
+ "max_output_tokens": 131100,
+ "max_tokens": 131100,
+ "mode": "chat",
+ "output_cost_per_token": 2.4e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/qwen/qwen3.5-27b": {
+ "input_cost_per_token": 3e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 2.4e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/qwen/qwen3.5-122b-a10b": {
+ "input_cost_per_token": 4.0000000000000003e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 3.2000000000000003e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/qwen/qwen3.5-35b-a3b": {
+ "input_cost_per_token": 2.5e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 2e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/qwen/qwen3.5-397b-a17b": {
+ "input_cost_per_token": 6e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 3.6000000000000003e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/minimax/minimax-m2.5": {
+ "cache_read_input_token_cost": 3e-08,
+ "input_cost_per_token": 3e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 204800,
+ "max_output_tokens": 131100,
+ "max_tokens": 131100,
+ "mode": "chat",
+ "output_cost_per_token": 1.2e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/zai-org/glm-5": {
+ "cache_read_input_token_cost": 2.0000000000000002e-07,
+ "input_cost_per_token": 1e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 202800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 3.2000000000000003e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/qwen/qwen3-coder-next": {
+ "input_cost_per_token": 2.0000000000000002e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 1.5e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/deepseek/deepseek-ocr-2": {
+ "input_cost_per_token": 3e-08,
+ "litellm_provider": "novita",
+ "max_input_tokens": 8192,
+ "max_output_tokens": 8192,
+ "max_tokens": 8192,
+ "mode": "chat",
+ "output_cost_per_token": 3e-08,
+ "source": "https://novita.ai/pricing",
+ "supports_vision": true
+ },
+ "novita/moonshotai/kimi-k2.5": {
+ "cache_read_input_token_cost": 1.0000000000000001e-07,
+ "input_cost_per_token": 6e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 3e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/zai-org/glm-4.7-h": {
+ "cache_read_input_token_cost": 1.1e-07,
+ "input_cost_per_token": 6e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 204800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 2.2e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/zai-org/glm-4.7-flash": {
+ "cache_read_input_token_cost": 1e-08,
+ "input_cost_per_token": 7e-08,
+ "litellm_provider": "novita",
+ "max_input_tokens": 200000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 4.0000000000000003e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/qwen/qwen3.6-35b-a3b": {
+ "input_cost_per_token": 2.48e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 1.4850000000000002e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/deepseek/deepseek_v3": {
+ "input_cost_per_token": 8.900000000000001e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 64000,
+ "max_output_tokens": 16000,
+ "max_tokens": 16000,
+ "mode": "chat",
+ "output_cost_per_token": 8.900000000000001e-07,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/deepseek/deepseek-r1": {
+ "input_cost_per_token": 4e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 64000,
+ "max_output_tokens": 16000,
+ "max_tokens": 16000,
+ "mode": "chat",
+ "output_cost_per_token": 4e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/deepseek/deepseek-v3/community": {
+ "input_cost_per_token": 8.900000000000001e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 64000,
+ "max_output_tokens": 8000,
+ "max_tokens": 8000,
+ "mode": "chat",
+ "output_cost_per_token": 8.900000000000001e-07,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/deepseek/deepseek-r1/community": {
+ "input_cost_per_token": 4e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 64000,
+ "max_output_tokens": 8000,
+ "max_tokens": 8000,
+ "mode": "chat",
+ "output_cost_per_token": 4e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/thudm/glm-4-32b-0414": {
+ "input_cost_per_token": 5.5e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 32000,
+ "max_output_tokens": 32000,
+ "max_tokens": 32000,
+ "mode": "chat",
+ "output_cost_per_token": 1.66e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/meta-llama/llama-3.2-1b-instruct": {
+ "input_cost_per_token": 2e-08,
+ "litellm_provider": "novita",
+ "max_input_tokens": 131000,
+ "max_output_tokens": 32000,
+ "max_tokens": 32000,
+ "mode": "chat",
+ "output_cost_per_token": 2e-08,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_response_schema": true,
+ "supports_vision": false
+ },
+ "wandb/deepseek-ai/DeepSeek-V4-Flash": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 1.4e-07,
+ "output_cost_per_token": 2.8e-07,
+ "cache_read_input_token_cost": 7e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/deepseek-ai/DeepSeek-V4-Flash-0731": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1.3e-07,
+ "output_cost_per_token": 2.8e-07,
+ "cache_read_input_token_cost": 7e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/deepseek-ai/DeepSeek-V4-Pro": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 1.15e-06,
+ "output_cost_per_token": 2.55e-06,
+ "cache_read_input_token_cost": 2e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/google/gemma-4-31B-it": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1e-07,
+ "output_cost_per_token": 3.4e-07,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/ibm-granite/granite-4.1-8b": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 5e-08,
+ "output_cost_per_token": 1e-07,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/JetBrains/Mellum2-12B-A2.5B-Instruct": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 5e-08,
+ "output_cost_per_token": 1e-07,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/meta-llama/Llama-3.1-70B-Instruct": {
+ "max_tokens": 128000,
+ "max_input_tokens": 128000,
+ "input_cost_per_token": 8e-07,
+ "output_cost_per_token": 8e-07,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/MiniMaxAI/MiniMax-M3": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 2.3e-07,
+ "output_cost_per_token": 9.6e-07,
+ "cache_read_input_token_cost": 5e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/moonshotai/Kimi-K2.7-Code": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 7.1e-07,
+ "output_cost_per_token": 3.5e-06,
+ "cache_read_input_token_cost": 1.5e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/moonshotai/Kimi-K2.6": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 6.5e-07,
+ "output_cost_per_token": 3.41e-06,
+ "cache_read_input_token_cost": 1.5e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1e-07,
+ "output_cost_per_token": 2.5e-07,
+ "cache_read_input_token_cost": 5e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 7.5e-07,
+ "output_cost_per_token": 2.75e-06,
+ "cache_read_input_token_cost": 1.5e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/OpenPipe/Qwen3-14B-Instruct": {
+ "max_tokens": 32768,
+ "max_input_tokens": 32768,
+ "input_cost_per_token": 5e-08,
+ "output_cost_per_token": 2.2e-07,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/Qwen/Qwen3.8-27B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 4e-07,
+ "output_cost_per_token": 3e-06,
+ "cache_read_input_token_cost": 1.5e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/Qwen/Qwen3.6-35B-A3B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 2.5e-07,
+ "output_cost_per_token": 1.25e-06,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/Qwen/Qwen3.6-27B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 6e-07,
+ "output_cost_per_token": 3.6e-06,
+ "cache_read_input_token_cost": 1.2e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/Qwen/Qwen3.5-35B-A3B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 2.5e-07,
+ "output_cost_per_token": 1.25e-06,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/Qwen/Qwen3-30B-A3B-Instruct-2507": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1e-07,
+ "output_cost_per_token": 3e-07,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/zai-org/GLM-5.2": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 7.6e-07,
+ "output_cost_per_token": 2.42e-06,
+ "cache_read_input_token_cost": 1.4e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "deepinfra/openai/gpt-oss-120b-Turbo": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 1.5e-07,
+ "output_cost_per_token": 6e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/MiniMaxAI/MiniMax-M2.7": {
+ "max_tokens": 196608,
+ "max_input_tokens": 196608,
+ "input_cost_per_token": 2.5e-07,
+ "output_cost_per_token": 1e-06,
+ "cache_read_input_token_cost": 5e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.8-27B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 4e-07,
+ "output_cost_per_token": 3e-06,
+ "cache_read_input_token_cost": 4e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemma-4-31B-it-Ultra": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 2.7e-07,
+ "output_cost_per_token": 7.6e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/moonshotai/Kimi-K2.5": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 4.5e-07,
+ "output_cost_per_token": 2.25e-06,
+ "cache_read_input_token_cost": 7e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/zai-org/GLM-4.7-Flash": {
+ "max_tokens": 202752,
+ "max_input_tokens": 202752,
+ "input_cost_per_token": 6e-08,
+ "output_cost_per_token": 4e-07,
+ "cache_read_input_token_cost": 1e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/zai-org/GLM-4.6": {
+ "max_tokens": 202752,
+ "max_input_tokens": 202752,
+ "input_cost_per_token": 5e-07,
+ "output_cost_per_token": 2e-06,
+ "cache_read_input_token_cost": 1e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/anthropic/claude-opus-4-8": {
+ "input_cost_per_token": 5e-06,
+ "litellm_provider": "deepinfra",
+ "max_input_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-05,
+ "prompt_cache_min_tokens": 1024,
+ "source": "https://deepinfra.com/pricing",
+ "supports_adaptive_thinking": true,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "deepinfra/anthropic/claude-sonnet-4-6": {
+ "max_tokens": 1000000,
+ "max_input_tokens": 1000000,
+ "input_cost_per_token": 3e-06,
+ "output_cost_per_token": 1.5e-05,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "prompt_cache_min_tokens": 1024,
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_adaptive_thinking": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemini-3.5-flash": {
+ "max_tokens": 1000000,
+ "max_input_tokens": 1000000,
+ "input_cost_per_token": 1.5e-06,
+ "output_cost_per_token": 9e-06,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/XiaomiMiMo/MiMo-V2.5": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 4e-07,
+ "output_cost_per_token": 2e-06,
+ "cache_read_input_token_cost": 8e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3-Max": {
+ "max_tokens": 256000,
+ "max_input_tokens": 256000,
+ "input_cost_per_token": 1.2e-06,
+ "output_cost_per_token": 6e-06,
+ "cache_read_input_token_cost": 2.4e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemma-4-31B-it-turbo": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 9e-08,
+ "output_cost_per_token": 3.4e-07,
+ "cache_read_input_token_cost": 5e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/thinkingmachines/Inkling-Small": {
+ "max_tokens": 524288,
+ "max_input_tokens": 524288,
+ "input_cost_per_token": 4.5e-07,
+ "output_cost_per_token": 1.2e-06,
+ "cache_read_input_token_cost": 1e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/meta-models/Muse-Glimmer-30B": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 3e-07,
+ "output_cost_per_token": 1.2e-06,
+ "cache_read_input_token_cost": 4e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3-Max-Thinking": {
+ "max_tokens": 256000,
+ "max_input_tokens": 256000,
+ "input_cost_per_token": 1.2e-06,
+ "output_cost_per_token": 6e-06,
+ "cache_read_input_token_cost": 2.4e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3-VL-235B-A22B-Instruct": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 2e-07,
+ "output_cost_per_token": 8.8e-07,
+ "cache_read_input_token_cost": 1.1e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3-VL-30B-A3B-Instruct": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1.5e-07,
+ "output_cost_per_token": 6e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.5-27B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 2.6e-07,
+ "output_cost_per_token": 2.6e-06,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.6-35B-A3B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1e-07,
+ "output_cost_per_token": 9.5e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/nvidia/Nemotron-Content-Safety-3.5": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 2e-07,
+ "output_cost_per_token": 2e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/anthropic/claude-opus-5": {
+ "input_cost_per_token": 5e-06,
+ "litellm_provider": "deepinfra",
+ "max_input_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-05,
+ "prompt_cache_min_tokens": 512,
+ "source": "https://deepinfra.com/pricing",
+ "supports_adaptive_thinking": true,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "deepinfra/thinkingmachines/Inkling": {
+ "max_tokens": 524288,
+ "max_input_tokens": 524288,
+ "input_cost_per_token": 9.5e-07,
+ "output_cost_per_token": 4.05e-06,
+ "cache_read_input_token_cost": 1.6e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/moonshotai/Kimi-K2.6": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 7.5e-07,
+ "output_cost_per_token": 3.5e-06,
+ "cache_read_input_token_cost": 1.5e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/deepseek-ai/DeepSeek-V4-Pro-0813": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 1.3e-06,
+ "output_cost_per_token": 2.6e-06,
+ "cache_read_input_token_cost": 1e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.7-Max": {
+ "max_tokens": 256000,
+ "max_input_tokens": 256000,
+ "input_cost_per_token": 2.5e-06,
+ "output_cost_per_token": 7.5e-06,
+ "cache_read_input_token_cost": 5e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/ByteDance/Seed-2.0-mini": {
+ "max_tokens": 256000,
+ "max_input_tokens": 256000,
+ "input_cost_per_token": 1e-07,
+ "output_cost_per_token": 4e-07,
+ "cache_read_input_token_cost": 2e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.8-2.4T-A95B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 2e-06,
+ "output_cost_per_token": 6e-06,
+ "cache_read_input_token_cost": 2e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/MiniMaxAI/MiniMax-M3": {
+ "max_tokens": 524288,
+ "max_input_tokens": 524288,
+ "input_cost_per_token": 2.8e-07,
+ "output_cost_per_token": 1.1e-06,
+ "cache_read_input_token_cost": 5.6e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemini-3.1-flash-lite": {
+ "max_tokens": 1000000,
+ "max_input_tokens": 1000000,
+ "input_cost_per_token": 2.5e-07,
+ "output_cost_per_token": 1.5e-06,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemini-3.7-flash": {
+ "max_tokens": 1000000,
+ "max_input_tokens": 1000000,
+ "input_cost_per_token": 7.5e-07,
+ "output_cost_per_token": 3.75e-06,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/inclusionAI/Ling-3.0-flash": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 6e-08,
+ "output_cost_per_token": 1.8e-07,
+ "cache_read_input_token_cost": 1.2e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/stepfun-ai/Step-3.7-Flash": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 2e-07,
+ "output_cost_per_token": 1.15e-06,
+ "cache_read_input_token_cost": 4e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.5-35B-A3B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1.4e-07,
+ "output_cost_per_token": 1e-06,
+ "cache_read_input_token_cost": 5e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/ByteDance/Seed-1.8": {
+ "max_tokens": 256000,
+ "max_input_tokens": 256000,
+ "input_cost_per_token": 2.5e-07,
+ "output_cost_per_token": 2e-06,
+ "cache_read_input_token_cost": 5e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/tencent/Hy3": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1.4e-07,
+ "output_cost_per_token": 5.8e-07,
+ "cache_read_input_token_cost": 3.5e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/ByteDance/Seed-2.0-code": {
+ "max_tokens": 256000,
+ "max_input_tokens": 256000,
+ "input_cost_per_token": 5e-07,
+ "output_cost_per_token": 3e-06,
+ "cache_read_input_token_cost": 1e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/ByteDance/Seed-2.0-pro": {
+ "max_tokens": 256000,
+ "max_input_tokens": 256000,
+ "input_cost_per_token": 5e-07,
+ "output_cost_per_token": 3e-06,
+ "cache_read_input_token_cost": 1e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/zai-org/GLM-5": {
+ "max_tokens": 202752,
+ "max_input_tokens": 202752,
+ "input_cost_per_token": 6e-07,
+ "output_cost_per_token": 2.08e-06,
+ "cache_read_input_token_cost": 1.2e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/nvidia/Nemotron-3-Nano-30B-A3B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 5e-08,
+ "output_cost_per_token": 2e-07,
+ "cache_read_input_token_cost": 2.5e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/moonshotai/Kimi-K2.7-Code": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 6.8e-07,
+ "output_cost_per_token": 3.4e-06,
+ "cache_read_input_token_cost": 1.36e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/anthropic/claude-sonnet-5": {
+ "input_cost_per_token": 2e-06,
+ "litellm_provider": "deepinfra",
+ "max_input_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 1e-05,
+ "prompt_cache_min_tokens": 1024,
+ "source": "https://deepinfra.com/pricing",
+ "supports_adaptive_thinking": true,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "deepinfra/Qwen/Qwen3.5-397B-A17B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 4.5e-07,
+ "output_cost_per_token": 3e-06,
+ "cache_read_input_token_cost": 2.2e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 8e-08,
+ "output_cost_per_token": 1.8e-07,
+ "cache_read_input_token_cost": 1.6e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemma-4-E4B-it": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 2e-08,
+ "output_cost_per_token": 1e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/deepseek-ai/DeepSeek-V3.2": {
+ "max_tokens": 163840,
+ "max_input_tokens": 163840,
+ "input_cost_per_token": 2.6e-07,
+ "output_cost_per_token": 3.8e-07,
+ "cache_read_input_token_cost": 1.3e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.8-Max": {
+ "max_tokens": 256000,
+ "max_input_tokens": 256000,
+ "input_cost_per_token": 1.65e-06,
+ "output_cost_per_token": 4.951e-06,
+ "cache_read_input_token_cost": 2.06e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/anthropic/claude-fable-5": {
+ "input_cost_per_token": 1e-05,
+ "litellm_provider": "deepinfra",
+ "max_input_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 5e-05,
+ "prompt_cache_min_tokens": 512,
+ "source": "https://deepinfra.com/pricing",
+ "supports_adaptive_thinking": true,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "thinking_always_on": true
+ },
+ "deepinfra/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 5e-07,
+ "output_cost_per_token": 2.2e-06,
+ "cache_read_input_token_cost": 1e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.5-122B-A10B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 2.9e-07,
+ "output_cost_per_token": 2.4e-06,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/zai-org/GLM-5.1": {
+ "max_tokens": 202752,
+ "max_input_tokens": 202752,
+ "input_cost_per_token": 1.05e-06,
+ "output_cost_per_token": 3.5e-06,
+ "cache_read_input_token_cost": 2.05e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/deepseek-ai/DeepSeek-V4-Pro": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 1.3e-06,
+ "output_cost_per_token": 2.6e-06,
+ "cache_read_input_token_cost": 1e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 8.5e-08,
+ "output_cost_per_token": 4e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/zai-org/GLM-5.2": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 7.5e-07,
+ "output_cost_per_token": 2.4e-06,
+ "cache_read_input_token_cost": 1.4e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/moonshotai/Kimi-K3": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 2.85e-06,
+ "output_cost_per_token": 1.425e-05,
+ "cache_read_input_token_cost": 2.85e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/anthropic/claude-opus-4-7": {
+ "input_cost_per_token": 5e-06,
+ "litellm_provider": "deepinfra",
+ "max_input_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-05,
+ "prompt_cache_min_tokens": 2048,
+ "source": "https://deepinfra.com/pricing",
+ "supports_adaptive_thinking": true,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "deepinfra/Qwen/Qwen3.6-27B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 3.2e-07,
+ "output_cost_per_token": 3.2e-06,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemma-4-26B-A4B-it": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 7e-08,
+ "output_cost_per_token": 3.4e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemini-3.1-pro": {
+ "max_tokens": 1000000,
+ "max_input_tokens": 1000000,
+ "input_cost_per_token": 2e-06,
+ "output_cost_per_token": 1.2e-05,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/XiaomiMiMo/MiMo-V2.5-Pro": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 1e-06,
+ "output_cost_per_token": 3e-06,
+ "cache_read_input_token_cost": 2e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/anthropic/claude-haiku-4-5": {
+ "max_tokens": 200000,
+ "max_input_tokens": 200000,
+ "input_cost_per_token": 1e-06,
+ "output_cost_per_token": 5e-06,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "prompt_cache_min_tokens": 4096,
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/deepseek-ai/DeepSeek-V4-Flash": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 9e-08,
+ "output_cost_per_token": 1.8e-07,
+ "cache_read_input_token_cost": 1.8e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/openai/gpt-oss-120b-Ultra": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 2e-07,
+ "output_cost_per_token": 9.5e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.5-9B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1e-07,
+ "output_cost_per_token": 1.5e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/MiniMaxAI/MiniMax-M2.7-Turbo": {
+ "max_tokens": 196608,
+ "max_input_tokens": 196608,
+ "input_cost_per_token": 3.8e-07,
+ "output_cost_per_token": 1.7e-06,
+ "cache_read_input_token_cost": 7e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/zai-org/GLM-4.7": {
+ "max_tokens": 202752,
+ "max_input_tokens": 202752,
+ "input_cost_per_token": 4e-07,
+ "output_cost_per_token": 1.75e-06,
+ "cache_read_input_token_cost": 8e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemma-4-31B-it": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1.3e-07,
+ "output_cost_per_token": 3.8e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "gemini/gemini-omni-1.1-flash": {
+ "input_cost_per_audio_token": 1.5e-06,
+ "input_cost_per_token": 1.5e-06,
+ "litellm_provider": "gemini",
+ "mode": "chat",
+ "output_cost_per_reasoning_token": 9e-06,
+ "output_cost_per_token": 9e-06,
+ "output_cost_per_video_token": 1.75e-05,
+ "rpm": 2000,
+ "source": "https://ai.google.dev/gemini-api/docs/pricing",
+ "supported_endpoints": [
+ "/v1/chat/completions"
+ ],
+ "supported_modalities": [
+ "text",
+ "image",
+ "audio",
+ "video"
+ ],
+ "supported_output_modalities": [
+ "text",
+ "video"
+ ],
+ "supports_audio_input": true,
+ "supports_reasoning": true,
+ "supports_system_messages": true,
+ "supports_video_input": true,
+ "supports_vision": true,
+ "tpm": 800000
+ },
+ "xai/grok-4.20": {
+ "cache_read_input_token_cost": 2e-07,
+ "input_cost_per_token": 1.25e-06,
+ "litellm_provider": "xai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_web_search": true,
+ "input_cost_per_token_above_200k_tokens": 2.5e-06,
+ "output_cost_per_token_above_200k_tokens": 5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+ "supports_prompt_caching": true,
+ "supports_response_schema": true
+ },
+ "xai/grok-4.20-reasoning": {
+ "cache_read_input_token_cost": 2e-07,
+ "input_cost_per_token": 1.25e-06,
+ "litellm_provider": "xai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_web_search": true,
+ "input_cost_per_token_above_200k_tokens": 2.5e-06,
+ "output_cost_per_token_above_200k_tokens": 5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+ "supports_prompt_caching": true,
+ "supports_response_schema": true
+ },
+ "xai/grok-4.20-reasoning-latest": {
+ "cache_read_input_token_cost": 2e-07,
+ "input_cost_per_token": 1.25e-06,
+ "litellm_provider": "xai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_web_search": true,
+ "input_cost_per_token_above_200k_tokens": 2.5e-06,
+ "output_cost_per_token_above_200k_tokens": 5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+ "supports_prompt_caching": true,
+ "supports_response_schema": true
+ },
+ "xai/grok-imagine-image": {
+ "input_cost_per_image": 0.002,
+ "litellm_provider": "xai",
+ "mode": "image_generation",
+ "output_cost_per_image": 0.02,
+ "source": "https://docs.x.ai/docs/models",
+ "supported_endpoints": [
+ "/v1/images/generations"
+ ],
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "image"
+ ]
+ },
+ "xai/grok-imagine-image-2026-03-02": {
+ "input_cost_per_image": 0.002,
+ "litellm_provider": "xai",
+ "mode": "image_generation",
+ "output_cost_per_image": 0.02,
+ "source": "https://docs.x.ai/docs/models",
+ "supported_endpoints": [
+ "/v1/images/generations"
+ ],
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "image"
+ ]
+ },
+ "xai/grok-imagine-image-quality": {
+ "input_cost_per_image": 0.01,
+ "litellm_provider": "xai",
+ "mode": "image_generation",
+ "output_cost_per_image": 0.05,
+ "source": "https://docs.x.ai/docs/models",
+ "supported_endpoints": [
+ "/v1/images/generations"
+ ],
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "image"
+ ]
+ },
+ "xai/grok-imagine-image-quality-20260403": {
+ "input_cost_per_image": 0.01,
+ "litellm_provider": "xai",
+ "mode": "image_generation",
+ "output_cost_per_image": 0.05,
+ "source": "https://docs.x.ai/docs/models",
+ "supported_endpoints": [
+ "/v1/images/generations"
+ ],
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "image"
+ ]
+ },
+ "xai/grok-imagine-image-quality-latest": {
+ "input_cost_per_image": 0.01,
+ "litellm_provider": "xai",
+ "mode": "image_generation",
+ "output_cost_per_image": 0.05,
+ "source": "https://docs.x.ai/docs/models",
+ "supported_endpoints": [
+ "/v1/images/generations"
+ ],
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "image"
+ ]
+ },
+ "xai/grok-imagine-image-pro": {
+ "input_cost_per_image": 0.01,
+ "litellm_provider": "xai",
+ "mode": "image_generation",
+ "output_cost_per_image": 0.05,
+ "source": "https://docs.x.ai/docs/models",
+ "supported_endpoints": [
+ "/v1/images/generations"
+ ],
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "image"
+ ]
+ },
+ "xai/grok-imagine-image-2.0": {
+ "input_cost_per_image": 0.01,
+ "litellm_provider": "xai",
+ "mode": "image_generation",
+ "output_cost_per_image": 0.06,
+ "source": "https://docs.x.ai/docs/models",
+ "supported_endpoints": [
+ "/v1/images/generations"
+ ],
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "image"
+ ]
+ },
+ "xai/grok-4.20-non-reasoning": {
+ "cache_read_input_token_cost": 2e-07,
+ "input_cost_per_token": 1.25e-06,
+ "litellm_provider": "xai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_web_search": true,
+ "input_cost_per_token_above_200k_tokens": 2.5e-06,
+ "output_cost_per_token_above_200k_tokens": 5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+ "supports_response_schema": true
+ },
+ "xai/grok-4.20-non-reasoning-latest": {
+ "cache_read_input_token_cost": 2e-07,
+ "input_cost_per_token": 1.25e-06,
+ "litellm_provider": "xai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_web_search": true,
+ "input_cost_per_token_above_200k_tokens": 2.5e-06,
+ "output_cost_per_token_above_200k_tokens": 5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+ "supports_response_schema": true
+ },
+ "xai/grok-4.20-multi-agent": {
+ "cache_read_input_token_cost": 2e-07,
+ "input_cost_per_token": 1.25e-06,
+ "litellm_provider": "xai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_web_search": true,
+ "input_cost_per_token_above_200k_tokens": 2.5e-06,
+ "output_cost_per_token_above_200k_tokens": 5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+ "supports_response_schema": true
+ },
+ "xai/grok-4.20-multi-agent-latest": {
+ "cache_read_input_token_cost": 2e-07,
+ "input_cost_per_token": 1.25e-06,
+ "litellm_provider": "xai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_web_search": true,
+ "input_cost_per_token_above_200k_tokens": 2.5e-06,
+ "output_cost_per_token_above_200k_tokens": 5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+ "supports_response_schema": true
}
}
diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
index 308813039ca..6a1b6851d3e 100644
--- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
+++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
@@ -5256,7 +5256,9 @@ class MCPServerManager:
proxy_logging_obj: Optional ProxyLogging object for hook integration
host_progress_callback: Optional callback for progress updates
hook_extra_headers: Optional headers injected by pre_mcp_call guardrail
- hooks. Merged last (highest priority) into outbound request headers.
+ hooks. Merged last into outbound request headers, except a hook
+ Authorization header is dropped when an upstream credential already
+ occupies the Authorization slot.
Returns:
CallToolResult from the MCP server
@@ -5347,27 +5349,26 @@ class MCPServerManager:
if hook_extra_headers:
if extra_headers is None:
extra_headers = {}
- if "Authorization" in hook_extra_headers:
- if "Authorization" in extra_headers:
- verbose_logger.warning(
- "MCPServerManager: hook_extra_headers 'Authorization' will overwrite "
- "the existing Authorization header from static_headers. "
- "The hook JWT will take precedence."
- )
- elif server_auth_header is not None:
- # server_auth_header is passed separately to _create_mcp_client as
- # auth_value. Both will reach the upstream server — warn so admins
- # know two Authorization credentials are being sent.
- verbose_logger.warning(
- "MCPServerManager: hook_extra_headers injects 'Authorization' while "
- "server '%s' already has a configured authentication_token. "
- "Both credentials will be sent; the hook header is in extra_headers "
- "and the server token is in auth_value — the upstream server decides "
- "which one wins. Consider unsetting authentication_token if you want "
- "the hook JWT to be the sole credential.",
- mcp_server.server_name or mcp_server.name,
- )
- extra_headers.update(hook_extra_headers)
+ hook_has_authorization: Final = any(k.lower() == "authorization" for k in hook_extra_headers)
+ existing_has_authorization: Final = any(k.lower() == "authorization" for k in extra_headers)
+ server_auth_occupies_authorization: Final = (
+ any(k.lower() == "authorization" for k in server_auth_header)
+ if isinstance(server_auth_header, dict)
+ else server_auth_header is not None and mcp_server.auth_type != MCPAuth.api_key
+ )
+ if hook_has_authorization and (existing_has_authorization or server_auth_occupies_authorization):
+ # Mirror the tools/list signer guard: an upstream credential (user OAuth,
+ # static header, or configured authentication_token) already occupies the
+ # Authorization slot, so the hook must not replace it.
+ verbose_logger.warning(
+ "MCPServerManager: dropping hook-injected 'Authorization' header for "
+ "server '%s' because an upstream credential already occupies the "
+ "Authorization slot; the existing credential is kept.",
+ mcp_server.server_name or mcp_server.name,
+ )
+ extra_headers.update({k: v for k, v in hook_extra_headers.items() if k.lower() != "authorization"})
+ else:
+ extra_headers.update(hook_extra_headers)
# Reset to None if no headers were actually added
if extra_headers is not None and len(extra_headers) == 0:
diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py
index 3c6eb06bc71..57e59dab2d1 100644
--- a/litellm/proxy/_experimental/mcp_server/server.py
+++ b/litellm/proxy/_experimental/mcp_server/server.py
@@ -246,11 +246,12 @@ def _mcp_meta_trace_carrier(req_ctx: object) -> dict[str, str] | None:
"""The W3C trace context (``traceparent``/``tracestate``) the MCP client
propagated in the request's ``params._meta`` (SEP-414), or ``None``.
- When present, per the OTel MCP semconv the MCP span parents to this propagated
- context rather than to the HTTP transport (which is recorded as a link instead).
- When absent, the span nests under the transport span of the request carrying
- this specific message, so a streamable-HTTP session that multiplexes many
- messages still does not glue every message under the session's first request;
+ When present, the MCP span records this propagated context as a span *link*,
+ never the parent — a remote parent would root the span in a trace whose root
+ never reaches the gateway's tracing backend. The span itself nests under the
+ transport span of the request carrying this specific message, so a
+ streamable-HTTP session that multiplexes many messages still does not glue
+ every message under the session's first request;
see ``resolve_mcp_span_context``. The client's W3C Baggage is
deliberately excluded: it is caller-controlled, and the otel baggage processor
stamps allowlisted baggage keys (``litellm.team.id``, ``litellm.metadata.*``,
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index bb26350e1b1..ed49ca2caa9 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -2510,6 +2510,18 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
"are skipped for on-demand GET /health as well as the background health loop."
),
)
+ background_health_check_model_groups: tuple[str, ...] | None = Field(
+ None,
+ description=(
+ "Opt-in allowlist of model group names for background health checks and "
+ "health-check routing. When set, the background loop probes only deployments "
+ "whose model_name is listed, and enable_health_check_routing filters unhealthy "
+ "deployments only within the listed groups; every other group, including newly "
+ "added deployments, is skipped and keeps its configured routing strategy. "
+ "When unset, all deployments participate (opt out per deployment via "
+ "model_info.disable_background_health_check)."
+ ),
+ )
model_list_healthy_only: bool | None = Field(
None,
description=(
diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py
index 90a16052b71..e92d090a2fb 100644
--- a/litellm/proxy/auth/user_api_key_auth.py
+++ b/litellm/proxy/auth/user_api_key_auth.py
@@ -1769,7 +1769,12 @@ async def _user_api_key_auth_builder(
return valid_token
- if valid_token is not None and isinstance(valid_token, UserAPIKeyAuth) and valid_token.team_id is not None:
+ if (
+ valid_token is not None
+ and isinstance(valid_token, UserAPIKeyAuth)
+ and valid_token.team_id is not None
+ and valid_token.team_id != UI_TEAM_ID
+ ):
## UPDATE TEAM VALUES BASED ON CACHED TEAM OBJECT - allows `/team/update` values to work for cached token
try:
team_obj: Final[LiteLLM_TeamTableCachedObj] = await get_team_object(
@@ -2149,6 +2154,8 @@ async def _user_api_key_auth_builder(
# Check 6: Additional Common Checks across jwt + key auth
if valid_token.team_id is not None:
try:
+ if valid_token.team_id == UI_TEAM_ID:
+ raise TeamNotFoundError(team_id=UI_TEAM_ID)
with tracer.trace("litellm.proxy.auth.get_team_object"):
_team_obj = await get_team_object(
team_id=valid_token.team_id,
@@ -2443,7 +2450,7 @@ async def _run_centralized_common_checks(
)
fetch_coros: Final = []
- if user_api_key_auth_obj.team_id is not None:
+ if user_api_key_auth_obj.team_id is not None and user_api_key_auth_obj.team_id != UI_TEAM_ID:
fetch_coros.append(
_safe_fetch(
"team",
@@ -2567,7 +2574,9 @@ async def _run_centralized_common_checks(
else:
raise team_result
else:
- team_object = team_result
+ team_object = (
+ _team_obj_from_token(user_api_key_auth_obj) if user_api_key_auth_obj.team_id == UI_TEAM_ID else team_result
+ )
user_object: LiteLLM_UserTable | None = None if isinstance(user_result, BaseException) else user_result
project_object: Final[LiteLLM_ProjectTableCachedObj | None] = (
diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py
index 4ebcc549cdd..b8b9500ff63 100644
--- a/litellm/proxy/common_utils/reset_budget_job.py
+++ b/litellm/proxy/common_utils/reset_budget_job.py
@@ -1,5 +1,6 @@
import asyncio
import json
+import math
import time
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
from dataclasses import dataclass
@@ -45,6 +46,7 @@ from litellm.repositories.table_repositories import (
)
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.unit_of_work import (
+ LinkedSpendResetWrites,
budget_cascade_unit_of_work,
spend_reset_unit_of_work,
)
@@ -59,7 +61,15 @@ _LINKED_KEYS_WHERE: Final[Mapping[str, object]] = MappingProxyType({"budget_dura
_SPENT_ROWS_WHERE: Final[Mapping[str, object]] = MappingProxyType({"spend": {"gt": 0}})
-class _TeamMembershipRow(Protocol):
+class _BudgetLinkedRow(Protocol):
+ @property
+ def spend(self) -> float | None: ...
+
+ @property
+ def budget_id(self) -> str | None: ...
+
+
+class _TeamMembershipRow(_BudgetLinkedRow, Protocol):
@property
def user_id(self) -> str: ...
@@ -67,26 +77,48 @@ class _TeamMembershipRow(Protocol):
def team_id(self) -> str: ...
-class _KeyRow(Protocol):
+class _KeyRow(_BudgetLinkedRow, Protocol):
@property
def token(self) -> str: ...
-class _OrgRow(Protocol):
+class _OrgRow(_BudgetLinkedRow, Protocol):
@property
def organization_id(self) -> str: ...
-class _TagRow(Protocol):
+class _TagRow(_BudgetLinkedRow, Protocol):
@property
def tag_name(self) -> str: ...
-class _EndUserRow(Protocol):
+class _EndUserRow(_BudgetLinkedRow, Protocol):
@property
def user_id(self) -> str: ...
+def _rollover_enabled() -> bool:
+ return litellm.budget_rollover is True
+
+
+def _rollover_cap(max_budget: float | None) -> float | None:
+ if max_budget is None or not math.isfinite(max_budget):
+ return None
+ return max_budget
+
+
+def _carried_spend(spend: float | None, cap: float | None) -> float:
+ if cap is None:
+ return 0.0
+ return max(0.0, (spend or 0.0) - cap)
+
+
+def _row_carried_spend(row: _BudgetLinkedRow, caps: Mapping[str, float]) -> float:
+ if not caps:
+ return 0.0
+ return _carried_spend(row.spend, caps.get(row.budget_id) if row.budget_id is not None else None)
+
+
def _team_membership_counter_key(row: _TeamMembershipRow) -> str:
return f"spend:team_member:{row.user_id}:{row.team_id}"
@@ -129,6 +161,59 @@ def _budget_link_where(
return {"budget_id": {"in": list(budget_ids)}, **extra}
+def _queue_budget_linked_resets(
+ writes: LinkedSpendResetWrites,
+ cascade: "_BudgetCascade",
+ extra: Mapping[str, object] = MappingProxyType({}),
+) -> None:
+ """Reset one linked table's spend for every expiring tier: tiers with a
+ rollover cap keep spend beyond the cap (decrement preserves writes racing
+ the reset), everything else is zeroed as before. Zero the under-cap rows
+ BEFORE decrementing the over-cap ones: the statements run sequentially in
+ one transaction, so the reverse order lets the zero re-match a row the
+ decrement just moved into the (0, cap] range and erase its carried spend."""
+ for budget_id, cap in cascade.rollover_caps.items():
+ writes.queue_spend_zero(
+ where={"budget_id": budget_id, **extra, "spend": {"gt": 0, "lte": cap}}
+ ) # mutable-ok: prisma where filter must be a dict
+ writes.queue_spend_decrement(
+ where={"budget_id": budget_id, **extra, "spend": {"gt": cap}}, amount=cap
+ ) # mutable-ok: prisma where filter must be a dict
+ plain_ids: Final = tuple(bid for bid in cascade.budget_ids if bid not in cascade.rollover_caps)
+ if plain_ids:
+ writes.queue_spend_zero(where=_budget_link_where(plain_ids, extra))
+
+
+def _queue_enduser_resets(writes: LinkedSpendResetWrites, cascade: "_BudgetCascade") -> None:
+ """End users are matched by id rather than budget link: rows with no
+ budget_id ride the default budget tier (litellm.max_end_user_budget_id).
+ Zero-before-decrement ordering matters here too (see
+ _queue_budget_linked_resets)."""
+ if not cascade.rollover_caps:
+ if cascade.endusers:
+ writes.queue_spend_zero(
+ where={"user_id": {"in": [row.user_id for row in cascade.endusers]}}
+ ) # mutable-ok: prisma where filter must be a dict
+ return
+ tiered: Final = tuple((row.budget_id or litellm.max_end_user_budget_id, row.user_id) for row in cascade.endusers)
+ for budget_id, cap in cascade.rollover_caps.items():
+ if not (
+ user_ids := [uid for bid, uid in tiered if bid == budget_id]
+ ): # mutable-ok: prisma "in" filter takes a list
+ continue
+ writes.queue_spend_zero(
+ where={"user_id": {"in": user_ids}, "spend": {"lte": cap}}
+ ) # mutable-ok: prisma where filter must be a dict
+ writes.queue_spend_decrement(
+ where={"user_id": {"in": user_ids}, "spend": {"gt": cap}}, amount=cap
+ ) # mutable-ok: prisma where filter must be a dict
+ plain: Final = [
+ uid for bid, uid in tiered if bid is None or bid not in cascade.rollover_caps
+ ] # mutable-ok: prisma "in" filter takes a list
+ if plain:
+ writes.queue_spend_zero(where={"user_id": {"in": plain}}) # mutable-ok: prisma where filter must be a dict
+
+
@dataclass(frozen=True, slots=True)
class _BudgetCascade:
"""Everything one budget-tier reset touches, resolved before any write."""
@@ -137,8 +222,9 @@ class _BudgetCascade:
budget_ids: tuple[str, ...] = ()
budget_resets: tuple[tuple[str, datetime], ...] = ()
endusers: tuple[_EndUserRow, ...] = ()
- counter_keys: tuple[str, ...] = ()
+ counter_resets: tuple[tuple[str, float], ...] = ()
cache_keys: tuple[str, ...] = ()
+ rollover_caps: Mapping[str, float] = MappingProxyType({})
@dataclass(frozen=True, slots=True)
@@ -404,8 +490,10 @@ class ResetBudgetJob:
)
@staticmethod
- async def _invalidate_spend_counter(counter_key: str) -> None:
- """Zero a spend counter so a DB-row reset takes effect immediately.
+ async def _invalidate_spend_counter(counter_key: str, new_spend: float = 0.0) -> None:
+ """Overwrite a spend counter with the post-reset value (0, or the carried
+ overage when budget rollover is enabled) so a DB-row reset takes effect
+ immediately.
Call AFTER the DB write commits. Clearing Redis before the DB
commit opens a window where get_current_spend reads 0 from Redis
@@ -414,10 +502,10 @@ class ResetBudgetJob:
try:
from litellm.proxy.proxy_server import spend_counter_cache
- spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=0.0, ttl=60)
+ spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=new_spend, ttl=60)
if spend_counter_cache.redis_cache is not None:
try:
- await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=0.0, ttl=60)
+ await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=new_spend, ttl=60)
except Exception as redis_err:
verbose_proxy_logger.warning(
"Failed to reset spend counter %s in Redis: %s. "
@@ -522,6 +610,15 @@ class ResetBudgetJob:
where=_budget_link_where(budget_ids, _SPENT_ROWS_WHERE),
log_subject="tags",
)
+ rollover_caps: Final[Mapping[str, float]] = MappingProxyType(
+ { # mutable-ok: MappingProxyType wraps a one-shot dict comprehension
+ b.budget_id: cap
+ for b in budgets_to_reset
+ if b.budget_id is not None and (cap := _rollover_cap(b.max_budget)) is not None
+ }
+ if _rollover_enabled()
+ else {} # mutable-ok: empty sentinel immediately frozen by MappingProxyType
+ )
return _BudgetCascade(
budgets=tuple(budgets_to_reset),
budget_ids=budget_ids,
@@ -534,12 +631,16 @@ class ResetBudgetJob:
if b.budget_id is not None and b.budget_duration is not None
),
endusers=await self._collect_endusers_to_reset(budget_ids),
- counter_keys=(
- *(_team_membership_counter_key(row) for row in team_memberships),
- *(_key_counter_key(row) for row in keys),
- *(_org_counter_key(row) for row in orgs),
- *(_tag_counter_key(row) for row in tags),
+ counter_resets=(
+ *(
+ (_team_membership_counter_key(row), _row_carried_spend(row, rollover_caps))
+ for row in team_memberships
+ ),
+ *((_key_counter_key(row), _row_carried_spend(row, rollover_caps)) for row in keys),
+ *((_org_counter_key(row), _row_carried_spend(row, rollover_caps)) for row in orgs),
+ *((_tag_counter_key(row), _row_carried_spend(row, rollover_caps)) for row in tags),
),
+ rollover_caps=rollover_caps,
cache_keys=(
*(key for row in team_memberships for key in _team_membership_cache_keys(row)),
*(key for row in keys for key in _key_cache_keys(row)),
@@ -565,20 +666,18 @@ class ResetBudgetJob:
)
async def _commit_budget_cascade_once(self, cascade: _BudgetCascade) -> None:
- enduser_ids: Final = tuple(row.user_id for row in cascade.endusers)
async with budget_cascade_unit_of_work(self.prisma_client.db.batch_) as uow:
- uow.team_memberships.queue_spend_zero(where=_budget_link_where(cascade.budget_ids))
- uow.keys.queue_spend_zero(where=_budget_link_where(cascade.budget_ids, _LINKED_KEYS_WHERE))
- uow.organizations.queue_spend_zero(where=_budget_link_where(cascade.budget_ids, _SPENT_ROWS_WHERE))
- uow.tags.queue_spend_zero(where=_budget_link_where(cascade.budget_ids, _SPENT_ROWS_WHERE))
- if enduser_ids:
- uow.endusers.queue_spend_zero(where={"user_id": {"in": list(enduser_ids)}})
+ _queue_budget_linked_resets(uow.team_memberships, cascade)
+ _queue_budget_linked_resets(uow.keys, cascade, extra=_LINKED_KEYS_WHERE)
+ _queue_budget_linked_resets(uow.organizations, cascade, extra=_SPENT_ROWS_WHERE)
+ _queue_budget_linked_resets(uow.tags, cascade, extra=_SPENT_ROWS_WHERE)
+ _queue_enduser_resets(uow.endusers, cascade)
for budget_id, budget_reset_at in cascade.budget_resets:
uow.budgets.queue_window_advance(budget_id=budget_id, budget_reset_at=budget_reset_at)
async def _invalidate_budget_cascade_caches(self, cascade: _BudgetCascade) -> None:
- for counter_key in cascade.counter_keys:
- await self._invalidate_spend_counter(counter_key)
+ for counter_key, new_spend in cascade.counter_resets:
+ await self._invalidate_spend_counter(counter_key, new_spend=new_spend)
for cache_key in cascade.cache_keys:
await self._invalidate_user_api_key_cache_entry(cache_key)
@@ -708,7 +807,11 @@ class ResetBudgetJob:
for k in updated_keys:
if k.token is None:
continue
- uow.keys.queue_spend_reset(token=k.token, budget_reset_at=k.budget_reset_at)
+ uow.keys.queue_spend_reset(
+ token=k.token,
+ budget_reset_at=k.budget_reset_at,
+ spend_decrement=k.max_budget if (k.spend or 0.0) > 0.0 else None,
+ )
async def _write_user_reset_updates(self, updated_users: list[LiteLLM_UserTable]) -> None:
"""
@@ -726,7 +829,11 @@ class ResetBudgetJob:
async def _write_user_reset_updates_once(self, updated_users: list[LiteLLM_UserTable]) -> None:
async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow:
for u in updated_users:
- uow.users.queue_spend_reset(user_id=u.user_id, budget_reset_at=u.budget_reset_at)
+ uow.users.queue_spend_reset(
+ user_id=u.user_id,
+ budget_reset_at=u.budget_reset_at,
+ spend_decrement=u.max_budget if (u.spend or 0.0) > 0.0 else None,
+ )
async def _write_team_reset_updates(self, updated_teams: list[LiteLLM_TeamTable]) -> None:
"""
@@ -744,7 +851,11 @@ class ResetBudgetJob:
async def _write_team_reset_updates_once(self, updated_teams: list[LiteLLM_TeamTable]) -> None:
async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow:
for t in updated_teams:
- uow.teams.queue_spend_reset(team_id=t.team_id, budget_reset_at=t.budget_reset_at)
+ uow.teams.queue_spend_reset(
+ team_id=t.team_id,
+ budget_reset_at=t.budget_reset_at,
+ spend_decrement=t.max_budget if (t.spend or 0.0) > 0.0 else None,
+ )
def _emit_phase_failure(
self,
@@ -820,7 +931,7 @@ class ResetBudgetJob:
for k in updated_keys:
token = getattr(k, "token", None)
if token:
- await self._invalidate_spend_counter(f"spend:key:{token}")
+ await self._invalidate_spend_counter(f"spend:key:{token}", new_spend=k.spend or 0.0)
end_time = time.time()
outcome: Final = _ChunkOutcome(
@@ -925,7 +1036,7 @@ class ResetBudgetJob:
for u in updated_users:
user_id = getattr(u, "user_id", None)
if user_id:
- await self._invalidate_spend_counter(f"spend:user:{user_id}")
+ await self._invalidate_spend_counter(f"spend:user:{user_id}", new_spend=u.spend or 0.0)
if user_id == LITELLM_PROXY_BUDGET_NAME:
await self._invalidate_global_proxy_spend_cache()
@@ -1034,7 +1145,7 @@ class ResetBudgetJob:
for t in updated_teams:
team_id = getattr(t, "team_id", None)
if team_id:
- await self._invalidate_spend_counter(f"spend:team:{team_id}")
+ await self._invalidate_spend_counter(f"spend:team:{team_id}", new_spend=t.spend or 0.0)
end_time = time.time()
outcome: Final = _ChunkOutcome(
@@ -1107,10 +1218,11 @@ class ResetBudgetJob:
reset_at: Final = datetime.fromisoformat(reset_at_str.replace("Z", "+00:00")).replace(tzinfo=None)
if reset_at > now:
return False
- spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=0.0)
+ new_value: Final = await ResetBudgetJob._window_carried_spend(window, counter_key, spend_counter_cache)
+ spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=new_value)
if spend_counter_cache.redis_cache is not None:
try:
- await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=0.0)
+ await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=new_value)
except Exception as redis_err:
verbose_proxy_logger.warning("Failed to reset Redis counter %s: %s", counter_key, redis_err)
window["reset_at"] = compute_budget_reset_at(
@@ -1118,6 +1230,27 @@ class ResetBudgetJob:
).isoformat()
return True
+ @staticmethod
+ async def _window_carried_spend(
+ window: Mapping[str, object], counter_key: str, spend_counter_cache: DualCache
+ ) -> float:
+ """Per-window spend lives only in the counter, so the carried overage is
+ read from it before the reset overwrites it."""
+ if not _rollover_enabled():
+ return 0.0
+ window_max: Final = window.get("max_budget")
+ cap: Final = _rollover_cap(window_max) if isinstance(window_max, (int, float)) else None
+ if cap is None:
+ return 0.0
+ try:
+ current: Final = await spend_counter_cache.async_get_cache(key=counter_key)
+ except Exception as e: # noqa: BLE001 # an unreadable counter falls back to a plain zero reset
+ verbose_proxy_logger.warning("Failed to read spend counter %s for rollover: %s", counter_key, e)
+ return 0.0
+ if not isinstance(current, (int, float)):
+ return 0.0
+ return _carried_spend(float(current), cap)
+
async def reset_budget_windows(self) -> None:
"""
For keys and teams with budget_limits, reset any individual windows where
@@ -1222,7 +1355,7 @@ class ResetBudgetJob:
still holds the pre-reset value, admitting requests past the cap.
"""
try:
- item.spend = 0.0
+ item.spend = _carried_spend(item.spend, _rollover_cap(item.max_budget)) if _rollover_enabled() else 0.0
if hasattr(item, "budget_duration") and item.budget_duration is not None:
item.budget_reset_at = compute_budget_reset_at(
budget_duration=item.budget_duration, settings=reset_settings
diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py
index fc761fc1831..4bd007769b8 100644
--- a/litellm/proxy/db/prisma_client.py
+++ b/litellm/proxy/db/prisma_client.py
@@ -887,6 +887,22 @@ class PrismaManager:
return
ProxyExtrasDBManager.apply_replica_identity_full_if_requested()
+ @staticmethod
+ def _raise_if_partitioned_spend_logs() -> None:
+ """`prisma db push` rewrites a doc-partitioned LiteLLM_SpendLogs
+ primary key back to ("request_id"), which Postgres rejects. Fail fast
+ with guidance instead of retrying into that raw error. No-op when
+ litellm-proxy-extras is absent."""
+ try:
+ from litellm_proxy_extras.utils import (
+ PARTITIONED_SPEND_LOGS_PUSH_ERROR,
+ ProxyExtrasDBManager,
+ )
+ except ImportError:
+ return
+ if ProxyExtrasDBManager.spend_logs_is_partitioned():
+ raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
+
@staticmethod
def setup_database(use_migrate: bool = False, use_v2_resolver: bool = False) -> bool:
"""
@@ -921,6 +937,7 @@ class PrismaManager:
use_v2_resolver=use_v2_resolver,
)
else:
+ PrismaManager._raise_if_partitioned_spend_logs()
# Use prisma db push with increased timeout
subprocess.run(
[
diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py
index 9b60595838d..219f6f270ed 100644
--- a/litellm/proxy/health_check.py
+++ b/litellm/proxy/health_check.py
@@ -7,8 +7,11 @@ import sys
import threading
import time
from collections.abc import Mapping, Sequence
+from collections.abc import Set as AbstractSet
from types import MappingProxyType
-from typing import TYPE_CHECKING, Final
+from typing import TYPE_CHECKING, Final, TypeVar
+
+from pydantic import TypeAdapter, ValidationError
import litellm
@@ -16,6 +19,7 @@ if TYPE_CHECKING:
from litellm.router import Router
logger: Final = logging.getLogger(__name__)
+_DeploymentT: Final = TypeVar("_DeploymentT", bound=Mapping[str, object])
from litellm.constants import (
BACKGROUND_HEALTH_CHECK_MAX_TOKENS,
BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING,
@@ -167,6 +171,38 @@ def health_check_filter_kwargs_from_general_settings(
}
+def parse_background_health_check_model_groups(
+ general_settings: Mapping[str, object] | None,
+) -> frozenset[str] | None:
+ """
+ Read ``general_settings.background_health_check_model_groups``.
+
+ ``None`` means the allowlist is unset and every deployment participates
+ (legacy behavior). A list scopes background health checks and health-check
+ routing to deployments whose ``model_name`` is listed. A malformed value
+ raises so the proxy fails at startup instead of silently probing everything.
+ """
+ raw: Final = (general_settings or {}).get("background_health_check_model_groups")
+ if raw is None:
+ return None
+ try:
+ return frozenset(TypeAdapter(list[str]).validate_python(raw))
+ except ValidationError as e:
+ raise ValueError(
+ "general_settings.background_health_check_model_groups must be a list of model group names"
+ ) from e
+
+
+def filter_deployments_to_model_groups(
+ model_list: Sequence[_DeploymentT],
+ model_groups: AbstractSet[str] | None,
+) -> tuple[_DeploymentT, ...]:
+ """Deployments whose ``model_name`` is in ``model_groups``; all of them when unset."""
+ if model_groups is None:
+ return tuple(model_list)
+ return tuple(x for x in model_list if x.get("model_name") in model_groups)
+
+
def filter_deployments_by_id(
model_list: Sequence[Mapping[str, object]],
) -> list:
diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py
index d1c08352919..97999cbb6d7 100644
--- a/litellm/proxy/management_endpoints/key_management_endpoints.py
+++ b/litellm/proxy/management_endpoints/key_management_endpoints.py
@@ -2190,7 +2190,7 @@ async def _get_and_validate_existing_key(
existing_key_row: Final[LiteLLM_VerificationToken | None] = await _prisma_table(
VerificationTokenRepository(prisma_client)
- ).find_unique(where={"token": hashed_token})
+ ).find_unique(where={"token": hashed_token}, include={"object_permission": True})
if existing_key_row is None:
raise ProxyException(
@@ -2442,11 +2442,13 @@ async def _validate_mcp_servers_for_key_update(
check_db_only=True,
)
object_permission_dict: Final = _object_permission_to_dict(data.object_permission)
+ team_unchanged: Final = data.team_id is None or data.team_id == existing_key_row.team_id
normalized_object_permission: Final = await validate_key_mcp_servers_against_team(
object_permission=object_permission_dict,
team_obj=effective_team_obj,
prisma_client=prisma_client,
is_proxy_admin=is_proxy_admin,
+ existing_key_object_permission=existing_key_row.object_permission if team_unchanged else None,
)
await validate_key_search_tools_against_team(
object_permission=object_permission_dict,
diff --git a/litellm/proxy/management_helpers/object_permission_utils.py b/litellm/proxy/management_helpers/object_permission_utils.py
index fb64914f6f5..13080a6cf83 100644
--- a/litellm/proxy/management_helpers/object_permission_utils.py
+++ b/litellm/proxy/management_helpers/object_permission_utils.py
@@ -447,6 +447,36 @@ async def enforce_all_proxy_mcp_servers_grant_is_admin_only(
)
+async def _get_grandfathered_key_mcp_server_ids(
+ existing_object_permission: Optional["LiteLLM_ObjectPermissionTable"],
+ prisma_client: PrismaClient | None,
+) -> frozenset[str]:
+ """
+ Resolve the canonical MCP server IDs a key's stored object_permission already
+ grants. Updates that keep or shrink those grants stay valid even when the
+ team allowlist has since changed; sentinels are excluded so they cannot
+ grandfather anything.
+ """
+ if existing_object_permission is None or prisma_client is None:
+ return frozenset()
+ raw_tool_perms: Final = existing_object_permission.mcp_tool_permissions or {}
+ tool_perm_keys: Final[frozenset[str]] = frozenset(
+ json.loads(raw_tool_perms).keys() if isinstance(raw_tool_perms, str) else raw_tool_perms.keys()
+ )
+ identifiers: Final = (frozenset(existing_object_permission.mcp_servers or []) | tool_perm_keys) - {
+ SpecialMCPServerNames.no_mcp_servers.value,
+ SpecialMCPServerName.all_proxy_servers.value,
+ }
+ return frozenset(
+ _flatten_resolved_mcp_server_ids(
+ await _resolve_mcp_server_identifiers_to_ids(
+ identifiers=set(identifiers),
+ prisma_client=prisma_client,
+ )
+ )
+ )
+
+
async def _get_team_allowed_mcp_servers(
team_obj: Optional["LiteLLM_TeamTableCachedObj"],
prisma_client: PrismaClient | None = None,
@@ -527,10 +557,16 @@ async def validate_key_mcp_servers_against_team(
team_obj: Optional["LiteLLM_TeamTableCachedObj"],
prisma_client: PrismaClient | None = None,
is_proxy_admin: bool = False,
+ existing_key_object_permission: Optional["LiteLLM_ObjectPermissionTable"] = None,
) -> ObjectPermissionDict | None:
"""
Validate that MCP servers requested on a key are within the allowed scope.
+ When ``existing_key_object_permission`` is provided (key updates), servers
+ the key already holds are grandfathered: keeping or removing them stays valid
+ even if the team allowlist has since shrunk, while adding new servers outside
+ the allowlist is still rejected.
+
Rules:
- If key is in a team: key's mcp_servers must be a subset of
(team's allowed servers + allow_all_keys servers)
@@ -589,7 +625,11 @@ async def validate_key_mcp_servers_against_team(
if teamless_admin_assignment:
allowed_servers = all_allowed_servers | active_requested_servers
- disallowed_servers: Final = active_requested_servers - allowed_servers
+ grandfathered_servers: Final = await _get_grandfathered_key_mcp_server_ids(
+ existing_object_permission=existing_key_object_permission,
+ prisma_client=prisma_client,
+ )
+ disallowed_servers: Final = active_requested_servers - allowed_servers - grandfathered_servers
if disallowed_servers:
if team_obj is not None:
team_id = team_obj.team_id
diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py
index 0449802abae..8ac63ba25c9 100644
--- a/litellm/proxy/proxy_cli.py
+++ b/litellm/proxy/proxy_cli.py
@@ -1321,10 +1321,10 @@ def run_server(
use_v2_resolver=use_v2_migration_resolver,
)
except RuntimeError as e:
- # v2 resolver raises on unrecoverable migration errors
- # (e.g. non-idempotent failures, permission issues).
- # v1 never raises here, so this only fires when the
- # operator opted into v2.
+ # Raised on unrecoverable migration errors: the v2
+ # resolver's non-idempotent failures and permission
+ # issues, and any `prisma db push` against a
+ # partitioned LiteLLM_SpendLogs.
print(
f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m",
file=sys.stderr,
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 990682f10a5..af26a9f669e 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -411,7 +411,9 @@ from litellm.proxy.guardrails.init_guardrails import (
initialize_guardrails,
)
from litellm.proxy.health_check import (
+ filter_deployments_to_model_groups,
health_check_filter_kwargs_from_general_settings,
+ parse_background_health_check_model_groups,
perform_health_check,
)
from litellm.proxy.health_endpoints._health_endpoints import router as health_router
@@ -3660,6 +3662,13 @@ async def _run_background_health_check():
_llm_model_list = [
m for m in _llm_model_list if not m.get("model_info", {}).get("disable_background_health_check", False)
]
+ scoped_model_groups = llm_router.background_health_check_model_groups if llm_router is not None else None
+ _llm_model_list = list(filter_deployments_to_model_groups(_llm_model_list, scoped_model_groups))
+ if scoped_model_groups is not None and not _llm_model_list:
+ verbose_proxy_logger.warning(
+ "background_health_check_model_groups matched no deployments; groups=%s",
+ sorted(scoped_model_groups),
+ )
model_count_enabled = len(_llm_model_list)
expected_peak_in_flight = model_count_enabled
if isinstance(health_check_concurrency, int) and health_check_concurrency > 0 and model_count_enabled > 0:
@@ -5239,6 +5248,7 @@ class ProxyConfig:
general_settings = config.get("general_settings", {})
if general_settings is None:
general_settings = {}
+ _bg_hc_model_groups: Final = parse_background_health_check_model_groups(general_settings)
_enable_hc_routing = False
_hc_staleness = None
_hc_ignore_transient = False
@@ -5434,13 +5444,14 @@ class ProxyConfig:
_hc_staleness = general_settings.get("health_check_staleness_threshold", None)
_hc_ignore_transient = general_settings.get("health_check_ignore_transient_errors", False)
verbose_proxy_logger.info(
- "background_health_check_config enabled=%s shared=%s interval_seconds=%s max_concurrency=%s details=%s health_check_routing=%s",
+ "background_health_check_config enabled=%s shared=%s interval_seconds=%s max_concurrency=%s details=%s health_check_routing=%s model_groups=%s",
use_background_health_checks,
use_shared_health_check,
health_check_interval,
health_check_concurrency,
health_check_details,
_enable_hc_routing,
+ sorted(_bg_hc_model_groups) if _bg_hc_model_groups is not None else None,
)
### RBAC ###
@@ -5472,6 +5483,8 @@ class ProxyConfig:
router_params["health_check_staleness_threshold"] = _hc_staleness
if _hc_ignore_transient:
router_params["health_check_ignore_transient_errors"] = True
+ if _bg_hc_model_groups is not None:
+ router_params["background_health_check_model_groups"] = sorted(_bg_hc_model_groups)
## MODEL LIST
model_list: Final = config.get("model_list", None)
if model_list:
@@ -9240,6 +9253,7 @@ class ProxyStartupEvent:
prisma_client,
pod_lock_manager=proxy_logging_obj.db_spend_update_writer.pod_lock_manager,
alert=_alert_ptu_rollup_failure,
+ router=llm_router,
)
scheduler.add_job(
@@ -16402,6 +16416,13 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: Final[dict[str, GeneralSettingsUILiteLLMFie
"tab": "prompt_caching",
"description": "Empty uses Anthropic's 5m default. 1h suits long sessions but doubles the cache write cost.",
},
+ "budget_rollover": { # mutable-ok: registry literal, frozen with its siblings below
+ "type": "Boolean",
+ "description": (
+ "Carry spend beyond max_budget into the next window when budgets reset, instead of "
+ "forgiving it. Applies to key, user, team, team member, org, tag and end-user budgets."
+ ),
+ },
"max_ui_session_budget": {
"type": "Dollar",
"default": 1.0,
diff --git a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py
index 6f1bbaa722b..0e6412a2c64 100644
--- a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py
+++ b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py
@@ -14,7 +14,6 @@ and share the existing unique constraint.
import asyncio
import json
-import sys
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass
from datetime import date, datetime, time, timedelta, timezone
@@ -326,16 +325,6 @@ class _LoadedDeployments:
scanned_ids: frozenset[str]
-def _running_router() -> object | None:
- """The proxy's router, or None outside a running proxy.
-
- Read out of ``sys.modules`` rather than imported, so a rollup driven from a test or a
- script does not pull the whole proxy server in behind it.
- """
- proxy_server: Final = sys.modules.get("litellm.proxy.proxy_server")
- return getattr(proxy_server, "llm_router", None) if proxy_server is not None else None
-
-
def _config_deployments(router: object | None, *, owned_by_db: frozenset[str]) -> tuple[_PTUDeployment, ...]:
"""Deployments the router holds that no ``LiteLLM_ProxyModelTable`` row owns.
@@ -356,15 +345,17 @@ def _config_deployments(router: object | None, *, owned_by_db: frozenset[str]) -
)
-async def _load_ptu_models(prisma_client: "PrismaClient") -> _LoadedDeployments:
+async def _load_ptu_models(prisma_client: "PrismaClient", *, router: object | None) -> _LoadedDeployments:
"""Every deployment carrying valid manual PTU config, and every id the scan saw.
Reserved capacity is billed by the provider whichever file declared it, so a
deployment the proxy only knows from config.yaml accrues alongside the stored ones.
+ The router is handed in rather than read off the proxy module, so a run prices exactly
+ the deployments its caller declares and nothing a co-resident process left behind.
"""
rows: Final = await prisma_client.db.litellm_proxymodeltable.find_many()
db_ids: Final = frozenset(model_id for row in rows if (model_id := str(getattr(row, "model_id", "") or "")))
- config_records: Final = _config_deployments(_running_router(), owned_by_db=db_ids)
+ config_records: Final = _config_deployments(router, owned_by_db=db_ids)
models: Final = tuple(
parsed for parsed in (_parse_ptu_model(row) for row in (*rows, *config_records)) if parsed is not None
)
@@ -380,6 +371,7 @@ async def run_ptu_flat_cost_rollup(
prisma_client: "PrismaClient",
target_date: date | None = None,
may_prune: bool = True,
+ router: object | None = None,
) -> RollupResult:
"""Rollup one UTC day of flat PTU cost across all PTU-configured model deployments.
@@ -406,7 +398,7 @@ async def run_ptu_flat_cost_rollup(
date_str: Final = day.isoformat()
run_started: Final = datetime.now(timezone.utc)
- loaded: Final = await _load_ptu_models(prisma_client)
+ loaded: Final = await _load_ptu_models(prisma_client, router=router)
ptu_models: Final = loaded.models
charges: Final = _aggregate_charges(ptu_models, day)
@@ -527,6 +519,7 @@ async def _existing_sentinel_keys(
async def run_ptu_flat_cost_backfill(
prisma_client: "PrismaClient",
today: date | None = None,
+ router: object | None = None,
) -> BackfillResult:
"""Price the elapsed days of every PTU window that carry no sentinel row yet.
@@ -546,7 +539,7 @@ async def run_ptu_flat_cost_backfill(
verbose_proxy_logger.warning("PTU backfill: prisma_client is None, skipping")
return BackfillResult(start=end, end=end, days_scanned=0, rows_written=0)
- ptu_models: Final = (await _load_ptu_models(prisma_client)).models
+ ptu_models: Final = (await _load_ptu_models(prisma_client, router=router)).models
days: Final = _backfill_window(ptu_models, end)
if not days:
@@ -591,6 +584,7 @@ async def run_scheduled_ptu_rollup(
pod_lock_manager: "PodLockManager | None" = None,
target_date: date | None = None,
alert: Callable[[str], Awaitable[None]] | None = None,
+ router: object | None = None,
) -> RollupResult | None:
"""Run the daily rollup under a cross-pod lock so only one proxy reconciles a day.
@@ -615,7 +609,7 @@ async def run_scheduled_ptu_rollup(
return None
if pod_lock_manager is None or pod_lock_manager.redis_cache is None:
- return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=False)
+ return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=False, router=router)
if not await pod_lock_manager.acquire_lock(cronjob_id=PTU_ROLLUP_JOB_ID, ttl=PTU_ROLLUP_LOCK_TTL_SECONDS):
if await _lock_is_held(pod_lock_manager):
@@ -629,10 +623,10 @@ async def run_scheduled_ptu_rollup(
"PTU rollup: could not take the rollup lock and no other pod holds it, "
"running unguarded rather than skipping the day"
)
- return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=False)
+ return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=False, router=router)
try:
- return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=True)
+ return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=True, router=router)
finally:
await pod_lock_manager.release_lock(cronjob_id=PTU_ROLLUP_JOB_ID)
@@ -657,6 +651,7 @@ async def _run_and_alert(
target_date: date | None,
alert: "Callable[[str], Awaitable[None]] | None",
may_prune: bool = True,
+ router: object | None = None,
) -> RollupResult:
"""Reconcile the day, catch up any days left unpriced, and alert on charges that did not land.
@@ -669,7 +664,9 @@ async def _run_and_alert(
explicit date means reconcile exactly that day, so it stays a single-day operation.
Its failure is contained: the day's own result is returned either way.
"""
- result: Final = await run_ptu_flat_cost_rollup(prisma_client, target_date=target_date, may_prune=may_prune)
+ result: Final = await run_ptu_flat_cost_rollup(
+ prisma_client, target_date=target_date, may_prune=may_prune, router=router
+ )
if result.rows_failed:
await _deliver_alert(
alert,
@@ -686,7 +683,7 @@ async def _run_and_alert(
"by the provider with nothing attributing it here. Extend the window, or retire the deployment.",
)
if target_date is None:
- await _backfill_and_alert(prisma_client, alert=alert)
+ await _backfill_and_alert(prisma_client, alert=alert, router=router)
return result
@@ -694,6 +691,7 @@ async def _backfill_and_alert(
prisma_client: "PrismaClient",
*,
alert: "Callable[[str], Awaitable[None]] | None",
+ router: object | None = None,
) -> None:
"""Catch up unpriced PTU days, alerting on charges that did not land.
@@ -701,7 +699,7 @@ async def _backfill_and_alert(
caller whatever the catch-up pass does.
"""
try:
- backfill: Final = await run_ptu_flat_cost_backfill(prisma_client)
+ backfill: Final = await run_ptu_flat_cost_backfill(prisma_client, router=router)
except Exception as exc: # noqa: BLE001 # the catch-up pass must not fail the day's rollup
verbose_proxy_logger.error("PTU backfill: catch-up pass failed, the day's rollup still stands: %s", exc)
return
diff --git a/litellm/repositories/unit_of_work.py b/litellm/repositories/unit_of_work.py
index e504baceb9f..eb11ebe3b9c 100644
--- a/litellm/repositories/unit_of_work.py
+++ b/litellm/repositories/unit_of_work.py
@@ -19,32 +19,57 @@ from collections.abc import AsyncGenerator, Callable, Mapping
from contextlib import asynccontextmanager
from dataclasses import dataclass
from datetime import datetime
+from typing import Final
from litellm.repositories.prisma_protocols import BatchTable, PrismaBatch
+def _spend_reset_data(budget_reset_at: datetime | None, spend_decrement: float | None) -> Mapping[str, object]:
+ spend: Final[object] = (
+ {"decrement": spend_decrement} # mutable-ok: prisma update payload must be a dict
+ if spend_decrement is not None
+ else 0
+ )
+ return {"spend": spend, "budget_reset_at": budget_reset_at} # mutable-ok: prisma update payload must be a dict
+
+
@dataclass(frozen=True, slots=True)
class KeySpendResetWrites:
table: BatchTable
- def queue_spend_reset(self, token: str, budget_reset_at: datetime | None) -> None:
- self.table.update(where={"token": token}, data={"spend": 0, "budget_reset_at": budget_reset_at})
+ def queue_spend_reset(
+ self, token: str, budget_reset_at: datetime | None, spend_decrement: float | None = None
+ ) -> None:
+ self.table.update(
+ where={"token": token}, # mutable-ok: prisma where filter must be a dict
+ data=_spend_reset_data(budget_reset_at, spend_decrement),
+ )
@dataclass(frozen=True, slots=True)
class UserSpendResetWrites:
table: BatchTable
- def queue_spend_reset(self, user_id: str, budget_reset_at: datetime | None) -> None:
- self.table.update(where={"user_id": user_id}, data={"spend": 0, "budget_reset_at": budget_reset_at})
+ def queue_spend_reset(
+ self, user_id: str, budget_reset_at: datetime | None, spend_decrement: float | None = None
+ ) -> None:
+ self.table.update(
+ where={"user_id": user_id}, # mutable-ok: prisma where filter must be a dict
+ data=_spend_reset_data(budget_reset_at, spend_decrement),
+ )
@dataclass(frozen=True, slots=True)
class TeamSpendResetWrites:
table: BatchTable
- def queue_spend_reset(self, team_id: str, budget_reset_at: datetime | None) -> None:
- self.table.update(where={"team_id": team_id}, data={"spend": 0, "budget_reset_at": budget_reset_at})
+ def queue_spend_reset(
+ self, team_id: str, budget_reset_at: datetime | None, spend_decrement: float | None = None
+ ) -> None:
+ self.table.update(
+ where={"team_id": team_id}, # mutable-ok: prisma where filter must be a dict
+ data=_spend_reset_data(budget_reset_at, spend_decrement),
+ )
@dataclass(frozen=True, slots=True)
@@ -54,6 +79,14 @@ class LinkedSpendResetWrites:
def queue_spend_zero(self, where: Mapping[str, object]) -> None:
self.table.update_many(where=where, data={"spend": 0})
+ def queue_spend_decrement(self, where: Mapping[str, object], amount: float) -> None:
+ """``decrement`` rather than a read-then-set, so spend written between the
+ cascade's read and its commit survives the reset instead of being erased."""
+ self.table.update_many(
+ where=where,
+ data={"spend": {"decrement": amount}}, # mutable-ok: prisma update payload must be a dict
+ )
+
@dataclass(frozen=True, slots=True)
class BudgetWindowWrites:
diff --git a/litellm/router.py b/litellm/router.py
index f0ebb539bb7..021dafa9791 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -602,6 +602,7 @@ class Router:
enable_health_check_routing: bool = False,
health_check_staleness_threshold: int | None = None,
health_check_ignore_transient_errors: bool = False,
+ background_health_check_model_groups: Sequence[str] | None = None,
enable_weighted_failover: bool = False,
) -> None:
"""
@@ -811,6 +812,11 @@ class Router:
self.enable_health_check_routing = enable_health_check_routing
self.enable_weighted_failover = enable_weighted_failover
self.health_check_ignore_transient_errors = health_check_ignore_transient_errors
+ self.background_health_check_model_groups: frozenset[str] | None = (
+ frozenset(background_health_check_model_groups)
+ if background_health_check_model_groups is not None
+ else None
+ )
_staleness: Final = health_check_staleness_threshold or (
DEFAULT_HEALTH_CHECK_INTERVAL * DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER
)
@@ -9210,7 +9216,11 @@ class Router:
}
if model_id is not None:
- litellm.register_model(model_cost={model_id: model_info}, persist_across_reloads=False)
+ litellm.register_model(
+ model_cost={model_id: model_info},
+ persist_across_reloads=False,
+ warning_display_name=model,
+ )
## OLD MODEL REGISTRATION ## Kept to prevent breaking changes
backend_keys: Final = Router._backend_cost_map_keys(model=model, custom_llm_provider=custom_llm_provider)
@@ -12719,6 +12729,10 @@ class Router:
"""
Filter out deployments marked unhealthy by background health checks.
No-op when enable_health_check_routing is False.
+ When background_health_check_model_groups is set, only deployments in the
+ listed model groups are filtered; every other group keeps its configured
+ routing strategy untouched, and a router-level allowed_fails_policy no
+ longer disables the filter for the listed groups.
Returns all deployments if health state is unavailable, stale, or would
exclude every candidate (safety net).
"""
@@ -12727,8 +12741,10 @@ class Router:
# When allowed_fails_policy is set, cooldown is the sole routing exclusion
# mechanism -- skip the binary health check filter so the policy threshold
- # is respected before any deployment is excluded.
- if self.allowed_fails_policy is not None:
+ # is respected before any deployment is excluded. With a model-group
+ # allowlist the filter is already scoped, so listed groups keep it.
+ scoped_groups: Final = self.background_health_check_model_groups
+ if self.allowed_fails_policy is not None and scoped_groups is None:
return healthy_deployments
unhealthy_ids: Final = await self.health_state_cache.async_get_unhealthy_deployment_ids(
@@ -12737,7 +12753,12 @@ class Router:
if not unhealthy_ids:
return healthy_deployments
- filtered: Final = [d for d in healthy_deployments if d["model_info"]["id"] not in unhealthy_ids]
+ filtered: Final = [
+ d
+ for d in healthy_deployments
+ if d["model_info"]["id"] not in unhealthy_ids
+ or (scoped_groups is not None and d["model_name"] not in scoped_groups)
+ ]
if not filtered:
verbose_router_logger.warning("All deployments marked unhealthy by health checks, bypassing health filter")
@@ -12754,14 +12775,20 @@ class Router:
if not self.enable_health_check_routing:
return healthy_deployments
- if self.allowed_fails_policy is not None:
+ scoped_groups: Final = self.background_health_check_model_groups
+ if self.allowed_fails_policy is not None and scoped_groups is None:
return healthy_deployments
unhealthy_ids: Final = self.health_state_cache.get_unhealthy_deployment_ids(parent_otel_span=parent_otel_span)
if not unhealthy_ids:
return healthy_deployments
- filtered: Final = [d for d in healthy_deployments if d["model_info"]["id"] not in unhealthy_ids]
+ filtered: Final = [
+ d
+ for d in healthy_deployments
+ if d["model_info"]["id"] not in unhealthy_ids
+ or (scoped_groups is not None and d["model_name"] not in scoped_groups)
+ ]
if not filtered:
verbose_router_logger.warning("All deployments marked unhealthy by health checks, bypassing health filter")
diff --git a/litellm/router_utils/health_state_cache.py b/litellm/router_utils/health_state_cache.py
index 95094f7abfa..22d816e13e9 100644
--- a/litellm/router_utils/health_state_cache.py
+++ b/litellm/router_utils/health_state_cache.py
@@ -43,12 +43,33 @@ class DeploymentHealthCache:
self.staleness_threshold = staleness_threshold
def set_deployment_health_states(self, states: dict[str, DeploymentHealthStateValue]) -> None:
- """Bulk-write all deployment health states as a single cache entry."""
+ """Merge the given states into the shared cache entry, pruning expired ones.
+
+ Merging instead of replacing lets writers probing different deployment
+ scopes (e.g. pods with different background health check allowlists)
+ coexist on the one shared entry without erasing each other's results.
+ The snapshot is read from Redis when available, since a pod-local read
+ would only ever see this writer's own previous merge. When the Redis
+ read comes back empty (a miss, or a swallowed connection error), the
+ pod-local copy of the last merge is used so peers are not erased.
+ """
try:
+ redis_raw: Final = (
+ self.cache.redis_cache.get_cache(self.CACHE_KEY) if self.cache.redis_cache is not None else None
+ )
+ raw: Final = redis_raw if isinstance(redis_raw, dict) else self.cache.get_cache(key=self.CACHE_KEY)
+ existing: Final = raw if isinstance(raw, dict) else {}
+ expiry_seconds: Final = self.staleness_threshold * 1.5
+ now: Final = time.time()
+ merged: Final = {
+ model_id: state
+ for model_id, state in {**existing, **states}.items()
+ if isinstance(state, dict) and (now - state.get("timestamp", 0)) < expiry_seconds
+ }
self.cache.set_cache(
key=self.CACHE_KEY,
- value=states,
- ttl=int(self.staleness_threshold * 1.5),
+ value=merged,
+ ttl=int(expiry_seconds),
)
except Exception as e:
verbose_logger.error(
diff --git a/litellm/utils.py b/litellm/utils.py
index a26b2c5b440..b164a9c4671 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -2948,7 +2948,12 @@ def reapply_runtime_model_cost_registrations() -> None:
register_model(model_cost=dict(_runtime_registered_model_cost)) # mutable-ok: snapshot, replay rewrites it
-def register_model(model_cost: str | dict, *, persist_across_reloads: bool = True):
+def register_model(
+ model_cost: str | dict,
+ *,
+ persist_across_reloads: bool = True,
+ warning_display_name: str | None = None,
+):
"""
Register new / Override existing models (and their pricing) to specific providers.
Provide EITHER a model cost dictionary or a url to a hosted json blob
@@ -2968,6 +2973,10 @@ def register_model(model_cost: str | dict, *, persist_across_reloads: bool = Tru
registering a model is declaring durable intent. Pass False for a
registration that only describes one request, so it is dropped rather than
re-asserted over every future catalog.
+
+ ``warning_display_name`` names the model in the missing-cache-pricing
+ warning instead of the registered key, for callers that register under an
+ opaque key (e.g. the router's hashed deployment ids).
"""
loaded_model_cost = {}
@@ -3014,10 +3023,15 @@ def register_model(model_cost: str | dict, *, persist_across_reloads: bool = Tru
elif (
value.get("cache_creation_input_token_cost") is None
and value.get("cache_read_input_token_cost") is None
+ and value.get("tiered_pricing") is None
+ and (
+ value.get("input_cost_per_token") is not None
+ or value.get("output_cost_per_token") is not None
+ )
):
verbose_logger.warning(
- "register_model: model=%s not in built-in cost map and no prefix/region variant matched; cache cost fields will default to 0. To track cache cost, add cache_creation_input_token_cost and cache_read_input_token_cost to model_info",
- key,
+ "register_model: model=%s has custom pricing but not in built-in cost map and no prefix/region variant matched; cache_creation_input_token_cost and cache_read_input_token_cost will default to 0 for this model (input/output cost tracking is unaffected). To track cache cost, add them to model_info",
+ warning_display_name or key,
)
# ``get_model_info`` returns ``litellm_provider: None`` when the
# provider is unknown (e.g. custom deployments registered via
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index fc10b61b7cd..b21e8a60599 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -4691,7 +4691,7 @@
"supports_web_search": false
},
"azure/gpt-4.1-nano": {
- "deprecation_date": "2026-10-14",
+ "deprecation_date": "2027-04-14",
"cache_read_input_token_cost": 2.5e-08,
"input_cost_per_token": 1e-07,
"input_cost_per_token_batches": 5e-08,
@@ -4724,7 +4724,7 @@
"supports_vision": true
},
"azure/gpt-4.1-nano-2025-04-14": {
- "deprecation_date": "2026-10-14",
+ "deprecation_date": "2027-04-14",
"cache_read_input_token_cost": 2.5e-08,
"input_cost_per_token": 1e-07,
"input_cost_per_token_batches": 5e-08,
@@ -15167,6 +15167,34 @@
"output_dbu_cost_per_token": 7.143e-06,
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
},
+ "databricks/databricks-glm-5-2": {
+ "cache_creation_input_token_cost": 1.4e-06,
+ "cache_read_input_token_cost": 2.5998e-07,
+ "input_cost_per_token": 1.4e-06,
+ "input_dbu_cost_per_token": 2e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 4.39999e-06,
+ "output_dbu_cost_per_token": 6.2857e-05,
+ "source": "https://www.databricks.com/product/pricing/foundation-model-serving",
+ "supported_modalities": [
+ "text"
+ ],
+ "supported_output_modalities": [
+ "text"
+ ],
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
"databricks/databricks-gpt-5": {
"cache_creation_input_token_cost": 1.24999e-06,
"cache_read_input_token_cost": 1.2502e-07,
@@ -15434,6 +15462,35 @@
"output_vector_size": 1024,
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
},
+ "databricks/databricks-kimi-k3": {
+ "cache_creation_input_token_cost": 2.99999e-06,
+ "cache_read_input_token_cost": 3.0002e-07,
+ "input_cost_per_token": 2.99999e-06,
+ "input_dbu_cost_per_token": 4.2857e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1048576,
+ "max_tokens": 1048576,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.500002e-05,
+ "output_dbu_cost_per_token": 0.000214286,
+ "source": "https://www.databricks.com/product/pricing/foundation-model-serving",
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "text"
+ ],
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
"databricks/databricks-llama-2-70b-chat": {
"cache_creation_input_token_cost": 5.0001e-07,
"cache_read_input_token_cost": 5.0001e-07,
@@ -16112,12 +16169,13 @@
"max_tokens": 4096,
"max_input_tokens": 4096,
"max_output_tokens": 4096,
- "input_cost_per_token": 8e-08,
- "output_cost_per_token": 9e-08,
+ "input_cost_per_token": 4e-07,
+ "output_cost_per_token": 4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/NousResearch/Hermes-3-Llama-3.1-405B": {
"max_tokens": 131072,
@@ -16134,11 +16192,12 @@
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 3e-07,
- "output_cost_per_token": 3e-07,
+ "input_cost_per_token": 7e-07,
+ "output_cost_per_token": 7e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
- "supports_tool_choice": false
+ "supports_tool_choice": false,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Qwen/QwQ-32B": {
"max_tokens": 131072,
@@ -16155,12 +16214,13 @@
"max_tokens": 32768,
"max_input_tokens": 32768,
"max_output_tokens": 32768,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3.9e-07,
+ "input_cost_per_token": 3.6e-07,
+ "output_cost_per_token": 4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Qwen/Qwen2.5-7B-Instruct": {
"max_tokens": 32768,
@@ -16188,12 +16248,13 @@
"max_tokens": 40960,
"max_input_tokens": 40960,
"max_output_tokens": 40960,
- "input_cost_per_token": 6e-08,
+ "input_cost_per_token": 1.2e-07,
"output_cost_per_token": 2.4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Qwen/Qwen3-235B-A22B": {
"max_tokens": 40960,
@@ -16211,11 +16272,12 @@
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"input_cost_per_token": 9e-08,
- "output_cost_per_token": 6e-07,
+ "output_cost_per_token": 5.5e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Qwen/Qwen3-235B-A22B-Thinking-2507": {
"max_tokens": 262144,
@@ -16232,23 +16294,25 @@
"max_tokens": 40960,
"max_input_tokens": 40960,
"max_output_tokens": 40960,
- "input_cost_per_token": 8e-08,
- "output_cost_per_token": 2.9e-07,
+ "input_cost_per_token": 1.2e-07,
+ "output_cost_per_token": 5e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Qwen/Qwen3-32B": {
"max_tokens": 40960,
"max_input_tokens": 40960,
"max_output_tokens": 40960,
- "input_cost_per_token": 1e-07,
+ "input_cost_per_token": 8e-08,
"output_cost_per_token": 2.8e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct": {
"max_tokens": 262144,
@@ -16265,23 +16329,27 @@
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
- "input_cost_per_token": 2.9e-07,
- "output_cost_per_token": 1.2e-06,
+ "input_cost_per_token": 3e-07,
+ "output_cost_per_token": 1e-06,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "cache_read_input_token_cost": 1e-07,
+ "supports_prompt_caching": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Qwen/Qwen3-Next-80B-A3B-Instruct": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
- "input_cost_per_token": 1.4e-07,
- "output_cost_per_token": 1.4e-06,
+ "input_cost_per_token": 9e-08,
+ "output_cost_per_token": 1.1e-06,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Qwen/Qwen3-Next-80B-A3B-Thinking": {
"max_tokens": 262144,
@@ -16308,11 +16376,12 @@
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 6.5e-07,
- "output_cost_per_token": 7.5e-07,
+ "input_cost_per_token": 8.5e-07,
+ "output_cost_per_token": 8.5e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
- "supports_tool_choice": false
+ "supports_tool_choice": false,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/Sao10K/L3.3-70B-Euryale-v2.3": {
"max_tokens": 131072,
@@ -16438,36 +16507,41 @@
"max_tokens": 163840,
"max_input_tokens": 163840,
"max_output_tokens": 163840,
- "input_cost_per_token": 3.8e-07,
+ "input_cost_per_token": 3.2e-07,
"output_cost_per_token": 8.9e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/deepseek-ai/DeepSeek-V3-0324": {
"max_tokens": 163840,
"max_input_tokens": 163840,
"max_output_tokens": 163840,
- "input_cost_per_token": 2.5e-07,
- "output_cost_per_token": 8.8e-07,
+ "input_cost_per_token": 2.4e-07,
+ "output_cost_per_token": 9e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "cache_read_input_token_cost": 1.35e-07,
+ "supports_prompt_caching": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/deepseek-ai/DeepSeek-V3.1": {
"max_tokens": 163840,
"max_input_tokens": 163840,
"max_output_tokens": 163840,
- "input_cost_per_token": 2.7e-07,
- "output_cost_per_token": 1e-06,
+ "input_cost_per_token": 2.5e-07,
+ "output_cost_per_token": 9.5e-07,
"cache_read_input_token_cost": 2.16e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
"supports_reasoning": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/deepseek-ai/DeepSeek-V3.1-Terminus": {
"max_tokens": 163840,
@@ -16521,33 +16595,36 @@
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"input_cost_per_token": 5e-08,
- "output_cost_per_token": 1e-07,
+ "output_cost_per_token": 1.5e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/google/gemma-3-27b-it": {
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 9e-08,
+ "input_cost_per_token": 8e-08,
"output_cost_per_token": 1.6e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/google/gemma-3-4b-it": {
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 4e-08,
- "output_cost_per_token": 8e-08,
+ "input_cost_per_token": 5e-08,
+ "output_cost_per_token": 1e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/meta-llama/Llama-3.2-11B-Vision-Instruct": {
"max_tokens": 131072,
@@ -16585,34 +16662,37 @@
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 1.3e-07,
- "output_cost_per_token": 3.9e-07,
+ "input_cost_per_token": 1e-07,
+ "output_cost_per_token": 3.2e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_function_calling": true,
- "supports_tool_choice": true
+ "supports_tool_choice": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": {
"max_tokens": 1048576,
"max_input_tokens": 1048576,
"max_output_tokens": 1048576,
- "input_cost_per_token": 1.5e-07,
- "output_cost_per_token": 6e-07,
+ "input_cost_per_token": 2e-07,
+ "output_cost_per_token": 8e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/meta-llama/Llama-4-Scout-17B-16E-Instruct": {
"max_tokens": 327680,
"max_input_tokens": 327680,
"max_output_tokens": 327680,
- "input_cost_per_token": 8e-08,
+ "input_cost_per_token": 1e-07,
"output_cost_per_token": 3e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/meta-llama/Llama-Guard-3-8B": {
"max_tokens": 131072,
@@ -16660,12 +16740,13 @@
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 1e-07,
- "output_cost_per_token": 2.8e-07,
+ "input_cost_per_token": 4e-07,
+ "output_cost_per_token": 4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct": {
"max_tokens": 131072,
@@ -16683,11 +16764,12 @@
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"input_cost_per_token": 2e-08,
- "output_cost_per_token": 3e-08,
+ "output_cost_per_token": 4e-08,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/microsoft/WizardLM-2-8x22B": {
"max_tokens": 65536,
@@ -16714,12 +16796,13 @@
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 2e-08,
- "output_cost_per_token": 4e-08,
+ "input_cost_per_token": 1.9e-08,
+ "output_cost_per_token": 3e-08,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/mistralai/Mistral-Small-24B-Instruct-2501": {
"max_tokens": 32768,
@@ -16801,14 +16884,16 @@
},
"deepinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning": {
"max_input_tokens": 262144,
- "input_cost_per_token": 5e-08,
+ "input_cost_per_token": 8e-08,
"output_cost_per_token": 2e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
- "source": "https://deepinfra.com/nvidia/NVIDIA-Nemotron-3.5-Lightning",
+ "source": "https://deepinfra.com/pricing",
"supports_tool_choice": true,
"supports_function_calling": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "cache_read_input_token_cost": 4e-08,
+ "supports_prompt_caching": true
},
"deepinfra/nvidia/NVIDIA-Nemotron-Nano-9B-v2": {
"max_tokens": 131072,
@@ -16825,23 +16910,25 @@
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 5e-08,
- "output_cost_per_token": 4.5e-07,
+ "input_cost_per_token": 3.7e-08,
+ "output_cost_per_token": 1.7e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/openai/gpt-oss-20b": {
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 4e-08,
- "output_cost_per_token": 1.5e-07,
+ "input_cost_per_token": 3e-08,
+ "output_cost_per_token": 1.4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
- "supports_function_calling": true
+ "supports_function_calling": true,
+ "source": "https://deepinfra.com/pricing"
},
"deepinfra/zai-org/GLM-4.5": {
"max_tokens": 131072,
@@ -18636,6 +18723,22 @@
"supports_tool_choice": true,
"supports_vision": false
},
+ "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813": {
+ "cache_read_input_token_cost": 4.4e-08,
+ "input_cost_per_token": 1.32e-06,
+ "litellm_provider": "fireworks_ai",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 3.96e-06,
+ "source": "https://docs.fireworks.ai/serverless/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
"fireworks_ai/accounts/fireworks/models/firefunction-v2": {
"input_cost_per_token": 9e-07,
"litellm_provider": "fireworks_ai",
@@ -20342,7 +20445,7 @@
"mode": "chat",
"output_cost_per_reasoning_token": 2.5e-06,
"output_cost_per_token": 2.5e-06,
- "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/",
+ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
@@ -20522,7 +20625,7 @@
"mode": "chat",
"output_cost_per_reasoning_token": 4e-07,
"output_cost_per_token": 4e-07,
- "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview",
+ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
@@ -22114,7 +22217,7 @@
"output_cost_per_reasoning_token": 2.5e-06,
"output_cost_per_token": 2.5e-06,
"rpm": 15,
- "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/",
+ "source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
@@ -22162,7 +22265,7 @@
"output_cost_per_reasoning_token": 2.5e-06,
"output_cost_per_token": 2.5e-06,
"rpm": 15,
- "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/",
+ "source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
@@ -22209,7 +22312,7 @@
"output_cost_per_reasoning_token": 4e-07,
"output_cost_per_token": 4e-07,
"rpm": 15,
- "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/",
+ "source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
@@ -22257,7 +22360,7 @@
"output_cost_per_reasoning_token": 4e-07,
"output_cost_per_token": 4e-07,
"rpm": 15,
- "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-lite",
+ "source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
@@ -22876,36 +22979,6 @@
"supports_vision": true,
"tpm": 800000
},
- "gemini/gemini-omni-1.1-flash": {
- "input_cost_per_audio_token": 1.5e-06,
- "input_cost_per_token": 1.5e-06,
- "litellm_provider": "gemini",
- "mode": "chat",
- "output_cost_per_reasoning_token": 9e-06,
- "output_cost_per_token": 9e-06,
- "output_cost_per_video_token": 1.75e-05,
- "rpm": 2000,
- "source": "https://ai.google.dev/gemini-api/docs/pricing",
- "supported_endpoints": [
- "/v1/chat/completions"
- ],
- "supported_modalities": [
- "text",
- "image",
- "audio",
- "video"
- ],
- "supported_output_modalities": [
- "text",
- "video"
- ],
- "supports_audio_input": true,
- "supports_reasoning": true,
- "supports_system_messages": true,
- "supports_video_input": true,
- "supports_vision": true,
- "tpm": 800000
- },
"gemini/gemini-3.1-pro-preview": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 2e-07,
@@ -30871,6 +30944,152 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
+ "mistral/ministral-14b-2512": {
+ "input_cost_per_token": 2e-07,
+ "litellm_provider": "mistral",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 2e-07,
+ "source": "https://docs.mistral.ai/models/ministral-3-14b-25-12",
+ "supports_assistant_prefill": true,
+ "supports_function_calling": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "mistral/ministral-14b-latest": {
+ "input_cost_per_token": 2e-07,
+ "litellm_provider": "mistral",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 2e-07,
+ "source": "https://docs.mistral.ai/models/ministral-3-14b-25-12",
+ "supports_assistant_prefill": true,
+ "supports_function_calling": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "mistral/ministral-3b-2512": {
+ "input_cost_per_token": 1e-07,
+ "litellm_provider": "mistral",
+ "max_input_tokens": 131072,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 1e-07,
+ "source": "https://docs.mistral.ai/models/ministral-3-3b-25-12",
+ "supports_assistant_prefill": true,
+ "supports_function_calling": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "mistral/ministral-3b-latest": {
+ "input_cost_per_token": 1e-07,
+ "litellm_provider": "mistral",
+ "max_input_tokens": 131072,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 1e-07,
+ "source": "https://docs.mistral.ai/models/ministral-3-3b-25-12",
+ "supports_assistant_prefill": true,
+ "supports_function_calling": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "mistral/mistral-embed-2312": {
+ "input_cost_per_token": 1e-07,
+ "litellm_provider": "mistral",
+ "max_input_tokens": 8192,
+ "max_tokens": 8192,
+ "mode": "embedding",
+ "source": "https://docs.mistral.ai/models/mistral-embed-23-12"
+ },
+ "mistral/mistral-medium-3": {
+ "input_cost_per_token": 1.5e-06,
+ "litellm_provider": "mistral",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 7.5e-06,
+ "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04",
+ "supports_assistant_prefill": true,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "mistral/voxtral-mini-transcribe-realtime-latest": {
+ "input_cost_per_second": 0.0001,
+ "litellm_provider": "mistral",
+ "mode": "audio_transcription",
+ "source": "https://docs.mistral.ai/models/model-cards/voxtral-mini-transcribe-realtime-26-02",
+ "supported_endpoints": [
+ "/v1/audio/transcriptions"
+ ],
+ "supported_modalities": [
+ "audio"
+ ],
+ "supported_output_modalities": [
+ "text"
+ ],
+ "supports_audio_input": true
+ },
+ "mistral/voxtral-mini-tts-latest": {
+ "litellm_provider": "mistral",
+ "mode": "audio_speech",
+ "output_cost_per_character": 1.6e-05,
+ "source": "https://docs.mistral.ai/models/model-cards/voxtral-tts-26-03",
+ "supported_endpoints": [
+ "/v1/audio/speech"
+ ],
+ "supported_modalities": [
+ "text"
+ ],
+ "supported_output_modalities": [
+ "audio"
+ ],
+ "supports_audio_output": true
+ },
+ "mistral/voxtral-small-2507": {
+ "input_cost_per_second": 6.666666666666667e-05,
+ "input_cost_per_token": 1e-07,
+ "litellm_provider": "mistral",
+ "max_input_tokens": 32768,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 4e-07,
+ "source": "https://docs.mistral.ai/models/voxtral-small-25-07",
+ "supports_audio_input": true,
+ "supports_function_calling": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true
+ },
+ "mistral/voxtral-small-latest": {
+ "input_cost_per_second": 6.666666666666667e-05,
+ "input_cost_per_token": 1e-07,
+ "litellm_provider": "mistral",
+ "max_input_tokens": 32768,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 4e-07,
+ "source": "https://docs.mistral.ai/models/voxtral-small-25-07",
+ "supports_audio_input": true,
+ "supports_function_calling": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true
+ },
"mistral/zai-glm-5-2": {
"cache_read_input_token_cost": 1.4e-07,
"input_cost_per_token": 1.4e-06,
@@ -38183,7 +38402,7 @@
"max_output_tokens": 20480,
"max_tokens": 20480,
"metadata": {
- "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro"
+ "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813"
},
"mode": "chat",
"output_cost_per_token": 7e-06,
@@ -38212,7 +38431,7 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"metadata": {
- "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro"
+ "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813"
},
"mode": "chat",
"output_cost_per_token": 1.25e-06,
@@ -38227,7 +38446,7 @@
"litellm_provider": "together_ai",
"max_tokens": 16384,
"metadata": {
- "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro"
+ "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813"
},
"mode": "chat",
"output_cost_per_token": 1.7e-06,
@@ -38518,6 +38737,7 @@
"supports_tool_choice": true
},
"together_ai/Qwen/Qwen3.5-397B-A17B": {
+ "cache_read_input_token_cost": 3.5e-07,
"deprecation_date": "2026-06-29",
"input_cost_per_token": 6e-07,
"litellm_provider": "together_ai",
@@ -38527,10 +38747,12 @@
"source": "https://www.together.ai/models/Qwen/Qwen3.5-397B-A17B",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/MiniMaxAI/MiniMax-M3": {
+ "cache_read_input_token_cost": 6e-08,
"input_cost_per_token": 3e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 524288,
@@ -38541,6 +38763,7 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@@ -38584,6 +38807,7 @@
"supports_reasoning": true
},
"together_ai/Qwen/Qwen3.7-Max": {
+ "cache_read_input_token_cost": 1.3e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1000000,
@@ -38591,7 +38815,8 @@
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 3.75e-06,
- "source": "https://docs.together.ai/docs/serverless-models"
+ "source": "https://docs.together.ai/docs/serverless-models",
+ "supports_prompt_caching": true
},
"together_ai/Qwen/Qwen3.7-Plus": {
"input_cost_per_token": 3.2e-07,
@@ -38604,6 +38829,7 @@
"source": "https://docs.together.ai/docs/serverless-models"
},
"together_ai/Qwen/Qwen3.8-2.4T-A95B": {
+ "cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 2.5e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1010000,
@@ -38611,7 +38837,8 @@
"max_tokens": 1010000,
"mode": "chat",
"output_cost_per_token": 6.25e-06,
- "source": "https://docs.together.ai/docs/serverless-models"
+ "source": "https://docs.together.ai/docs/serverless-models",
+ "supports_prompt_caching": true
},
"together_ai/arize-ai/qwen-2-1.5b-instruct": {
"input_cost_per_token": 1e-07,
@@ -38624,6 +38851,7 @@
"source": "https://docs.together.ai/docs/serverless-models"
},
"together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": {
+ "cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 1.4e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
@@ -38634,10 +38862,13 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/deepseek-ai/DeepSeek-V4-Pro": {
+ "deprecation_date": "2026-08-27",
+ "cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1.74e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 512000,
@@ -38648,11 +38879,13 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": {
+ "cache_read_input_token_cost": 1.3e-07,
"input_cost_per_token": 1.32e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
@@ -38663,10 +38896,12 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/google/gemma-3n-E4B-it": {
+ "deprecation_date": "2026-08-25",
"input_cost_per_token": 6e-08,
"litellm_provider": "together_ai",
"max_input_tokens": 32768,
@@ -38702,6 +38937,7 @@
"source": "https://docs.together.ai/docs/serverless-models"
},
"together_ai/meta-llama/Llama-Guard-4-12B": {
+ "deprecation_date": "2026-08-25",
"input_cost_per_token": 2e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
@@ -38712,6 +38948,7 @@
"source": "https://docs.together.ai/docs/serverless-models"
},
"together_ai/meta-models/Muse-Glimmer-30B": {
+ "cache_read_input_token_cost": 4e-08,
"input_cost_per_token": 3.5e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 131072,
@@ -38719,9 +38956,12 @@
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.5e-06,
- "source": "https://docs.together.ai/docs/serverless-models"
+ "source": "https://docs.together.ai/docs/serverless-models",
+ "supports_prompt_caching": true
},
"together_ai/moonshotai/Kimi-K2.7-Code": {
+ "deprecation_date": "2026-08-27",
+ "cache_read_input_token_cost": 1.9e-07,
"input_cost_per_token": 9.5e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 262144,
@@ -38732,11 +38972,13 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"together_ai/moonshotai/Kimi-K3": {
+ "cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
@@ -38747,12 +38989,15 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"together_ai/nvidia/nemotron-3-ultra-550b-a55b": {
+ "deprecation_date": "2026-08-27",
+ "cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 6e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 512288,
@@ -38763,11 +39008,13 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/pearl-ai/gemma-4-31b-it": {
+ "deprecation_date": "2026-08-27",
"input_cost_per_token": 2.8e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 262144,
@@ -38778,6 +39025,7 @@
"source": "https://docs.together.ai/docs/serverless-models"
},
"together_ai/thinkingmachines/Inkling": {
+ "cache_read_input_token_cost": 1.7e-07,
"input_cost_per_token": 1e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 524288,
@@ -38788,10 +39036,12 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/thinkingmachines/Inkling-Small": {
+ "cache_read_input_token_cost": 1e-07,
"input_cost_per_token": 5e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 524288,
@@ -38799,9 +39049,11 @@
"max_tokens": 524288,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
- "source": "https://docs.together.ai/docs/serverless-models"
+ "source": "https://docs.together.ai/docs/serverless-models",
+ "supports_prompt_caching": true
},
"together_ai/zai-org/GLM-5.2": {
+ "cache_read_input_token_cost": 2.6e-07,
"input_cost_per_token": 1.4e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1048575,
@@ -38812,6 +39064,7 @@
"source": "https://docs.together.ai/docs/serverless-models",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
+ "supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
@@ -43090,19 +43343,21 @@
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 0.015,
- "output_cost_per_token": 0.06,
+ "input_cost_per_token": 3e-08,
+ "output_cost_per_token": 1.7e-07,
"litellm_provider": "wandb",
- "mode": "chat"
+ "mode": "chat",
+ "source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/openai/gpt-oss-20b": {
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
- "input_cost_per_token": 0.005,
- "output_cost_per_token": 0.02,
+ "input_cost_per_token": 3e-08,
+ "output_cost_per_token": 1.3e-07,
"litellm_provider": "wandb",
- "mode": "chat"
+ "mode": "chat",
+ "source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/zai-org/GLM-4.5": {
"max_tokens": 131072,
@@ -43126,10 +43381,11 @@
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
- "input_cost_per_token": 0.1,
- "output_cost_per_token": 0.15,
+ "input_cost_per_token": 1e-06,
+ "output_cost_per_token": 1.5e-06,
"litellm_provider": "wandb",
- "mode": "chat"
+ "mode": "chat",
+ "source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": {
"max_tokens": 262144,
@@ -43181,19 +43437,21 @@
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
- "input_cost_per_token": 0.022,
- "output_cost_per_token": 0.022,
+ "input_cost_per_token": 2.2e-07,
+ "output_cost_per_token": 2.2e-07,
"litellm_provider": "wandb",
- "mode": "chat"
+ "mode": "chat",
+ "source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/deepseek-ai/DeepSeek-V3.1": {
"max_tokens": 128000,
- "max_input_tokens": 128000,
+ "max_input_tokens": 161000,
"max_output_tokens": 128000,
- "input_cost_per_token": 0.055,
- "output_cost_per_token": 0.165,
+ "input_cost_per_token": 5.5e-07,
+ "output_cost_per_token": 1.65e-06,
"litellm_provider": "wandb",
- "mode": "chat"
+ "mode": "chat",
+ "source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/deepseek-ai/DeepSeek-R1-0528": {
"max_tokens": 161000,
@@ -43217,10 +43475,11 @@
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
- "input_cost_per_token": 0.071,
- "output_cost_per_token": 0.071,
+ "input_cost_per_token": 7.1e-07,
+ "output_cost_per_token": 7.1e-07,
"litellm_provider": "wandb",
- "mode": "chat"
+ "mode": "chat",
+ "source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": {
"max_tokens": 64000,
@@ -43606,85 +43865,6 @@
"/v1/audio/transcriptions"
]
},
- "xai/grok-2": {
- "input_cost_per_token": 2e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "max_tokens": 131072,
- "mode": "chat",
- "output_cost_per_token": 1e-05,
- "supports_function_calling": true,
- "supports_tool_choice": true,
- "supports_web_search": true
- },
- "xai/grok-2-1212": {
- "input_cost_per_token": 2e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "max_tokens": 131072,
- "mode": "chat",
- "output_cost_per_token": 1e-05,
- "supports_function_calling": true,
- "supports_tool_choice": true,
- "supports_web_search": true
- },
- "xai/grok-2-latest": {
- "input_cost_per_token": 2e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "max_tokens": 131072,
- "mode": "chat",
- "output_cost_per_token": 1e-05,
- "supports_function_calling": true,
- "supports_tool_choice": true,
- "supports_web_search": true
- },
- "xai/grok-2-vision": {
- "input_cost_per_image": 2e-06,
- "input_cost_per_token": 2e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "max_tokens": 32768,
- "mode": "chat",
- "output_cost_per_token": 1e-05,
- "supports_function_calling": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true
- },
- "xai/grok-2-vision-1212": {
- "deprecation_date": "2026-02-28",
- "input_cost_per_image": 2e-06,
- "input_cost_per_token": 2e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "max_tokens": 32768,
- "mode": "chat",
- "output_cost_per_token": 1e-05,
- "supports_function_calling": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true
- },
- "xai/grok-2-vision-latest": {
- "input_cost_per_image": 2e-06,
- "input_cost_per_token": 2e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "max_tokens": 32768,
- "mode": "chat",
- "output_cost_per_token": 1e-05,
- "supports_function_calling": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true
- },
"xai/grok-3": {
"cache_read_input_token_cost": 7.5e-07,
"input_cost_per_token": 3e-06,
@@ -44070,7 +44250,7 @@
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
- "mode": "chat",
+ "mode": "responses",
"output_cost_per_token": 2.5e-06,
"source": "https://docs.x.ai/docs/models",
"supports_function_calling": true,
@@ -44082,7 +44262,10 @@
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_response_schema": true
+ "supports_response_schema": true,
+ "supported_endpoints": [
+ "/v1/responses"
+ ]
},
"xai/grok-4.20-beta-0309-reasoning": {
"cache_read_input_token_cost": 2e-07,
@@ -44126,69 +44309,6 @@
"supports_prompt_caching": true,
"supports_response_schema": true
},
- "xai/grok-4.20": {
- "cache_read_input_token_cost": 2e-07,
- "input_cost_per_token": 1.25e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 1000000,
- "max_output_tokens": 1000000,
- "max_tokens": 1000000,
- "mode": "chat",
- "output_cost_per_token": 2.5e-06,
- "source": "https://docs.x.ai/docs/models",
- "supports_function_calling": true,
- "supports_reasoning": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true,
- "input_cost_per_token_above_200k_tokens": 2.5e-06,
- "output_cost_per_token_above_200k_tokens": 5e-06,
- "cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_prompt_caching": true,
- "supports_response_schema": true
- },
- "xai/grok-4.20-reasoning": {
- "cache_read_input_token_cost": 2e-07,
- "input_cost_per_token": 1.25e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 1000000,
- "max_output_tokens": 1000000,
- "max_tokens": 1000000,
- "mode": "chat",
- "output_cost_per_token": 2.5e-06,
- "source": "https://docs.x.ai/docs/models",
- "supports_function_calling": true,
- "supports_reasoning": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true,
- "input_cost_per_token_above_200k_tokens": 2.5e-06,
- "output_cost_per_token_above_200k_tokens": 5e-06,
- "cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_prompt_caching": true,
- "supports_response_schema": true
- },
- "xai/grok-4.20-reasoning-latest": {
- "cache_read_input_token_cost": 2e-07,
- "input_cost_per_token": 1.25e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 1000000,
- "max_output_tokens": 1000000,
- "max_tokens": 1000000,
- "mode": "chat",
- "output_cost_per_token": 2.5e-06,
- "source": "https://docs.x.ai/docs/models",
- "supports_function_calling": true,
- "supports_reasoning": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true,
- "input_cost_per_token_above_200k_tokens": 2.5e-06,
- "output_cost_per_token_above_200k_tokens": 5e-06,
- "cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_prompt_caching": true,
- "supports_response_schema": true
- },
"xai/grok-4.20-beta-0309-non-reasoning": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1.25e-06,
@@ -44314,19 +44434,6 @@
"supports_vision": true,
"supports_web_search": true
},
- "xai/grok-beta": {
- "input_cost_per_token": 5e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "max_tokens": 131072,
- "mode": "chat",
- "output_cost_per_token": 1.5e-05,
- "supports_function_calling": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true
- },
"xai/grok-code-fast": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1e-06,
@@ -44390,139 +44497,6 @@
"supports_vision": true,
"deprecation_date": "2026-05-15"
},
- "xai/grok-imagine-image": {
- "input_cost_per_image": 0.002,
- "litellm_provider": "xai",
- "mode": "image_generation",
- "output_cost_per_image": 0.02,
- "source": "https://docs.x.ai/docs/models",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "supported_modalities": [
- "text",
- "image"
- ],
- "supported_output_modalities": [
- "image"
- ]
- },
- "xai/grok-imagine-image-2026-03-02": {
- "input_cost_per_image": 0.002,
- "litellm_provider": "xai",
- "mode": "image_generation",
- "output_cost_per_image": 0.02,
- "source": "https://docs.x.ai/docs/models",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "supported_modalities": [
- "text",
- "image"
- ],
- "supported_output_modalities": [
- "image"
- ]
- },
- "xai/grok-imagine-image-quality": {
- "input_cost_per_image": 0.01,
- "litellm_provider": "xai",
- "mode": "image_generation",
- "output_cost_per_image": 0.05,
- "source": "https://docs.x.ai/docs/models",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "supported_modalities": [
- "text",
- "image"
- ],
- "supported_output_modalities": [
- "image"
- ]
- },
- "xai/grok-imagine-image-quality-20260403": {
- "input_cost_per_image": 0.01,
- "litellm_provider": "xai",
- "mode": "image_generation",
- "output_cost_per_image": 0.05,
- "source": "https://docs.x.ai/docs/models",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "supported_modalities": [
- "text",
- "image"
- ],
- "supported_output_modalities": [
- "image"
- ]
- },
- "xai/grok-imagine-image-quality-latest": {
- "input_cost_per_image": 0.01,
- "litellm_provider": "xai",
- "mode": "image_generation",
- "output_cost_per_image": 0.05,
- "source": "https://docs.x.ai/docs/models",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "supported_modalities": [
- "text",
- "image"
- ],
- "supported_output_modalities": [
- "image"
- ]
- },
- "xai/grok-imagine-image-pro": {
- "input_cost_per_image": 0.01,
- "litellm_provider": "xai",
- "mode": "image_generation",
- "output_cost_per_image": 0.05,
- "source": "https://docs.x.ai/docs/models",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "supported_modalities": [
- "text",
- "image"
- ],
- "supported_output_modalities": [
- "image"
- ]
- },
- "xai/grok-imagine-image-2.0": {
- "input_cost_per_image": 0.01,
- "litellm_provider": "xai",
- "mode": "image_generation",
- "output_cost_per_image": 0.06,
- "source": "https://docs.x.ai/docs/models",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "supported_modalities": [
- "text",
- "image"
- ],
- "supported_output_modalities": [
- "image"
- ]
- },
- "xai/grok-vision-beta": {
- "input_cost_per_image": 5e-06,
- "input_cost_per_token": 5e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 8192,
- "max_output_tokens": 8192,
- "max_tokens": 8192,
- "mode": "chat",
- "output_cost_per_token": 1.5e-05,
- "supports_function_calling": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true
- },
"zai.glm-4.7": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock_converse",
@@ -44581,6 +44555,21 @@
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing"
},
+ "zai/glm-5.3": {
+ "cache_creation_input_token_cost": 0,
+ "cache_read_input_token_cost": 2.6e-07,
+ "input_cost_per_token": 1.4e-06,
+ "litellm_provider": "zai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 4.4e-06,
+ "source": "https://docs.z.ai/guides/overview/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true
+ },
"zai/glm-5.1": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 2.6e-07,
@@ -44786,6 +44775,7 @@
]
},
"azure/sora-2": {
+ "deprecation_date": "2026-10-15",
"litellm_provider": "azure",
"mode": "video_generation",
"output_cost_per_video_per_second": 0.1,
@@ -47369,8 +47359,8 @@
"novita/xiaomimimo/mimo-v2-flash": {
"litellm_provider": "novita",
"mode": "chat",
- "input_cost_per_token": 1e-07,
- "output_cost_per_token": 3e-07,
+ "input_cost_per_token": 1.1e-07,
+ "output_cost_per_token": 3.3e-07,
"max_input_tokens": 262144,
"max_output_tokens": 32000,
"max_tokens": 32000,
@@ -47379,8 +47369,8 @@
"supports_tool_choice": true,
"supports_system_messages": true,
"supports_response_schema": true,
- "cache_read_input_token_cost": 2e-08,
- "input_cost_per_token_cache_hit": 2e-08,
+ "cache_read_input_token_cost": 2.4e-08,
+ "input_cost_per_token_cache_hit": 2.4e-08,
"supports_reasoning": true
},
"novita/zai-org/autoglm-phone-9b-multilingual": {
@@ -47400,14 +47390,16 @@
"input_cost_per_token": 6e-07,
"output_cost_per_token": 2.5e-06,
"max_input_tokens": 262144,
- "max_output_tokens": 262144,
- "max_tokens": 262144,
+ "max_output_tokens": 100352,
+ "max_tokens": 100352,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
"supports_response_schema": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "cache_read_input_token_cost": 1.5e-07,
+ "supports_prompt_caching": true
},
"novita/minimax/minimax-m2": {
"litellm_provider": "novita",
@@ -47423,7 +47415,8 @@
"supports_system_messages": true,
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token_cache_hit": 3e-08,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "supports_response_schema": true
},
"novita/paddlepaddle/paddleocr-vl": {
"litellm_provider": "novita",
@@ -47461,7 +47454,9 @@
"max_tokens": 32768,
"supports_vision": true,
"supports_system_messages": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "supports_function_calling": true,
+ "supports_tool_choice": true
},
"novita/zai-org/glm-4.6v": {
"litellm_provider": "novita",
@@ -47526,7 +47521,8 @@
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
- "supports_response_schema": true
+ "supports_response_schema": true,
+ "supports_reasoning": true
},
"novita/qwen/qwen3-next-80b-a3b-thinking": {
"litellm_provider": "novita",
@@ -47638,8 +47634,8 @@
"input_cost_per_token": 6e-07,
"output_cost_per_token": 2.5e-06,
"max_input_tokens": 262144,
- "max_output_tokens": 262144,
- "max_tokens": 262144,
+ "max_output_tokens": 100352,
+ "max_tokens": 100352,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
@@ -47649,8 +47645,8 @@
"novita/qwen/qwen3-coder-480b-a35b-instruct": {
"litellm_provider": "novita",
"mode": "chat",
- "input_cost_per_token": 3e-07,
- "output_cost_per_token": 1.3e-06,
+ "input_cost_per_token": 3.8e-07,
+ "output_cost_per_token": 1.55e-06,
"max_input_tokens": 262144,
"max_output_tokens": 65536,
"max_tokens": 65536,
@@ -47696,8 +47692,8 @@
"input_cost_per_token": 5.7e-07,
"output_cost_per_token": 2.3e-06,
"max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "max_tokens": 131072,
+ "max_output_tokens": 100352,
+ "max_tokens": 100352,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
@@ -47710,8 +47706,8 @@
"input_cost_per_token": 2.7e-07,
"output_cost_per_token": 1.12e-06,
"max_input_tokens": 163840,
- "max_output_tokens": 163840,
- "max_tokens": 163840,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
@@ -47758,7 +47754,8 @@
"max_input_tokens": 16384,
"max_output_tokens": 16384,
"max_tokens": 16384,
- "supports_system_messages": true
+ "supports_system_messages": true,
+ "supports_response_schema": true
},
"novita/google/gemma-3-12b-it": {
"litellm_provider": "novita",
@@ -47837,13 +47834,14 @@
"mode": "chat",
"input_cost_per_token": 1.35e-07,
"output_cost_per_token": 4e-07,
- "max_input_tokens": 131072,
- "max_output_tokens": 120000,
- "max_tokens": 120000,
+ "max_input_tokens": 12288,
+ "max_output_tokens": 12288,
+ "max_tokens": 12288,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
- "supports_system_messages": true
+ "supports_system_messages": true,
+ "supports_response_schema": true
},
"novita/qwen/qwen-2.5-72b-instruct": {
"litellm_provider": "novita",
@@ -47883,7 +47881,8 @@
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "supports_response_schema": true
},
"novita/deepseek/deepseek-r1-0528": {
"litellm_provider": "novita",
@@ -47923,7 +47922,8 @@
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
- "supports_system_messages": true
+ "supports_system_messages": true,
+ "supports_response_schema": true
},
"novita/microsoft/wizardlm-2-8x22b": {
"litellm_provider": "novita",
@@ -47933,7 +47933,8 @@
"max_input_tokens": 65535,
"max_output_tokens": 8000,
"max_tokens": 8000,
- "supports_system_messages": true
+ "supports_system_messages": true,
+ "supports_response_schema": true
},
"novita/deepseek/deepseek-r1-0528-qwen3-8b": {
"litellm_provider": "novita",
@@ -47980,7 +47981,8 @@
"max_output_tokens": 20000,
"max_tokens": 20000,
"supports_system_messages": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "supports_response_schema": true
},
"novita/meta-llama/llama-4-maverick-17b-128e-instruct-fp8": {
"litellm_provider": "novita",
@@ -47991,7 +47993,8 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"supports_vision": true,
- "supports_system_messages": true
+ "supports_system_messages": true,
+ "supports_response_schema": true
},
"novita/meta-llama/llama-4-scout-17b-16e-instruct": {
"litellm_provider": "novita",
@@ -48127,7 +48130,9 @@
"max_output_tokens": 20000,
"max_tokens": 20000,
"supports_system_messages": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "supports_function_calling": true,
+ "supports_tool_choice": true
},
"novita/google/gemma-3-27b-it": {
"litellm_provider": "novita",
@@ -48165,7 +48170,8 @@
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "supports_response_schema": true
},
"novita/Sao10K/L3-8B-Stheno-v3.2": {
"litellm_provider": "novita",
@@ -48233,7 +48239,9 @@
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "cache_read_input_token_cost": 2.5e-08,
+ "supports_prompt_caching": true
},
"novita/qwen/qwen3-vl-30b-a3b-instruct": {
"litellm_provider": "novita",
@@ -48354,10 +48362,12 @@
"input_cost_per_token": 3e-08,
"output_cost_per_token": 3e-08,
"max_input_tokens": 128000,
- "max_output_tokens": 20000,
- "max_tokens": 20000,
+ "max_output_tokens": 8192,
+ "max_tokens": 8192,
"supports_system_messages": true,
- "supports_reasoning": true
+ "supports_reasoning": true,
+ "supports_function_calling": true,
+ "supports_tool_choice": true
},
"novita/qwen/qwen2.5-7b-instruct": {
"litellm_provider": "novita",
@@ -48365,8 +48375,8 @@
"input_cost_per_token": 7e-08,
"output_cost_per_token": 7e-08,
"max_input_tokens": 32000,
- "max_output_tokens": 32000,
- "max_tokens": 32000,
+ "max_output_tokens": 8192,
+ "max_tokens": 8192,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
@@ -49778,14 +49788,14 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-5.6-sol": {
- "input_cost_per_token": 5.5e-06,
- "input_cost_per_token_above_272k_tokens": 1.1e-05,
- "cache_creation_input_token_cost": 6.875e-06,
- "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05,
- "cache_read_input_token_cost": 5.5e-07,
- "cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
- "output_cost_per_token": 3.3e-05,
- "output_cost_per_token_above_272k_tokens": 4.95e-05,
+ "input_cost_per_token": 4.4e-06,
+ "input_cost_per_token_above_272k_tokens": 8.8e-06,
+ "cache_creation_input_token_cost": 5.5e-06,
+ "cache_creation_input_token_cost_above_272k_tokens": 1.1e-05,
+ "cache_read_input_token_cost": 4.4e-07,
+ "cache_read_input_token_cost_above_272k_tokens": 8.8e-07,
+ "output_cost_per_token": 2.2e-05,
+ "output_cost_per_token_above_272k_tokens": 3.3e-05,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
@@ -49843,6 +49853,34 @@
"supports_tool_choice": true,
"supports_vision": true
},
+ "bedrock_mantle/openai.gpt-5.6-cyber": {
+ "input_cost_per_token": 1.375e-05,
+ "cache_creation_input_token_cost": 1.71875e-05,
+ "cache_read_input_token_cost": 1.375e-06,
+ "output_cost_per_token": 8.25e-05,
+ "litellm_provider": "bedrock_mantle",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "responses",
+ "use_openai_responses_path": true,
+ "supported_endpoints": [
+ "/v1/responses"
+ ],
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "text"
+ ],
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
"bedrock_mantle/openai.gpt-5.6-luna": {
"input_cost_per_token": 2.2e-07,
"input_cost_per_token_above_272k_tokens": 4.4e-07,
@@ -49877,14 +49915,14 @@
"supports_vision": true
},
"us.openai.gpt-5.6-sol": {
- "input_cost_per_token": 5.5e-06,
- "input_cost_per_token_above_272k_tokens": 1.1e-05,
- "cache_creation_input_token_cost": 6.875e-06,
- "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05,
- "cache_read_input_token_cost": 5.5e-07,
- "cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
- "output_cost_per_token": 3.3e-05,
- "output_cost_per_token_above_272k_tokens": 4.95e-05,
+ "input_cost_per_token": 4.4e-06,
+ "input_cost_per_token_above_272k_tokens": 8.8e-06,
+ "cache_creation_input_token_cost": 5.5e-06,
+ "cache_creation_input_token_cost_above_272k_tokens": 1.1e-05,
+ "cache_read_input_token_cost": 4.4e-07,
+ "cache_read_input_token_cost_above_272k_tokens": 8.8e-07,
+ "output_cost_per_token": 2.2e-05,
+ "output_cost_per_token_above_272k_tokens": 3.3e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
@@ -49903,14 +49941,14 @@
"supports_vision": true
},
"global.openai.gpt-5.6-sol": {
- "input_cost_per_token": 5e-06,
- "input_cost_per_token_above_272k_tokens": 1e-05,
- "cache_creation_input_token_cost": 6.25e-06,
- "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
- "cache_read_input_token_cost": 5e-07,
- "cache_read_input_token_cost_above_272k_tokens": 1e-06,
- "output_cost_per_token": 3e-05,
- "output_cost_per_token_above_272k_tokens": 4.5e-05,
+ "input_cost_per_token": 4e-06,
+ "input_cost_per_token_above_272k_tokens": 8e-06,
+ "cache_creation_input_token_cost": 5e-06,
+ "cache_creation_input_token_cost_above_272k_tokens": 1e-05,
+ "cache_read_input_token_cost": 4e-07,
+ "cache_read_input_token_cost_above_272k_tokens": 8e-07,
+ "output_cost_per_token": 2e-05,
+ "output_cost_per_token_above_272k_tokens": 3e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
@@ -51185,46 +51223,6 @@
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"supports_response_schema": true
},
- "xai/grok-4.20-non-reasoning": {
- "cache_read_input_token_cost": 2e-07,
- "input_cost_per_token": 1.25e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 1000000,
- "max_output_tokens": 1000000,
- "max_tokens": 1000000,
- "mode": "chat",
- "output_cost_per_token": 2.5e-06,
- "source": "https://docs.x.ai/docs/models",
- "supports_function_calling": true,
- "supports_prompt_caching": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true,
- "input_cost_per_token_above_200k_tokens": 2.5e-06,
- "output_cost_per_token_above_200k_tokens": 5e-06,
- "cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_response_schema": true
- },
- "xai/grok-4.20-non-reasoning-latest": {
- "cache_read_input_token_cost": 2e-07,
- "input_cost_per_token": 1.25e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 1000000,
- "max_output_tokens": 1000000,
- "max_tokens": 1000000,
- "mode": "chat",
- "output_cost_per_token": 2.5e-06,
- "source": "https://docs.x.ai/docs/models",
- "supports_function_calling": true,
- "supports_prompt_caching": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true,
- "input_cost_per_token_above_200k_tokens": 2.5e-06,
- "output_cost_per_token_above_200k_tokens": 5e-06,
- "cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_response_schema": true
- },
"xai/grok-4.20-multi-agent-0309": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1.25e-06,
@@ -51232,7 +51230,7 @@
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
- "mode": "chat",
+ "mode": "responses",
"output_cost_per_token": 2.5e-06,
"source": "https://docs.x.ai/docs/models",
"supports_function_calling": true,
@@ -51244,49 +51242,10 @@
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_response_schema": true
- },
- "xai/grok-4.20-multi-agent": {
- "cache_read_input_token_cost": 2e-07,
- "input_cost_per_token": 1.25e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 1000000,
- "max_output_tokens": 1000000,
- "max_tokens": 1000000,
- "mode": "chat",
- "output_cost_per_token": 2.5e-06,
- "source": "https://docs.x.ai/docs/models",
- "supports_function_calling": true,
- "supports_prompt_caching": true,
- "supports_reasoning": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true,
- "input_cost_per_token_above_200k_tokens": 2.5e-06,
- "output_cost_per_token_above_200k_tokens": 5e-06,
- "cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_response_schema": true
- },
- "xai/grok-4.20-multi-agent-latest": {
- "cache_read_input_token_cost": 2e-07,
- "input_cost_per_token": 1.25e-06,
- "litellm_provider": "xai",
- "max_input_tokens": 1000000,
- "max_output_tokens": 1000000,
- "max_tokens": 1000000,
- "mode": "chat",
- "output_cost_per_token": 2.5e-06,
- "source": "https://docs.x.ai/docs/models",
- "supports_function_calling": true,
- "supports_prompt_caching": true,
- "supports_reasoning": true,
- "supports_tool_choice": true,
- "supports_vision": true,
- "supports_web_search": true,
- "input_cost_per_token_above_200k_tokens": 2.5e-06,
- "output_cost_per_token_above_200k_tokens": 5e-06,
- "cache_read_input_token_cost_above_200k_tokens": 4e-07,
- "supports_response_schema": true
+ "supports_response_schema": true,
+ "supported_endpoints": [
+ "/v1/responses"
+ ]
},
"xai/grok-build-0.1": {
"cache_read_input_token_cost": 2e-07,
@@ -51657,7 +51616,6 @@
"litellm_provider": "gemini",
"mode": "audio_transcription",
"output_cost_per_token": 1.2e-05,
- "rpm": 10,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/audio/transcriptions"
@@ -51670,7 +51628,8 @@
"text"
],
"supports_audio_input": true,
- "tpm": 250000
+ "tpm": 800000,
+ "rpm": 2000
},
"gemini/gemini-3.5-transcribe-live": {
"input_cost_per_audio_token": 3.5e-06,
@@ -51678,7 +51637,6 @@
"litellm_provider": "gemini",
"mode": "audio_transcription",
"output_cost_per_token": 2.1e-05,
- "rpm": 10,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/realtime"
@@ -51690,7 +51648,8 @@
"text"
],
"supports_audio_input": true,
- "tpm": 250000
+ "tpm": 250000,
+ "rpm": 10
},
"perplexity/pplx-embed-context-v1-0.6b": {
"input_cost_per_token": 8e-09,
@@ -51774,14 +51733,14 @@
"supports_embedding_image_input": true
},
"fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731": {
- "cache_read_input_token_cost": 2.8e-08,
- "input_cost_per_token": 1.4e-07,
+ "cache_read_input_token_cost": 7e-09,
+ "input_cost_per_token": 2.2e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
- "output_cost_per_token": 2.8e-07,
+ "output_cost_per_token": 6.6e-07,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
@@ -52088,5 +52047,2328 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
+ },
+ "novita/zai-org/glm-5.3": {
+ "cache_read_input_token_cost": 2.6e-07,
+ "input_cost_per_token": 1.4e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4.4e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/deepseek/deepseek-v4-pro-0813": {
+ "cache_read_input_token_cost": 1.3200000000000002e-07,
+ "input_cost_per_token": 1.32e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 393216,
+ "max_tokens": 393216,
+ "mode": "chat",
+ "output_cost_per_token": 3.96e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/moonshotai/kimi-k3": {
+ "cache_read_input_token_cost": 3e-07,
+ "input_cost_per_token": 3e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 1048576,
+ "max_tokens": 1048576,
+ "mode": "chat",
+ "output_cost_per_token": 1.5e-05,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/tencent/hy3": {
+ "cache_read_input_token_cost": 3.5e-08,
+ "input_cost_per_token": 1.4e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 5.8e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/zai-org/glm-5.2": {
+ "cache_read_input_token_cost": 2.6e-07,
+ "input_cost_per_token": 1.4e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4.4e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/moonshotai/kimi-k2.7-code": {
+ "cache_read_input_token_cost": 1.9e-07,
+ "input_cost_per_token": 9.499999999999999e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 4e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/deepseek/deepseek-v4-flash-vision-exp": {
+ "cache_read_input_token_cost": 2.8e-08,
+ "input_cost_per_token": 4.4e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 393216,
+ "max_tokens": 393216,
+ "mode": "chat",
+ "output_cost_per_token": 1.32e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/deepseek/deepseek-v4-flash-0731": {
+ "cache_read_input_token_cost": 2.8e-08,
+ "input_cost_per_token": 4.4e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 393216,
+ "max_tokens": 393216,
+ "mode": "chat",
+ "output_cost_per_token": 1.32e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/mindai/macaron-v1-venti": {
+ "cache_read_input_token_cost": 3e-07,
+ "input_cost_per_token": 1.5e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4.5e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/minimax/minimax-m3": {
+ "cache_read_input_token_cost": 6e-08,
+ "input_cost_per_token": 3e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 1.2e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/deepseek/deepseek-v4-flash": {
+ "cache_read_input_token_cost": 2.8e-08,
+ "input_cost_per_token": 1.4e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 393216,
+ "max_tokens": 393216,
+ "mode": "chat",
+ "output_cost_per_token": 2.8e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/deepseek/deepseek-v4-pro": {
+ "cache_read_input_token_cost": 1.35e-07,
+ "input_cost_per_token": 1.6000000000000001e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 393216,
+ "max_tokens": 393216,
+ "mode": "chat",
+ "output_cost_per_token": 3.2000000000000003e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/inclusionai/ling-3.0-flash-fast": {
+ "cache_read_input_token_cost": 1.2e-08,
+ "input_cost_per_token": 6e-08,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 1.8e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/qwen/qwen3.8-max": {
+ "cache_read_input_token_cost": 2.5e-07,
+ "input_cost_per_token": 2e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 6e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/inclusionai/ling-3.0-flash": {
+ "cache_read_input_token_cost": 1.2e-08,
+ "input_cost_per_token": 6e-08,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 1.8e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/mindai/macaron-v1-tall": {
+ "cache_read_input_token_cost": 8e-08,
+ "input_cost_per_token": 4.5000000000000003e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 2.6e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/stepfun/step-3.7-flash": {
+ "cache_read_input_token_cost": 4e-08,
+ "input_cost_per_token": 2.0000000000000002e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 256000,
+ "max_tokens": 256000,
+ "mode": "chat",
+ "output_cost_per_token": 1.15e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/nvidia/nemotron-3-nano-30b-a3b": {
+ "input_cost_per_token": 5.0000000000000004e-08,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 32768,
+ "max_tokens": 32768,
+ "mode": "chat",
+ "output_cost_per_token": 2.0000000000000002e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/baidu/cobuddy": {
+ "cache_read_input_token_cost": 7e-08,
+ "input_cost_per_token": 2.8e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 131072,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 1.13e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/xiaomimimo/mimo-v2.5": {
+ "cache_read_input_token_cost": 3.4e-09,
+ "input_cost_per_token": 1.6800000000000002e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 3.3600000000000004e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/qwen/qwen3.7-max": {
+ "cache_read_input_token_cost": 2.5e-07,
+ "input_cost_per_token": 1.25e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 3.75e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/xiaomimimo/mimo-v2.5-pro": {
+ "cache_read_input_token_cost": 4.3e-09,
+ "input_cost_per_token": 5.22e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 1.044e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/qwen/qwen3.6-27b": {
+ "input_cost_per_token": 6e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 3.6000000000000003e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/moonshotai/kimi-k2.6": {
+ "cache_read_input_token_cost": 1.6e-07,
+ "input_cost_per_token": 8.000000000000001e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 3.4e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/zai-org/glm-5.1": {
+ "cache_read_input_token_cost": 2.6e-07,
+ "input_cost_per_token": 1.38e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 204800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4.4e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/minimax/minimax-m2.7-highspeed": {
+ "cache_read_input_token_cost": 6e-08,
+ "input_cost_per_token": 6e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 204800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 2.4e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/zai-org/glm-5v-turbo": {
+ "cache_read_input_token_cost": 2.4e-07,
+ "input_cost_per_token": 1.2e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 204800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/google/gemma-4-26b-a4b-it": {
+ "input_cost_per_token": 1.3e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4.0000000000000003e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/google/gemma-4-31b-it": {
+ "input_cost_per_token": 1.4e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4.0000000000000003e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/zai-org/glm-5-turbo": {
+ "cache_read_input_token_cost": 2.4e-07,
+ "input_cost_per_token": 1.2e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 202800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 4e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/minimax/minimax-m2.7": {
+ "cache_read_input_token_cost": 6e-08,
+ "input_cost_per_token": 3e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 204800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 1.2e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/minimax/minimax-m2.5-highspeed": {
+ "cache_read_input_token_cost": 3e-08,
+ "input_cost_per_token": 6e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 204800,
+ "max_output_tokens": 131100,
+ "max_tokens": 131100,
+ "mode": "chat",
+ "output_cost_per_token": 2.4e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/qwen/qwen3.5-27b": {
+ "input_cost_per_token": 3e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 2.4e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/qwen/qwen3.5-122b-a10b": {
+ "input_cost_per_token": 4.0000000000000003e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 3.2000000000000003e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/qwen/qwen3.5-35b-a3b": {
+ "input_cost_per_token": 2.5e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 2e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/qwen/qwen3.5-397b-a17b": {
+ "input_cost_per_token": 6e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 3.6000000000000003e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/minimax/minimax-m2.5": {
+ "cache_read_input_token_cost": 3e-08,
+ "input_cost_per_token": 3e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 204800,
+ "max_output_tokens": 131100,
+ "max_tokens": 131100,
+ "mode": "chat",
+ "output_cost_per_token": 1.2e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/zai-org/glm-5": {
+ "cache_read_input_token_cost": 2.0000000000000002e-07,
+ "input_cost_per_token": 1e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 202800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 3.2000000000000003e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/qwen/qwen3-coder-next": {
+ "input_cost_per_token": 2.0000000000000002e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 1.5e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/deepseek/deepseek-ocr-2": {
+ "input_cost_per_token": 3e-08,
+ "litellm_provider": "novita",
+ "max_input_tokens": 8192,
+ "max_output_tokens": 8192,
+ "max_tokens": 8192,
+ "mode": "chat",
+ "output_cost_per_token": 3e-08,
+ "source": "https://novita.ai/pricing",
+ "supports_vision": true
+ },
+ "novita/moonshotai/kimi-k2.5": {
+ "cache_read_input_token_cost": 1.0000000000000001e-07,
+ "input_cost_per_token": 6e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 3e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/zai-org/glm-4.7-h": {
+ "cache_read_input_token_cost": 1.1e-07,
+ "input_cost_per_token": 6e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 204800,
+ "max_output_tokens": 131072,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 2.2e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/zai-org/glm-4.7-flash": {
+ "cache_read_input_token_cost": 1e-08,
+ "input_cost_per_token": 7e-08,
+ "litellm_provider": "novita",
+ "max_input_tokens": 200000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 4.0000000000000003e-07,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/qwen/qwen3.6-35b-a3b": {
+ "input_cost_per_token": 2.48e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "mode": "chat",
+ "output_cost_per_token": 1.4850000000000002e-06,
+ "source": "https://novita.ai/pricing",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "novita/deepseek/deepseek_v3": {
+ "input_cost_per_token": 8.900000000000001e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 64000,
+ "max_output_tokens": 16000,
+ "max_tokens": 16000,
+ "mode": "chat",
+ "output_cost_per_token": 8.900000000000001e-07,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/deepseek/deepseek-r1": {
+ "input_cost_per_token": 4e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 64000,
+ "max_output_tokens": 16000,
+ "max_tokens": 16000,
+ "mode": "chat",
+ "output_cost_per_token": 4e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/deepseek/deepseek-v3/community": {
+ "input_cost_per_token": 8.900000000000001e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 64000,
+ "max_output_tokens": 8000,
+ "max_tokens": 8000,
+ "mode": "chat",
+ "output_cost_per_token": 8.900000000000001e-07,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/deepseek/deepseek-r1/community": {
+ "input_cost_per_token": 4e-06,
+ "litellm_provider": "novita",
+ "max_input_tokens": 64000,
+ "max_output_tokens": 8000,
+ "max_tokens": 8000,
+ "mode": "chat",
+ "output_cost_per_token": 4e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/thudm/glm-4-32b-0414": {
+ "input_cost_per_token": 5.5e-07,
+ "litellm_provider": "novita",
+ "max_input_tokens": 32000,
+ "max_output_tokens": 32000,
+ "max_tokens": 32000,
+ "mode": "chat",
+ "output_cost_per_token": 1.66e-06,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_function_calling": true,
+ "supports_response_schema": true,
+ "supports_tool_choice": true,
+ "supports_vision": false
+ },
+ "novita/meta-llama/llama-3.2-1b-instruct": {
+ "input_cost_per_token": 2e-08,
+ "litellm_provider": "novita",
+ "max_input_tokens": 131000,
+ "max_output_tokens": 32000,
+ "max_tokens": 32000,
+ "mode": "chat",
+ "output_cost_per_token": 2e-08,
+ "source": "https://api.novita.ai/v3/openai/models",
+ "supports_response_schema": true,
+ "supports_vision": false
+ },
+ "wandb/deepseek-ai/DeepSeek-V4-Flash": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 1.4e-07,
+ "output_cost_per_token": 2.8e-07,
+ "cache_read_input_token_cost": 7e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/deepseek-ai/DeepSeek-V4-Flash-0731": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1.3e-07,
+ "output_cost_per_token": 2.8e-07,
+ "cache_read_input_token_cost": 7e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/deepseek-ai/DeepSeek-V4-Pro": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 1.15e-06,
+ "output_cost_per_token": 2.55e-06,
+ "cache_read_input_token_cost": 2e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/google/gemma-4-31B-it": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1e-07,
+ "output_cost_per_token": 3.4e-07,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/ibm-granite/granite-4.1-8b": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 5e-08,
+ "output_cost_per_token": 1e-07,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/JetBrains/Mellum2-12B-A2.5B-Instruct": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 5e-08,
+ "output_cost_per_token": 1e-07,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/meta-llama/Llama-3.1-70B-Instruct": {
+ "max_tokens": 128000,
+ "max_input_tokens": 128000,
+ "input_cost_per_token": 8e-07,
+ "output_cost_per_token": 8e-07,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/MiniMaxAI/MiniMax-M3": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 2.3e-07,
+ "output_cost_per_token": 9.6e-07,
+ "cache_read_input_token_cost": 5e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/moonshotai/Kimi-K2.7-Code": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 7.1e-07,
+ "output_cost_per_token": 3.5e-06,
+ "cache_read_input_token_cost": 1.5e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/moonshotai/Kimi-K2.6": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 6.5e-07,
+ "output_cost_per_token": 3.41e-06,
+ "cache_read_input_token_cost": 1.5e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1e-07,
+ "output_cost_per_token": 2.5e-07,
+ "cache_read_input_token_cost": 5e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 7.5e-07,
+ "output_cost_per_token": 2.75e-06,
+ "cache_read_input_token_cost": 1.5e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/OpenPipe/Qwen3-14B-Instruct": {
+ "max_tokens": 32768,
+ "max_input_tokens": 32768,
+ "input_cost_per_token": 5e-08,
+ "output_cost_per_token": 2.2e-07,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/Qwen/Qwen3.8-27B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 4e-07,
+ "output_cost_per_token": 3e-06,
+ "cache_read_input_token_cost": 1.5e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/Qwen/Qwen3.6-35B-A3B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 2.5e-07,
+ "output_cost_per_token": 1.25e-06,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/Qwen/Qwen3.6-27B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 6e-07,
+ "output_cost_per_token": 3.6e-06,
+ "cache_read_input_token_cost": 1.2e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/Qwen/Qwen3.5-35B-A3B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 2.5e-07,
+ "output_cost_per_token": 1.25e-06,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/Qwen/Qwen3-30B-A3B-Instruct-2507": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1e-07,
+ "output_cost_per_token": 3e-07,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "wandb/zai-org/GLM-5.2": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 7.6e-07,
+ "output_cost_per_token": 2.42e-06,
+ "cache_read_input_token_cost": 1.4e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "wandb",
+ "mode": "chat",
+ "supports_vision": false,
+ "source": "https://wandb.ai/site/pricing/tokens/"
+ },
+ "deepinfra/openai/gpt-oss-120b-Turbo": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 1.5e-07,
+ "output_cost_per_token": 6e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/MiniMaxAI/MiniMax-M2.7": {
+ "max_tokens": 196608,
+ "max_input_tokens": 196608,
+ "input_cost_per_token": 2.5e-07,
+ "output_cost_per_token": 1e-06,
+ "cache_read_input_token_cost": 5e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.8-27B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 4e-07,
+ "output_cost_per_token": 3e-06,
+ "cache_read_input_token_cost": 4e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemma-4-31B-it-Ultra": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 2.7e-07,
+ "output_cost_per_token": 7.6e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/moonshotai/Kimi-K2.5": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 4.5e-07,
+ "output_cost_per_token": 2.25e-06,
+ "cache_read_input_token_cost": 7e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/zai-org/GLM-4.7-Flash": {
+ "max_tokens": 202752,
+ "max_input_tokens": 202752,
+ "input_cost_per_token": 6e-08,
+ "output_cost_per_token": 4e-07,
+ "cache_read_input_token_cost": 1e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/zai-org/GLM-4.6": {
+ "max_tokens": 202752,
+ "max_input_tokens": 202752,
+ "input_cost_per_token": 5e-07,
+ "output_cost_per_token": 2e-06,
+ "cache_read_input_token_cost": 1e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/anthropic/claude-opus-4-8": {
+ "input_cost_per_token": 5e-06,
+ "litellm_provider": "deepinfra",
+ "max_input_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-05,
+ "prompt_cache_min_tokens": 1024,
+ "source": "https://deepinfra.com/pricing",
+ "supports_adaptive_thinking": true,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "deepinfra/anthropic/claude-sonnet-4-6": {
+ "max_tokens": 1000000,
+ "max_input_tokens": 1000000,
+ "input_cost_per_token": 3e-06,
+ "output_cost_per_token": 1.5e-05,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "prompt_cache_min_tokens": 1024,
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_adaptive_thinking": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemini-3.5-flash": {
+ "max_tokens": 1000000,
+ "max_input_tokens": 1000000,
+ "input_cost_per_token": 1.5e-06,
+ "output_cost_per_token": 9e-06,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/XiaomiMiMo/MiMo-V2.5": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 4e-07,
+ "output_cost_per_token": 2e-06,
+ "cache_read_input_token_cost": 8e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3-Max": {
+ "max_tokens": 256000,
+ "max_input_tokens": 256000,
+ "input_cost_per_token": 1.2e-06,
+ "output_cost_per_token": 6e-06,
+ "cache_read_input_token_cost": 2.4e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemma-4-31B-it-turbo": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 9e-08,
+ "output_cost_per_token": 3.4e-07,
+ "cache_read_input_token_cost": 5e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/thinkingmachines/Inkling-Small": {
+ "max_tokens": 524288,
+ "max_input_tokens": 524288,
+ "input_cost_per_token": 4.5e-07,
+ "output_cost_per_token": 1.2e-06,
+ "cache_read_input_token_cost": 1e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/meta-models/Muse-Glimmer-30B": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 3e-07,
+ "output_cost_per_token": 1.2e-06,
+ "cache_read_input_token_cost": 4e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3-Max-Thinking": {
+ "max_tokens": 256000,
+ "max_input_tokens": 256000,
+ "input_cost_per_token": 1.2e-06,
+ "output_cost_per_token": 6e-06,
+ "cache_read_input_token_cost": 2.4e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3-VL-235B-A22B-Instruct": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 2e-07,
+ "output_cost_per_token": 8.8e-07,
+ "cache_read_input_token_cost": 1.1e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3-VL-30B-A3B-Instruct": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1.5e-07,
+ "output_cost_per_token": 6e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.5-27B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 2.6e-07,
+ "output_cost_per_token": 2.6e-06,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.6-35B-A3B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1e-07,
+ "output_cost_per_token": 9.5e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/nvidia/Nemotron-Content-Safety-3.5": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 2e-07,
+ "output_cost_per_token": 2e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/anthropic/claude-opus-5": {
+ "input_cost_per_token": 5e-06,
+ "litellm_provider": "deepinfra",
+ "max_input_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-05,
+ "prompt_cache_min_tokens": 512,
+ "source": "https://deepinfra.com/pricing",
+ "supports_adaptive_thinking": true,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "deepinfra/thinkingmachines/Inkling": {
+ "max_tokens": 524288,
+ "max_input_tokens": 524288,
+ "input_cost_per_token": 9.5e-07,
+ "output_cost_per_token": 4.05e-06,
+ "cache_read_input_token_cost": 1.6e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/moonshotai/Kimi-K2.6": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 7.5e-07,
+ "output_cost_per_token": 3.5e-06,
+ "cache_read_input_token_cost": 1.5e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/deepseek-ai/DeepSeek-V4-Pro-0813": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 1.3e-06,
+ "output_cost_per_token": 2.6e-06,
+ "cache_read_input_token_cost": 1e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.7-Max": {
+ "max_tokens": 256000,
+ "max_input_tokens": 256000,
+ "input_cost_per_token": 2.5e-06,
+ "output_cost_per_token": 7.5e-06,
+ "cache_read_input_token_cost": 5e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/ByteDance/Seed-2.0-mini": {
+ "max_tokens": 256000,
+ "max_input_tokens": 256000,
+ "input_cost_per_token": 1e-07,
+ "output_cost_per_token": 4e-07,
+ "cache_read_input_token_cost": 2e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.8-2.4T-A95B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 2e-06,
+ "output_cost_per_token": 6e-06,
+ "cache_read_input_token_cost": 2e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/MiniMaxAI/MiniMax-M3": {
+ "max_tokens": 524288,
+ "max_input_tokens": 524288,
+ "input_cost_per_token": 2.8e-07,
+ "output_cost_per_token": 1.1e-06,
+ "cache_read_input_token_cost": 5.6e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemini-3.1-flash-lite": {
+ "max_tokens": 1000000,
+ "max_input_tokens": 1000000,
+ "input_cost_per_token": 2.5e-07,
+ "output_cost_per_token": 1.5e-06,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemini-3.7-flash": {
+ "max_tokens": 1000000,
+ "max_input_tokens": 1000000,
+ "input_cost_per_token": 7.5e-07,
+ "output_cost_per_token": 3.75e-06,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/inclusionAI/Ling-3.0-flash": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 6e-08,
+ "output_cost_per_token": 1.8e-07,
+ "cache_read_input_token_cost": 1.2e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/stepfun-ai/Step-3.7-Flash": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 2e-07,
+ "output_cost_per_token": 1.15e-06,
+ "cache_read_input_token_cost": 4e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.5-35B-A3B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1.4e-07,
+ "output_cost_per_token": 1e-06,
+ "cache_read_input_token_cost": 5e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/ByteDance/Seed-1.8": {
+ "max_tokens": 256000,
+ "max_input_tokens": 256000,
+ "input_cost_per_token": 2.5e-07,
+ "output_cost_per_token": 2e-06,
+ "cache_read_input_token_cost": 5e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/tencent/Hy3": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1.4e-07,
+ "output_cost_per_token": 5.8e-07,
+ "cache_read_input_token_cost": 3.5e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/ByteDance/Seed-2.0-code": {
+ "max_tokens": 256000,
+ "max_input_tokens": 256000,
+ "input_cost_per_token": 5e-07,
+ "output_cost_per_token": 3e-06,
+ "cache_read_input_token_cost": 1e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/ByteDance/Seed-2.0-pro": {
+ "max_tokens": 256000,
+ "max_input_tokens": 256000,
+ "input_cost_per_token": 5e-07,
+ "output_cost_per_token": 3e-06,
+ "cache_read_input_token_cost": 1e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/zai-org/GLM-5": {
+ "max_tokens": 202752,
+ "max_input_tokens": 202752,
+ "input_cost_per_token": 6e-07,
+ "output_cost_per_token": 2.08e-06,
+ "cache_read_input_token_cost": 1.2e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/nvidia/Nemotron-3-Nano-30B-A3B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 5e-08,
+ "output_cost_per_token": 2e-07,
+ "cache_read_input_token_cost": 2.5e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/moonshotai/Kimi-K2.7-Code": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 6.8e-07,
+ "output_cost_per_token": 3.4e-06,
+ "cache_read_input_token_cost": 1.36e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/anthropic/claude-sonnet-5": {
+ "input_cost_per_token": 2e-06,
+ "litellm_provider": "deepinfra",
+ "max_input_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 1e-05,
+ "prompt_cache_min_tokens": 1024,
+ "source": "https://deepinfra.com/pricing",
+ "supports_adaptive_thinking": true,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "deepinfra/Qwen/Qwen3.5-397B-A17B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 4.5e-07,
+ "output_cost_per_token": 3e-06,
+ "cache_read_input_token_cost": 2.2e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 8e-08,
+ "output_cost_per_token": 1.8e-07,
+ "cache_read_input_token_cost": 1.6e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemma-4-E4B-it": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 2e-08,
+ "output_cost_per_token": 1e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/deepseek-ai/DeepSeek-V3.2": {
+ "max_tokens": 163840,
+ "max_input_tokens": 163840,
+ "input_cost_per_token": 2.6e-07,
+ "output_cost_per_token": 3.8e-07,
+ "cache_read_input_token_cost": 1.3e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.8-Max": {
+ "max_tokens": 256000,
+ "max_input_tokens": 256000,
+ "input_cost_per_token": 1.65e-06,
+ "output_cost_per_token": 4.951e-06,
+ "cache_read_input_token_cost": 2.06e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/anthropic/claude-fable-5": {
+ "input_cost_per_token": 1e-05,
+ "litellm_provider": "deepinfra",
+ "max_input_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 5e-05,
+ "prompt_cache_min_tokens": 512,
+ "source": "https://deepinfra.com/pricing",
+ "supports_adaptive_thinking": true,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "thinking_always_on": true
+ },
+ "deepinfra/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 5e-07,
+ "output_cost_per_token": 2.2e-06,
+ "cache_read_input_token_cost": 1e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.5-122B-A10B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 2.9e-07,
+ "output_cost_per_token": 2.4e-06,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/zai-org/GLM-5.1": {
+ "max_tokens": 202752,
+ "max_input_tokens": 202752,
+ "input_cost_per_token": 1.05e-06,
+ "output_cost_per_token": 3.5e-06,
+ "cache_read_input_token_cost": 2.05e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/deepseek-ai/DeepSeek-V4-Pro": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 1.3e-06,
+ "output_cost_per_token": 2.6e-06,
+ "cache_read_input_token_cost": 1e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 8.5e-08,
+ "output_cost_per_token": 4e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/zai-org/GLM-5.2": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 7.5e-07,
+ "output_cost_per_token": 2.4e-06,
+ "cache_read_input_token_cost": 1.4e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/moonshotai/Kimi-K3": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 2.85e-06,
+ "output_cost_per_token": 1.425e-05,
+ "cache_read_input_token_cost": 2.85e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/anthropic/claude-opus-4-7": {
+ "input_cost_per_token": 5e-06,
+ "litellm_provider": "deepinfra",
+ "max_input_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-05,
+ "prompt_cache_min_tokens": 2048,
+ "source": "https://deepinfra.com/pricing",
+ "supports_adaptive_thinking": true,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true
+ },
+ "deepinfra/Qwen/Qwen3.6-27B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 3.2e-07,
+ "output_cost_per_token": 3.2e-06,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemma-4-26B-A4B-it": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 7e-08,
+ "output_cost_per_token": 3.4e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemini-3.1-pro": {
+ "max_tokens": 1000000,
+ "max_input_tokens": 1000000,
+ "input_cost_per_token": 2e-06,
+ "output_cost_per_token": 1.2e-05,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/XiaomiMiMo/MiMo-V2.5-Pro": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 1e-06,
+ "output_cost_per_token": 3e-06,
+ "cache_read_input_token_cost": 2e-07,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/anthropic/claude-haiku-4-5": {
+ "max_tokens": 200000,
+ "max_input_tokens": 200000,
+ "input_cost_per_token": 1e-06,
+ "output_cost_per_token": 5e-06,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "prompt_cache_min_tokens": 4096,
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/deepseek-ai/DeepSeek-V4-Flash": {
+ "max_tokens": 1048576,
+ "max_input_tokens": 1048576,
+ "input_cost_per_token": 9e-08,
+ "output_cost_per_token": 1.8e-07,
+ "cache_read_input_token_cost": 1.8e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/openai/gpt-oss-120b-Ultra": {
+ "max_tokens": 131072,
+ "max_input_tokens": 131072,
+ "input_cost_per_token": 2e-07,
+ "output_cost_per_token": 9.5e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/Qwen/Qwen3.5-9B": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1e-07,
+ "output_cost_per_token": 1.5e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/MiniMaxAI/MiniMax-M2.7-Turbo": {
+ "max_tokens": 196608,
+ "max_input_tokens": 196608,
+ "input_cost_per_token": 3.8e-07,
+ "output_cost_per_token": 1.7e-06,
+ "cache_read_input_token_cost": 7e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/zai-org/GLM-4.7": {
+ "max_tokens": 202752,
+ "max_input_tokens": 202752,
+ "input_cost_per_token": 4e-07,
+ "output_cost_per_token": 1.75e-06,
+ "cache_read_input_token_cost": 8e-08,
+ "supports_prompt_caching": true,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": false,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "deepinfra/google/gemma-4-31B-it": {
+ "max_tokens": 262144,
+ "max_input_tokens": 262144,
+ "input_cost_per_token": 1.3e-07,
+ "output_cost_per_token": 3.8e-07,
+ "litellm_provider": "deepinfra",
+ "mode": "chat",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_response_schema": true,
+ "supports_reasoning": true,
+ "supports_vision": true,
+ "source": "https://deepinfra.com/pricing"
+ },
+ "gemini/gemini-omni-1.1-flash": {
+ "input_cost_per_audio_token": 1.5e-06,
+ "input_cost_per_token": 1.5e-06,
+ "litellm_provider": "gemini",
+ "mode": "chat",
+ "output_cost_per_reasoning_token": 9e-06,
+ "output_cost_per_token": 9e-06,
+ "output_cost_per_video_token": 1.75e-05,
+ "rpm": 2000,
+ "source": "https://ai.google.dev/gemini-api/docs/pricing",
+ "supported_endpoints": [
+ "/v1/chat/completions"
+ ],
+ "supported_modalities": [
+ "text",
+ "image",
+ "audio",
+ "video"
+ ],
+ "supported_output_modalities": [
+ "text",
+ "video"
+ ],
+ "supports_audio_input": true,
+ "supports_reasoning": true,
+ "supports_system_messages": true,
+ "supports_video_input": true,
+ "supports_vision": true,
+ "tpm": 800000
+ },
+ "xai/grok-4.20": {
+ "cache_read_input_token_cost": 2e-07,
+ "input_cost_per_token": 1.25e-06,
+ "litellm_provider": "xai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_web_search": true,
+ "input_cost_per_token_above_200k_tokens": 2.5e-06,
+ "output_cost_per_token_above_200k_tokens": 5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+ "supports_prompt_caching": true,
+ "supports_response_schema": true
+ },
+ "xai/grok-4.20-reasoning": {
+ "cache_read_input_token_cost": 2e-07,
+ "input_cost_per_token": 1.25e-06,
+ "litellm_provider": "xai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_web_search": true,
+ "input_cost_per_token_above_200k_tokens": 2.5e-06,
+ "output_cost_per_token_above_200k_tokens": 5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+ "supports_prompt_caching": true,
+ "supports_response_schema": true
+ },
+ "xai/grok-4.20-reasoning-latest": {
+ "cache_read_input_token_cost": 2e-07,
+ "input_cost_per_token": 1.25e-06,
+ "litellm_provider": "xai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_web_search": true,
+ "input_cost_per_token_above_200k_tokens": 2.5e-06,
+ "output_cost_per_token_above_200k_tokens": 5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+ "supports_prompt_caching": true,
+ "supports_response_schema": true
+ },
+ "xai/grok-imagine-image": {
+ "input_cost_per_image": 0.002,
+ "litellm_provider": "xai",
+ "mode": "image_generation",
+ "output_cost_per_image": 0.02,
+ "source": "https://docs.x.ai/docs/models",
+ "supported_endpoints": [
+ "/v1/images/generations"
+ ],
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "image"
+ ]
+ },
+ "xai/grok-imagine-image-2026-03-02": {
+ "input_cost_per_image": 0.002,
+ "litellm_provider": "xai",
+ "mode": "image_generation",
+ "output_cost_per_image": 0.02,
+ "source": "https://docs.x.ai/docs/models",
+ "supported_endpoints": [
+ "/v1/images/generations"
+ ],
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "image"
+ ]
+ },
+ "xai/grok-imagine-image-quality": {
+ "input_cost_per_image": 0.01,
+ "litellm_provider": "xai",
+ "mode": "image_generation",
+ "output_cost_per_image": 0.05,
+ "source": "https://docs.x.ai/docs/models",
+ "supported_endpoints": [
+ "/v1/images/generations"
+ ],
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "image"
+ ]
+ },
+ "xai/grok-imagine-image-quality-20260403": {
+ "input_cost_per_image": 0.01,
+ "litellm_provider": "xai",
+ "mode": "image_generation",
+ "output_cost_per_image": 0.05,
+ "source": "https://docs.x.ai/docs/models",
+ "supported_endpoints": [
+ "/v1/images/generations"
+ ],
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "image"
+ ]
+ },
+ "xai/grok-imagine-image-quality-latest": {
+ "input_cost_per_image": 0.01,
+ "litellm_provider": "xai",
+ "mode": "image_generation",
+ "output_cost_per_image": 0.05,
+ "source": "https://docs.x.ai/docs/models",
+ "supported_endpoints": [
+ "/v1/images/generations"
+ ],
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "image"
+ ]
+ },
+ "xai/grok-imagine-image-pro": {
+ "input_cost_per_image": 0.01,
+ "litellm_provider": "xai",
+ "mode": "image_generation",
+ "output_cost_per_image": 0.05,
+ "source": "https://docs.x.ai/docs/models",
+ "supported_endpoints": [
+ "/v1/images/generations"
+ ],
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "image"
+ ]
+ },
+ "xai/grok-imagine-image-2.0": {
+ "input_cost_per_image": 0.01,
+ "litellm_provider": "xai",
+ "mode": "image_generation",
+ "output_cost_per_image": 0.06,
+ "source": "https://docs.x.ai/docs/models",
+ "supported_endpoints": [
+ "/v1/images/generations"
+ ],
+ "supported_modalities": [
+ "text",
+ "image"
+ ],
+ "supported_output_modalities": [
+ "image"
+ ]
+ },
+ "xai/grok-4.20-non-reasoning": {
+ "cache_read_input_token_cost": 2e-07,
+ "input_cost_per_token": 1.25e-06,
+ "litellm_provider": "xai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_web_search": true,
+ "input_cost_per_token_above_200k_tokens": 2.5e-06,
+ "output_cost_per_token_above_200k_tokens": 5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+ "supports_response_schema": true
+ },
+ "xai/grok-4.20-non-reasoning-latest": {
+ "cache_read_input_token_cost": 2e-07,
+ "input_cost_per_token": 1.25e-06,
+ "litellm_provider": "xai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_web_search": true,
+ "input_cost_per_token_above_200k_tokens": 2.5e-06,
+ "output_cost_per_token_above_200k_tokens": 5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+ "supports_response_schema": true
+ },
+ "xai/grok-4.20-multi-agent": {
+ "cache_read_input_token_cost": 2e-07,
+ "input_cost_per_token": 1.25e-06,
+ "litellm_provider": "xai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_web_search": true,
+ "input_cost_per_token_above_200k_tokens": 2.5e-06,
+ "output_cost_per_token_above_200k_tokens": 5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+ "supports_response_schema": true
+ },
+ "xai/grok-4.20-multi-agent-latest": {
+ "cache_read_input_token_cost": 2e-07,
+ "input_cost_per_token": 1.25e-06,
+ "litellm_provider": "xai",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
+ "mode": "chat",
+ "output_cost_per_token": 2.5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_web_search": true,
+ "input_cost_per_token_above_200k_tokens": 2.5e-06,
+ "output_cost_per_token_above_200k_tokens": 5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+ "supports_response_schema": true
}
}
diff --git a/pyproject.toml b/pyproject.toml
index 1c3f5a4875c..eba9e5afc98 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -67,8 +67,8 @@ proxy = [
"azure-identity>=1.25.2,<2.0",
"azure-storage-blob>=12.28.0,<13.0",
"mcp>=1.28.1,<2.0",
- "litellm-proxy-extras==0.4.89",
- "litellm-enterprise==0.1.60",
+ "litellm-proxy-extras==0.4.90",
+ "litellm-enterprise==0.1.61",
"RestrictedPython>=8.1,<9.0",
"rich>=13.9.4,<14.0",
"InquirerPy>=0.3.4,<1.0",
diff --git a/tests/code_coverage_tests/check_licenses.py b/tests/code_coverage_tests/check_licenses.py
index 389e534b1ff..158e25180e1 100644
--- a/tests/code_coverage_tests/check_licenses.py
+++ b/tests/code_coverage_tests/check_licenses.py
@@ -5,8 +5,9 @@ import json
from pathlib import Path
import re
import sys
+import time
import tomllib
-from typing import Dict, List, Optional, Set, Tuple
+from typing import Callable, Dict, Final, List, Optional, Protocol, Set, Tuple
from packaging.requirements import Requirement
import requests
@@ -37,6 +38,13 @@ DEFAULT_TRANSITIVE_PIN_PACKAGES = (
# of the identifier, not an operator.
_SPDX_OPERATOR_SPLIT = re.compile(r"\s+(?:OR|AND)\s+")
_SPDX_WITH_SUFFIX = re.compile(r"\s+WITH\s+.*", re.DOTALL)
+_PYPI_FETCH_ATTEMPTS: Final[int] = 3
+_PYPI_FETCH_BACKOFF_SECONDS: Final[float] = 0.5
+
+
+class _HttpGet(Protocol):
+ def __call__(self, url: str, *, timeout: float) -> requests.Response:
+ ...
@dataclass
@@ -50,7 +58,10 @@ class PackageLicense:
class LicenseChecker:
def __init__(
- self, config_file: Path = Path("./tests/code_coverage_tests/liccheck.ini")
+ self,
+ config_file: Path = Path("./tests/code_coverage_tests/liccheck.ini"),
+ http_get: Optional[_HttpGet] = None,
+ sleep: Optional[Callable[[float], None]] = None,
):
if not config_file.exists():
print(f"Error: Config file {config_file} not found")
@@ -79,6 +90,8 @@ class LicenseChecker:
# Track package results
self.package_results: List[PackageLicense] = []
+ self._http_get = http_get
+ self._sleep = sleep
@staticmethod
def _normalize_package_name(package_name: str) -> str:
@@ -123,21 +136,38 @@ class LicenseChecker:
last resort derives the license from the ``License :: OSI Approved ::
...`` trove classifiers.
"""
- try:
- url = f"https://pypi.org/pypi/{package_name}/{version}/json"
- response = requests.get(url, timeout=10)
- response.raise_for_status()
- info = response.json().get("info", {}) or {}
- return (
- info.get("license_expression")
- or info.get("license")
- or self._license_from_classifiers(info.get("classifiers") or [])
- )
- except Exception as e:
- print(
- f"Warning: Failed to fetch license for {package_name} {version}: {str(e)}"
- )
- return None
+ url = f"https://pypi.org/pypi/{package_name}/{version}/json"
+ http_get = self._http_get if self._http_get is not None else requests.get
+ sleep = self._sleep if self._sleep is not None else time.sleep
+
+ for attempt in range(_PYPI_FETCH_ATTEMPTS):
+ try:
+ response = http_get(url, timeout=10)
+ response.raise_for_status()
+ info = response.json().get("info", {}) or {}
+ return (
+ info.get("license_expression")
+ or info.get("license")
+ or self._license_from_classifiers(info.get("classifiers") or [])
+ )
+ except Exception as error:
+ if self._is_retryable_pypi_error(error) and attempt < _PYPI_FETCH_ATTEMPTS - 1:
+ sleep(_PYPI_FETCH_BACKOFF_SECONDS)
+ continue
+ print(
+ f"Warning: Failed to fetch license for {package_name} {version}: {str(error)}"
+ )
+ return None
+ return None
+
+ @staticmethod
+ def _is_retryable_pypi_error(error: Exception) -> bool:
+ if isinstance(error, (requests.ConnectionError, requests.Timeout)):
+ return True
+ if not isinstance(error, requests.HTTPError) or error.response is None:
+ return False
+ status_code = error.response.status_code
+ return status_code == 429 or status_code >= 500
@staticmethod
def _license_from_classifiers(classifiers: List[str]) -> Optional[str]:
diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py
index b2bd41e19ba..387280c8023 100644
--- a/tests/e2e/management/management_client.py
+++ b/tests/e2e/management/management_client.py
@@ -9,8 +9,21 @@ from __future__ import annotations
import time
from dataclasses import dataclass
+import jwt
+
+from e2e_config import MASTER_KEY
from proxy_client import ProxyClient
-from e2e_http import NoBody, ProbeResult, Result, StreamingResponse, Success, UnknownApiError, unwrap
+from e2e_http import (
+ AuthHeaders,
+ NetworkError,
+ NoBody,
+ ProbeResult,
+ Result,
+ StreamingResponse,
+ Success,
+ UnknownApiError,
+ unwrap,
+)
from models import (
ChatBody,
ChatMessage,
@@ -50,6 +63,9 @@ from models import (
TeamNewBody,
TeamNewResponse,
TeamUpdateBody,
+ UiLoginBody,
+ UiLoginResponse,
+ UiSessionClaims,
UserDeleteBody,
UserDeleteResponse,
UserInfoParams,
@@ -63,38 +79,73 @@ from models import (
MODEL_ACCESS_DENIED_MARKER = "key_model_access_denied"
ROUTE_NOT_ALLOWED_MARKER = "not allowed to call this route"
+DASHBOARD_SESSION_TEAM_ID = "litellm-dashboard"
_TEAM_READY_ATTEMPTS = 15
_TEAM_READY_SLEEP_SECONDS = 0.4
+_KEY_WRITE_ATTEMPTS = 5
+_TRANSIENT_BACKEND_MARKERS = ("connecting to redis", "name resolution")
+
+
+@dataclass(frozen=True, slots=True)
+class DashboardSession:
+ """What a dashboard sign-in hands the Admin UI: the session key it sends as
+ its bearer on every subsequent call, the claims it renders the signed-in user
+ from, and where it lands the browser."""
+
+ session_key: str
+ claims: UiSessionClaims
+ redirect_url: str
@dataclass(frozen=True, slots=True)
class ManagementClient:
proxy: ProxyClient
+ master_key: str
def llm_only_key(self) -> str:
return self.proxy.generate_key(KeyGenerateBody(models=[], allowed_routes=["llm_api_routes"]))
- def update_key_models(self, key: str, models: list[str]) -> None:
- last: Result[NoBody] | None = None
- for attempt in range(5):
+ def generate_key(self, body: KeyGenerateBody, *, caller_key: str | None = None) -> Result[KeyGenerateResponse]:
+ """POST /key/generate. `caller_key` is who is creating the key: the master
+ key by default, or a virtual key (an admin filling in Create New Key on the
+ dashboard creates it under the session key their sign-in minted). Returns
+ the outcome rather than unwrapping it, so a caller can poll a route that is
+ only transiently refusing."""
+ headers = self.proxy.transport.master if caller_key is None else self.proxy.transport.bearer(caller_key)
+ return self.proxy.transport.post(
+ "/key/generate",
+ headers=headers,
+ json=body,
+ response_type=KeyGenerateResponse,
+ )
+
+ def update_key(self, body: KeyUpdateBody, *, caller_key: str | None = None) -> Result[NoBody]:
+ """POST /key/update. `caller_key` is who is editing: the master key by
+ default, or a virtual key (the dashboard edits under the session key its
+ sign-in minted, never the master key). Returns the outcome rather than
+ unwrapping it, so a caller can poll a route that is only transiently
+ refusing; `update_key_models` is the unwrapping shorthand."""
+ headers = self.proxy.transport.master if caller_key is None else self.proxy.transport.bearer(caller_key)
+ last: Result[NoBody] = NetworkError(message="/key/update was never attempted")
+ for attempt in range(_KEY_WRITE_ATTEMPTS):
last = self.proxy.transport.post(
"/key/update",
- headers=self.proxy.transport.master,
- json=KeyUpdateBody(key=key, models=models),
+ headers=headers,
+ json=body,
response_type=NoBody,
)
match last:
- case Success():
- return
- case UnknownApiError(body=body) if (
- "connecting to redis" in body.lower() or "name resolution" in body.lower()
+ case UnknownApiError(body=error_body) if any(
+ marker in error_body.lower() for marker in _TRANSIENT_BACKEND_MARKERS
):
time.sleep(0.5 * (attempt + 1))
continue
case _:
break
- assert last is not None
- raise AssertionError(last)
+ return last
+
+ def update_key_models(self, key: str, models: list[str]) -> None:
+ _ = unwrap(self.update_key(KeyUpdateBody(key=key, models=models)))
def delete_key_strict(self, key: str) -> None:
"""Strict delete for the act phase of a test: a failed delete is a hard
@@ -150,15 +201,42 @@ class ManagementClient:
)
).key
+ def key_list(self, key_alias: str, *, caller_key: str | None = None) -> Result[KeyListResponse]:
+ """GET /key/list, the Virtual Keys page's own inventory call. `caller_key` is
+ who is asking: the master key by default, or a virtual key."""
+ headers = self.proxy.transport.master if caller_key is None else self.proxy.transport.bearer(caller_key)
+ return self.proxy.transport.get(
+ "/key/list",
+ headers=headers,
+ params=KeyListParams(key_alias=key_alias),
+ response_type=KeyListResponse,
+ )
+
def key_alias_count(self, key_alias: str) -> int:
- return unwrap(
- self.proxy.transport.get(
- "/key/list",
- headers=self.proxy.transport.master,
- params=KeyListParams(key_alias=key_alias),
- response_type=KeyListResponse,
+ return unwrap(self.key_list(key_alias)).total_count
+
+ def dashboard_login(self, username: str, password: str) -> DashboardSession:
+ """POST /v2/login, the call the Admin UI's sign-in form makes.
+
+ The proxy authenticates the credentials, mints a UI session key for the
+ signed-in user, and hands it back inside a JWT signed with the master key.
+ Decoding that JWT is the only way to reach the session key, and it is what
+ the dashboard itself does before it can call a single management route."""
+ response = unwrap(
+ self.proxy.transport.post(
+ "/v2/login",
+ headers=AuthHeaders(),
+ json=UiLoginBody(username=username, password=password),
+ response_type=UiLoginResponse,
)
- ).total_count
+ )
+ decoded: object = jwt.decode(response.token, self.master_key, algorithms=["HS256"])
+ claims = UiSessionClaims.model_validate(decoded)
+ return DashboardSession(
+ session_key=claims.key,
+ claims=claims,
+ redirect_url=response.redirect_url,
+ )
def create_team(self, body: TeamNewBody) -> str:
team_id = unwrap(
@@ -465,4 +543,4 @@ class ManagementClient:
def build_client(proxy: ProxyClient) -> ManagementClient:
- return ManagementClient(proxy=proxy)
+ return ManagementClient(proxy=proxy, master_key=MASTER_KEY)
diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py
index 9b398963ac9..a56eb853823 100644
--- a/tests/e2e/management/test_management_e2e.py
+++ b/tests/e2e/management/test_management_e2e.py
@@ -15,15 +15,30 @@ from collections.abc import Callable
import pytest
-from e2e_config import unique_marker
-from e2e_http import StreamingResponse
+from e2e_config import UI_PASSWORD, UI_USERNAME, unique_marker
+from e2e_http import StreamingResponse, Success
from lifecycle import ResourceManager
from management_client import (
+ DASHBOARD_SESSION_TEAM_ID,
MODEL_ACCESS_DENIED_MARKER,
ROUTE_NOT_ALLOWED_MARKER,
ManagementClient,
)
-from models import KeyGenerateBody, OrgInfoResponse, OrgNewBody, OrgUpdateBody, TagListEntry, TagNewBody, TeamNewBody, TeamUpdateBody, UserNewBody, UserUpdateBody, LiteLLMParamsBody, ModelInfoEntry
+from models import (
+ KeyGenerateBody,
+ KeyUpdateBody,
+ LiteLLMParamsBody,
+ ModelInfoEntry,
+ OrgInfoResponse,
+ OrgNewBody,
+ OrgUpdateBody,
+ TagListEntry,
+ TagNewBody,
+ TeamNewBody,
+ TeamUpdateBody,
+ UserNewBody,
+ UserUpdateBody,
+)
pytestmark = pytest.mark.e2e
@@ -199,6 +214,132 @@ class TestKeyRoutes:
return True if client.proxy.key_info(key).blocked else None
_ = _poll(client, blocked, "/key/info never reported the key blocked after /key/block before the deadline")
+
+
+class TestDashboardKeyRoutes:
+ """The /key writes as the Admin UI makes them. Signing in mints the session key
+ the dashboard authenticates with, and every key an admin creates or edits in the
+ browser is written under that session key rather than the master key, so these
+ are the same routes the API-surface tests cover with a different caller."""
+
+ @pytest.mark.covers("mgmt.key.generate.happy_path")
+ def test_creating_a_key_from_the_dashboard_persists_and_works(
+ self, client: ManagementClient, resources: ResourceManager
+ ) -> None:
+ session = client.dashboard_login(UI_USERNAME, UI_PASSWORD)
+ resources.defer(lambda: client.proxy.delete_key(session.session_key))
+
+ assert session.claims.login_method == "username_password", (
+ f"/v2/login reports login_method {session.claims.login_method!r} for a username/password sign-in"
+ )
+ assert session.claims.user_role == "proxy_admin", (
+ f"/v2/login reports user_role {session.claims.user_role!r} for the admin credentials, "
+ "expected 'proxy_admin'"
+ )
+ assert session.redirect_url.endswith("/ui?login=success"), (
+ f"/v2/login sends the browser to {session.redirect_url!r} instead of the dashboard"
+ )
+
+ session_info = client.proxy.key_info(session.session_key)
+ assert session_info.team_id == DASHBOARD_SESSION_TEAM_ID, (
+ f"the minted session key reports team_id {session_info.team_id!r}, expected the dashboard's "
+ f"{DASHBOARD_SESSION_TEAM_ID!r}"
+ )
+
+ alias = f"e2e-mgmt-uicreate-{unique_marker()}"
+
+ def dashboard_creates_the_key() -> str | None:
+ match client.generate_key(
+ KeyGenerateBody(models=["gemini-2.5-flash"], key_alias=alias, tpm_limit=100),
+ caller_key=session.session_key,
+ ):
+ case Success(data=created):
+ return created.key
+ case _:
+ return None
+
+ created = _poll(
+ client,
+ dashboard_creates_the_key,
+ "the dashboard session key was never accepted on /key/generate before the deadline",
+ )
+ resources.defer(lambda: client.proxy.delete_key(created))
+
+ created_info = client.proxy.key_info(created)
+ assert created_info.key_alias == alias, (
+ f"/key/info reports key_alias {created_info.key_alias!r} for the key the dashboard created, "
+ f"expected {alias!r}"
+ )
+ assert created_info.models == ["gemini-2.5-flash"], (
+ f"/key/info reports models {created_info.models} for the key the dashboard created"
+ )
+ assert created_info.tpm_limit == 100, (
+ f"/key/info reports tpm_limit {created_info.tpm_limit} for the key the dashboard created, expected 100"
+ )
+
+ def dashboard_lists_the_key() -> bool | None:
+ match client.key_list(alias, caller_key=session.session_key):
+ case Success(data=listing) if listing.total_count == 1:
+ return True
+ case _:
+ return None
+
+ _ = _poll(
+ client,
+ dashboard_lists_the_key,
+ f"the session key never saw {alias!r} in /key/list before the deadline, so the dashboard "
+ "would render no keys",
+ )
+
+ _poll_chat_ok(client, created, "gemini-2.5-flash")
+ _assert_model_denied(client.chat_status(created, "gpt-5.5", f"say hi {unique_marker()}"), "gpt-5.5")
+
+ @pytest.mark.covers("mgmt.key.update.happy_path")
+ def test_editing_a_key_from_the_dashboard_persists_and_is_enforced(
+ self, client: ManagementClient, resources: ResourceManager
+ ) -> None:
+ alias = f"e2e-mgmt-uiedit-{unique_marker()}"
+ target = _generate_key(
+ client,
+ resources,
+ KeyGenerateBody(models=["gemini-2.5-flash"], key_alias=alias, tpm_limit=100, rpm_limit=200),
+ )
+ _poll_chat_ok(client, target, "gemini-2.5-flash")
+ _assert_model_denied(client.chat_status(target, "gpt-5.5", f"say hi {unique_marker()}"), "gpt-5.5")
+
+ session = client.dashboard_login(UI_USERNAME, UI_PASSWORD)
+ resources.defer(lambda: client.proxy.delete_key(session.session_key))
+
+ def dashboard_saves_the_edit() -> bool | None:
+ match client.update_key(
+ KeyUpdateBody(key=target, models=["gpt-5.5"], tpm_limit=300, rpm_limit=400),
+ caller_key=session.session_key,
+ ):
+ case Success():
+ return True
+ case _:
+ return None
+
+ _ = _poll(
+ client,
+ dashboard_saves_the_edit,
+ "the dashboard session key was never accepted on /key/update before the deadline",
+ )
+
+ info = client.proxy.key_info(target)
+ assert info.models == ["gpt-5.5"], (
+ f"/key/info reports models {info.models} after the dashboard edit to ['gpt-5.5']"
+ )
+ assert info.tpm_limit == 300, f"/key/info reports tpm_limit {info.tpm_limit} after the dashboard edit to 300"
+ assert info.rpm_limit == 400, f"/key/info reports rpm_limit {info.rpm_limit} after the dashboard edit to 400"
+ assert info.key_alias == alias, (
+ f"the dashboard edit renamed the key to {info.key_alias!r}, it should still be {alias!r}"
+ )
+
+ _poll_model_access_granted(client, target, "gpt-5.5")
+ _poll_chat_denied(client, target, "gemini-2.5-flash")
+
+
class TestKeyRegeneration:
@pytest.mark.covers("mgmt.key.regenerate.happy_path")
def test_regenerate_rotates_to_a_working_new_key(
diff --git a/tests/e2e/models.py b/tests/e2e/models.py
index 8d1b17ca256..7c4c07a678e 100644
--- a/tests/e2e/models.py
+++ b/tests/e2e/models.py
@@ -893,7 +893,10 @@ class CredentialCreateResponse(BaseModel):
class KeyUpdateBody(BaseModel):
key: str
- models: list[str]
+ models: list[str] | None = None
+ key_alias: str | None = None
+ tpm_limit: int | None = None
+ rpm_limit: int | None = None
class KeyBlockBody(BaseModel):
@@ -908,6 +911,27 @@ class KeyListResponse(BaseModel):
total_count: int
+# ---------- admin UI session ----------
+
+
+class UiLoginBody(BaseModel):
+ username: str
+ password: str
+
+
+class UiLoginResponse(BaseModel):
+ token: str
+ redirect_url: str
+
+
+class UiSessionClaims(BaseModel):
+ user_id: str
+ key: str
+ user_role: str
+ login_method: Literal["sso", "username_password"]
+ exp: int
+
+
class TeamMemberEntry(BaseModel):
role: Literal["admin", "user"]
user_id: str
diff --git a/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts b/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts
new file mode 100644
index 00000000000..98bd1b84f11
--- /dev/null
+++ b/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts
@@ -0,0 +1,59 @@
+import { expect, test, type Page as PlaywrightPage } from "@playwright/test";
+import { ADMIN_STORAGE_PATH } from "../../constants";
+import { navigateToPage } from "../../helpers/navigation";
+import { Page } from "../../fixtures/pages";
+
+/**
+ * Opens Add Auto Router and returns the Template select's trigger, which is the
+ * shallowest real page that renders SelectContent with tall multi-line options.
+ */
+async function openTemplateSelect(page: PlaywrightPage) {
+ await navigateToPage(page, Page.Models);
+ await page.getByRole("tab", { name: "Auto-Routers" }).click();
+ await page.getByRole("button", { name: "Add Auto Router" }).click();
+
+ const trigger = page.getByTestId("template-selector");
+ await expect(trigger).toBeVisible();
+ return trigger;
+}
+
+test.describe("Auto Router template select anchoring", () => {
+ test.use({ storageState: ADMIN_STORAGE_PATH });
+
+ test("opens the options below the trigger rather than over it", async ({ page }) => {
+ await page.setViewportSize({ width: 1280, height: 900 });
+ const trigger = await openTemplateSelect(page);
+ const triggerBox = await trigger.boundingBox();
+
+ await trigger.click();
+ const popup = page.locator('[data-slot="select-content"]');
+ await expect(popup).toBeVisible();
+ const popupBox = await popup.boundingBox();
+
+ expect(triggerBox).not.toBeNull();
+ expect(popupBox).not.toBeNull();
+
+ // Item-aligned mode reports "none" and puts the active item over the trigger.
+ await expect(popup).toHaveAttribute("data-side", "bottom");
+ expect(popupBox!.y).toBeGreaterThanOrEqual(triggerBox!.y + triggerBox!.height);
+ });
+
+ test("flips above the trigger instead of covering it when there is no room below", async ({ page }) => {
+ await page.setViewportSize({ width: 1280, height: 560 });
+ const trigger = await openTemplateSelect(page);
+ await trigger.scrollIntoViewIfNeeded();
+ const triggerBox = await trigger.boundingBox();
+
+ await trigger.click();
+ const popup = page.locator('[data-slot="select-content"]');
+ await expect(popup).toBeVisible();
+ const popupBox = await popup.boundingBox();
+
+ expect(triggerBox).not.toBeNull();
+ expect(popupBox).not.toBeNull();
+
+ const overlaps =
+ popupBox!.y < triggerBox!.y + triggerBox!.height && popupBox!.y + popupBox!.height > triggerBox!.y;
+ expect(overlaps).toBe(false);
+ });
+});
diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py
index 09f3e0ba34f..498d0cb4723 100644
--- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py
+++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py
@@ -12,7 +12,11 @@ sys.path.insert(
),
)
-from litellm_proxy_extras.utils import ProxyExtrasDBManager
+from litellm_proxy_extras.utils import (
+ PARTITIONED_SPEND_LOGS_PUSH_ERROR,
+ ProxyExtrasDBManager,
+ filter_partitioned_spend_logs_diff,
+)
# Path to the migrations directory
_MIGRATIONS_DIR = os.path.abspath(
@@ -475,3 +479,205 @@ class TestMigrationGuardScope:
if not self._run_rules([(TestMigrationGuardScope._NEW, by_name[name])])
]
assert not redundant, f"these no longer violate and should be removed: {redundant}"
+
+
+_PARTITIONED_DRIFT_SQL = """-- AlterTable
+ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN "updated_by" TEXT;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_SpendLogs" DROP CONSTRAINT "LiteLLM_SpendLogs_pkey",
+ADD COLUMN "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ADD COLUMN "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ADD CONSTRAINT "LiteLLM_SpendLogs_pkey" PRIMARY KEY ("request_id");
+
+-- DropTable
+DROP TABLE "LiteLLM_SpendLogs_legacy";
+"""
+
+
+class TestPartitionedSpendLogsDriftFilter:
+ """A doc-partitioned LiteLLM_SpendLogs (db_scripts/partition_spend_logs.sql) has a
+ composite primary key that schema.prisma cannot express, so `prisma migrate diff`
+ emits a primary-key rewrite that Postgres rejects, aborting the whole drift script
+ before its legitimate statements run."""
+
+ def test_pk_rewrite_and_runbook_artifact_drops_are_removed(self):
+ filtered = filter_partitioned_spend_logs_diff(_PARTITIONED_DRIFT_SQL)
+ assert 'DROP CONSTRAINT "LiteLLM_SpendLogs_pkey"' not in filtered
+ assert 'PRIMARY KEY ("request_id")' not in filtered
+ assert "LiteLLM_SpendLogs_legacy" not in filtered
+
+ def test_legitimate_statements_in_the_same_script_are_kept(self):
+ filtered = filter_partitioned_spend_logs_diff(_PARTITIONED_DRIFT_SQL)
+ assert 'ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN "updated_by" TEXT;' in filtered
+ assert 'ADD COLUMN "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP' in filtered
+ assert 'ADD COLUMN "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP' in filtered
+ assert filtered.count('ALTER TABLE "LiteLLM_SpendLogs"') == 1
+
+ def test_an_alter_containing_only_the_pk_rewrite_is_dropped_entirely(self):
+ sql = (
+ 'ALTER TABLE "LiteLLM_SpendLogs" DROP CONSTRAINT "LiteLLM_SpendLogs_pkey",\n'
+ 'ADD CONSTRAINT "LiteLLM_SpendLogs_pkey" PRIMARY KEY ("request_id");\n'
+ )
+ assert filter_partitioned_spend_logs_diff(sql).strip() == ""
+
+ def test_other_tables_pk_changes_are_untouched(self):
+ sql = (
+ 'ALTER TABLE "LiteLLM_TeamTable" DROP CONSTRAINT "LiteLLM_TeamTable_pkey",\n'
+ 'ADD CONSTRAINT "LiteLLM_TeamTable_pkey" PRIMARY KEY ("team_id");\n'
+ )
+ filtered = filter_partitioned_spend_logs_diff(sql)
+ assert 'DROP CONSTRAINT "LiteLLM_TeamTable_pkey"' in filtered
+ assert 'PRIMARY KEY ("team_id")' in filtered
+
+
+class _FakeCompleted:
+ stdout = ""
+ stderr = ""
+
+
+class TestResolveAllMigrationsLedger:
+ def _run(self, monkeypatch, tmp_path, partitioned, execute_fails):
+ import subprocess as subprocess_module
+
+ import litellm_proxy_extras.utils as utils_module
+
+ monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:5432/db")
+ monkeypatch.delenv("DIRECT_URL", raising=False)
+ monkeypatch.setattr(
+ ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: partitioned)
+ )
+ monkeypatch.setattr(
+ ProxyExtrasDBManager,
+ "_get_migration_names",
+ staticmethod(lambda migrations_dir: ["20250326162113_baseline"]),
+ )
+ calls = []
+
+ def fake_run(cmd, **kwargs):
+ calls.append(cmd)
+ if "diff" in cmd:
+ kwargs["stdout"].write(_PARTITIONED_DRIFT_SQL)
+ return _FakeCompleted()
+ if "execute" in cmd:
+ executed_sql = open(cmd[cmd.index("--file") + 1]).read()
+ calls.append(("executed_sql", executed_sql))
+ if execute_fails:
+ raise subprocess_module.CalledProcessError(1, cmd, stderr="boom")
+ return _FakeCompleted()
+ return _FakeCompleted()
+
+ monkeypatch.setattr(utils_module.subprocess, "run", fake_run)
+ ProxyExtrasDBManager._resolve_all_migrations(str(tmp_path), "schema.prisma")
+ return calls
+
+ def _resolved(self, calls):
+ return [c for c in calls if isinstance(c, list) and "resolve" in c]
+
+ def _executed_sql(self, calls):
+ return next(c[1] for c in calls if isinstance(c, tuple) and c[0] == "executed_sql")
+
+ def test_failed_drift_apply_does_not_mark_migrations_applied(self, monkeypatch, tmp_path):
+ calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=True)
+ assert self._resolved(calls) == []
+
+ def test_successful_drift_apply_still_marks_migrations_applied(self, monkeypatch, tmp_path):
+ calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=False)
+ assert len(self._resolved(calls)) == 1
+
+ def test_partitioned_spend_logs_gets_the_filtered_drift_script(self, monkeypatch, tmp_path):
+ calls = self._run(monkeypatch, tmp_path, partitioned=True, execute_fails=False)
+ executed_sql = self._executed_sql(calls)
+ assert 'PRIMARY KEY ("request_id")' not in executed_sql
+ assert "LiteLLM_SpendLogs_legacy" not in executed_sql
+ assert 'ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN "updated_by" TEXT;' in executed_sql
+ assert 'ADD COLUMN "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP' in executed_sql
+ assert len(self._resolved(calls)) == 1
+
+ def test_unpartitioned_spend_logs_drift_script_is_untouched(self, monkeypatch, tmp_path):
+ calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=False)
+ assert self._executed_sql(calls) == _PARTITIONED_DRIFT_SQL
+
+
+class TestPartitionedSpendLogsPushGuard:
+ def _forbid_subprocess(self, monkeypatch):
+ import litellm_proxy_extras.utils as utils_module
+
+ def fail_run(cmd, **kwargs):
+ raise AssertionError(f"subprocess.run should not be called, got: {cmd}")
+
+ monkeypatch.setattr(utils_module.subprocess, "run", fail_run)
+
+ def test_v1_db_push_fails_fast_with_guidance(self, monkeypatch):
+ monkeypatch.setattr(
+ ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: True)
+ )
+ self._forbid_subprocess(monkeypatch)
+ with pytest.raises(RuntimeError) as err:
+ ProxyExtrasDBManager._run_migrations(use_migrate=False, use_v2_resolver=False)
+ assert str(err.value) == PARTITIONED_SPEND_LOGS_PUSH_ERROR
+
+ def test_v2_db_push_fails_fast_with_guidance(self, monkeypatch):
+ monkeypatch.setattr(
+ ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: True)
+ )
+ self._forbid_subprocess(monkeypatch)
+ with pytest.raises(RuntimeError) as err:
+ ProxyExtrasDBManager._setup_database_v2(use_migrate=False)
+ assert str(err.value) == PARTITIONED_SPEND_LOGS_PUSH_ERROR
+
+
+class _FakeCursor:
+ def fetchone(self):
+ return (1,)
+
+
+class _FakePsycopgConn:
+ def __init__(self, executed):
+ self._executed = executed
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ return False
+
+ def execute(self, query, params):
+ self._executed.append((query, params))
+ return _FakeCursor()
+
+
+class TestSpendLogsPartitionDetectionSchemaScope:
+ """A same-named LiteLLM_SpendLogs in another schema must not trip the
+ detector: the catalog lookup has to be scoped to Prisma's target schema."""
+
+ def _detect(self, monkeypatch, database_url):
+ import sys
+ import types
+
+ executed = []
+ fake_psycopg = types.ModuleType("psycopg")
+ fake_psycopg.connect = lambda url, **kwargs: _FakePsycopgConn(executed)
+ fake_psycopg.OperationalError = type("OperationalError", (Exception,), {})
+ fake_psycopg.DatabaseError = type("DatabaseError", (Exception,), {})
+ monkeypatch.setitem(sys.modules, "psycopg", fake_psycopg)
+ monkeypatch.setenv("DATABASE_URL", database_url)
+ assert ProxyExtrasDBManager.spend_logs_is_partitioned() is True
+ return executed[0]
+
+ def test_lookup_is_scoped_to_the_schema_url_param(self, monkeypatch):
+ query, params = self._detect(
+ monkeypatch, "postgresql://u:p@localhost:5432/db?schema=tenant_a"
+ )
+ assert "pg_namespace" in query
+ assert "n.nspname = %s" in query
+ assert params == ("tenant_a",)
+
+ def test_lookup_falls_back_to_public_without_a_schema_param(self, monkeypatch):
+ query, params = self._detect(monkeypatch, "postgresql://u:p@localhost:5432/db")
+ assert "n.nspname = %s" in query
+ assert params == ("public",)
+
+ def test_only_partitioned_relations_match(self, monkeypatch):
+ query, _ = self._detect(monkeypatch, "postgresql://u:p@localhost:5432/db")
+ assert "pg_partitioned_table" in query
diff --git a/tests/mcp_tests/conftest.py b/tests/mcp_tests/conftest.py
index 5823893afc0..eff32f27aec 100644
--- a/tests/mcp_tests/conftest.py
+++ b/tests/mcp_tests/conftest.py
@@ -45,6 +45,22 @@ def setup_and_teardown():
asyncio.set_event_loop(None) # Remove the reference to the loop
+@pytest.fixture(scope="function", autouse=True)
+async def drain_logging_worker():
+ """
+ The logging queue is bound to the running loop, so anything left queued when a test's loop
+ goes away is carried onto the next test's loop and fires against its callbacks.
+ """
+ from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
+
+ yield
+
+ try:
+ await asyncio.wait_for(GLOBAL_LOGGING_WORKER.clear_queue(), timeout=10)
+ except asyncio.TimeoutError:
+ pass
+
+
def pytest_collection_modifyitems(config, items):
# Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests
custom_logger_tests = [
diff --git a/tests/test_litellm/conftest.py b/tests/test_litellm/conftest.py
index 1fe73b552da..62c95cb100b 100644
--- a/tests/test_litellm/conftest.py
+++ b/tests/test_litellm/conftest.py
@@ -375,6 +375,9 @@ def isolate_litellm_state():
litellm.in_memory_llm_clients_cache.flush_cache()
image_handling_module.in_memory_cache.flush_cache()
_reset_module_level_aws_auth_caches()
+ # litellm.get_model_info() memoizes ModelInfo built from litellm.model_cost, so a
+ # test that rebinds the cost map leaves later tests pricing against the old map.
+ litellm_utils_module._invalidate_model_cost_lowercase_map()
# Clear all callback lists to prevent cross-test contamination
if hasattr(litellm, "callbacks"):
@@ -418,6 +421,7 @@ def isolate_litellm_state():
litellm_utils_module._runtime_registered_model_cost.clear()
litellm_utils_module._runtime_registered_model_cost.update(original_runtime_registered_model_cost)
+ litellm_utils_module._invalidate_model_cost_lowercase_map()
for _router in tuple(litellm_router_module._live_routers):
litellm_router_module._live_routers.discard(_router)
diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py
index 455d84c764f..4973bda29e0 100644
--- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py
+++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py
@@ -778,11 +778,15 @@ def test_mcp_span_roots_without_transport_or_propagated_context(
@pytest.mark.parametrize("make_payload, span_name", _MCP_SPAN_CASES)
-def test_mcp_span_parents_to_propagated_meta_trace_context(make_payload, span_name):
+def test_mcp_span_links_propagated_meta_trace_context_and_nests_under_transport(
+ make_payload, span_name
+):
"""When the client propagates W3C trace context in the request's
- ``params._meta`` (SEP-414), the MCP span parents to it (one distributed trace)
- and still links the transport span — never falling through to the
- ambient/session span."""
+ ``params._meta`` (SEP-414), the MCP span still nests under the gateway's own
+ transport span — one renderable trace — and records the client's context as a
+ span *link*. Parenting to the remote context instead would root the span in a
+ trace whose root span never reaches the gateway's tracing backend, leaving the
+ span unreachable from the trace view."""
logger, exporter = _logger()
transport = logger._emitter.start_span(
SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
@@ -801,12 +805,65 @@ def test_mcp_span_parents_to_propagated_meta_trace_context(make_payload, span_na
reset_mcp_message_trace_carrier(token)
transport.end()
span = next(s for s in exporter.get_finished_spans() if s.name == span_name)
- assert span.context.trace_id == 0x11111111111111111111111111111111
assert span.parent is not None
- assert span.parent.span_id == 0x2222222222222222
- assert [link.context.span_id for link in span.links] == [
- transport.get_span_context().span_id
+ assert span.parent.span_id == transport.get_span_context().span_id
+ assert span.context.trace_id == transport.get_span_context().trace_id
+ assert [link.context.trace_id for link in span.links] == [
+ 0x11111111111111111111111111111111
]
+ assert [link.context.span_id for link in span.links] == [0x2222222222222222]
+
+
+@pytest.mark.parametrize("make_payload, span_name", _MCP_SPAN_CASES)
+def test_mcp_span_without_transport_roots_and_links_propagated_context(
+ make_payload, span_name
+):
+ """With no transport span at all there is nothing of the gateway's to anchor
+ to, so the span starts its own root trace — and the client context stays a
+ span link there too, so the event keeps one shape everywhere."""
+ logger, exporter = _logger()
+ token = set_mcp_message_trace_carrier(
+ {"traceparent": "00-11111111111111111111111111111111-2222222222222222-01"}
+ )
+ try:
+ asyncio.run(
+ logger.async_log_success_event(
+ {"standard_logging_object": make_payload()}, None, None, None
+ )
+ )
+ finally:
+ reset_mcp_message_trace_carrier(token)
+ span = next(s for s in exporter.get_finished_spans() if s.name == span_name)
+ assert span.parent is None
+ assert span.context.trace_id != 0x11111111111111111111111111111111
+ assert [link.context.span_id for link in span.links] == [0x2222222222222222]
+
+
+def test_mcp_span_links_unsampled_client_traceparent():
+ """A client traceparent with the sampled flag off ('-00') still yields a valid
+ remote context, so the link is recorded; the span's own recording follows the
+ transport's sampling decision, never the client's flag."""
+ logger, exporter = _logger()
+ transport = logger._emitter.start_span(
+ SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
+ )
+ set_request_root_span(transport)
+ token = set_mcp_message_trace_carrier(
+ {"traceparent": "00-11111111111111111111111111111111-2222222222222222-00"}
+ )
+ try:
+ asyncio.run(
+ logger.async_log_success_event(
+ {"standard_logging_object": _mcp_list_payload()}, None, None, None
+ )
+ )
+ finally:
+ reset_mcp_message_trace_carrier(token)
+ transport.end()
+ span = next(s for s in exporter.get_finished_spans() if s.name == "tools/list")
+ assert span.parent is not None
+ assert span.parent.span_id == transport.get_span_context().span_id
+ assert [link.context.span_id for link in span.links] == [0x2222222222222222]
@pytest.mark.parametrize("make_payload, span_name", _MCP_SPAN_CASES)
@@ -839,8 +896,11 @@ def test_mcp_span_ignores_client_supplied_baggage(make_payload, span_name):
reset_mcp_message_trace_carrier(token)
transport.end()
span = next(s for s in exporter.get_finished_spans() if s.name == span_name)
- # Trace context still honored: proves the carrier was processed, not dropped wholesale.
- assert span.parent is not None and span.parent.span_id == 0x2222222222222222
+ # Trace context still honored (as a link): proves the carrier was processed,
+ # not dropped wholesale.
+ assert [link.context.span_id for link in span.links] == [0x2222222222222222]
+ assert span.parent is not None
+ assert span.parent.span_id == transport.get_span_context().span_id
# Identity is the authenticated payload's team, never the client's spoofed value.
assert span.attributes[LiteLLM.TEAM_ID] == "t1"
assert "litellm.metadata.user_api_key_user_id" not in span.attributes
@@ -888,10 +948,10 @@ def test_mcp_span_malformed_traceparent_nests_under_transport():
assert span.links == ()
-def test_mcp_span_links_this_messages_transport_when_context_is_propagated():
- """On the semconv path the transport is recorded as a link, and that link must
- point at the POST carrying this message too. Reading the stale session anchor
- would attribute the tool call to whichever request opened the session."""
+def test_mcp_span_with_propagated_context_nests_under_this_messages_transport():
+ """With client context propagated, the span must still anchor to the POST
+ carrying this message, not the stale session anchor — otherwise the tool call
+ is attributed to whichever request opened the session."""
logger, exporter = _logger()
session_opener = logger._emitter.start_span(
SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
@@ -916,10 +976,10 @@ def test_mcp_span_links_this_messages_transport_when_context_is_propagated():
session_opener.end()
this_message.end()
span = next(s for s in exporter.get_finished_spans() if s.name == "tools/list")
- assert span.parent is not None and span.parent.span_id == 0x2222222222222222
- assert [link.context.span_id for link in span.links] == [
- this_message.get_span_context().span_id
- ]
+ assert span.parent is not None
+ assert span.parent.span_id == this_message.get_span_context().span_id
+ assert span.context.trace_id == this_message.get_span_context().trace_id
+ assert [link.context.span_id for link in span.links] == [0x2222222222222222]
def test_pre_call_idempotent_keeps_first_span():
diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py
index cc9b311084e..baa72b5a7fe 100644
--- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py
+++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py
@@ -107,32 +107,29 @@ def test_registry_parent_integrity_no_orphans():
def test_registry_hierarchy_shape():
- # MCP roles have no in-process parent: per the MCP semconv they root (or adopt
- # the client's propagated _meta context), so they sit alongside PROXY_REQUEST.
- assert set(root_roles()) == {
- SpanRole.PROXY_REQUEST,
- SpanRole.MCP_TOOL_CALL,
- SpanRole.MCP_LIST_TOOLS,
- }
+ assert set(root_roles()) == {SpanRole.PROXY_REQUEST}
# Guardrails parent to the request span, not the LLM call: a pre-call
- # guardrail runs before the LLM call exists, so it's a sibling of it.
+ # guardrail runs before the LLM call exists, so it's a sibling of it. MCP
+ # spans nest under the transport span of the request carrying that message.
assert set(child_roles(SpanRole.PROXY_REQUEST)) == {
SpanRole.LLM_CALL,
SpanRole.GUARDRAIL,
SpanRole.DB_CALL,
SpanRole.SERVICE,
+ SpanRole.MCP_TOOL_CALL,
+ SpanRole.MCP_LIST_TOOLS,
}
assert SPAN_REGISTRY[SpanRole.LLM_CALL].kind is LiteLLMSpanKind.CLIENT
# The proxy is an MCP client to the upstream tool server: CLIENT span. Listing
# tools is the same client relationship, so it's a CLIENT span too.
assert SPAN_REGISTRY[SpanRole.MCP_TOOL_CALL].kind is LiteLLMSpanKind.CLIENT
assert SPAN_REGISTRY[SpanRole.MCP_LIST_TOOLS].kind is LiteLLMSpanKind.CLIENT
- # MCP spans don't nest under the transport: they link the PROXY_REQUEST span
- # instead of parenting to it (OTel GenAI MCP semconv).
- assert SPAN_REGISTRY[SpanRole.MCP_TOOL_CALL].parent is None
- assert SPAN_REGISTRY[SpanRole.MCP_LIST_TOOLS].parent is None
- assert SPAN_REGISTRY[SpanRole.MCP_TOOL_CALL].links is SpanRole.PROXY_REQUEST
- assert SPAN_REGISTRY[SpanRole.MCP_LIST_TOOLS].links is SpanRole.PROXY_REQUEST
+ # MCP spans nest under the transport span of the request carrying that
+ # message (resolved per message at emit time); a client-propagated context
+ # becomes a span link to that remote context, which is not a registry role
+ # (SpanSpec declares no link field at all).
+ assert SPAN_REGISTRY[SpanRole.MCP_TOOL_CALL].parent is SpanRole.PROXY_REQUEST
+ assert SPAN_REGISTRY[SpanRole.MCP_LIST_TOOLS].parent is SpanRole.PROXY_REQUEST
assert SPAN_REGISTRY[SpanRole.PROXY_REQUEST].kind is LiteLLMSpanKind.SERVER
assert SPAN_REGISTRY[SpanRole.GUARDRAIL].parent is SpanRole.PROXY_REQUEST
# An outbound datastore call is a CLIENT span; an internal service is INTERNAL.
diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py
index 1b71c2f1f9b..52e88db753a 100644
--- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py
+++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py
@@ -3019,3 +3019,95 @@ async def test_provider_config_path_captures_transcription_usage():
and message.get("usage") == usage
)
assert len(usage_events) == 1
+
+
+@pytest.mark.asyncio
+async def test_session_close_flushes_unbilled_transcription_usage():
+ """Trailing audio appended after the last transcript frame must still be billed:
+ on session close the provider's unbilled estimate is flushed into the logged
+ messages before log_messages runs, and never forwarded to the client."""
+ from typing import Final
+
+ from litellm.types.realtime import RealtimeInputAudioTranscriptionUsage
+
+ client_ws: Final = MagicMock()
+ client_ws.send_text = AsyncMock()
+ backend_ws: Final = MagicMock()
+ backend_ws.recv = AsyncMock(side_effect=ConnectionClosed(None, None))
+ logging_obj: Final = MagicMock()
+ logging_obj.async_success_handler = AsyncMock()
+ logging_obj.success_handler = MagicMock()
+
+ usage: Final[RealtimeInputAudioTranscriptionUsage] = {
+ "type": "tokens",
+ "input_tokens": 153,
+ "output_tokens": 18,
+ "total_tokens": 171,
+ "input_token_details": {"text_tokens": 0, "audio_tokens": 153},
+ }
+ provider_config: Final = MagicMock()
+ provider_config.unbilled_usage_on_session_close = MagicMock(return_value=usage)
+
+ streaming: Final = RealTimeStreaming(
+ client_ws,
+ backend_ws,
+ logging_obj,
+ provider_config=provider_config,
+ model="gemini-3.5-transcribe-live",
+ )
+ logged_snapshots: Final[list[tuple]] = []
+
+ original_log_messages: Final = streaming.log_messages
+
+ async def _snapshot_then_log():
+ logged_snapshots.append(tuple(streaming.messages))
+ await original_log_messages()
+
+ streaming.log_messages = _snapshot_then_log
+
+ await streaming.backend_to_client_send_messages()
+
+ provider_config.unbilled_usage_on_session_close.assert_called_once_with("gemini-3.5-transcribe-live")
+ flushed: Final = tuple(
+ message
+ for message in streaming.messages
+ if isinstance(message, dict)
+ and message.get("type") == "conversation.item.input_audio_transcription.completed"
+ and message.get("usage") == usage
+ )
+ assert len(flushed) == 1
+ assert flushed[0] in logged_snapshots[0]
+ assert not client_ws.send_text.called
+
+
+@pytest.mark.asyncio
+async def test_session_close_flush_noop_without_unbilled_usage():
+ """Everything already billed mid-stream: the session-close flush must not append
+ a duplicate transcription event."""
+ from typing import Final
+
+ client_ws: Final = MagicMock()
+ client_ws.send_text = AsyncMock()
+ backend_ws: Final = MagicMock()
+ backend_ws.recv = AsyncMock(side_effect=ConnectionClosed(None, None))
+ logging_obj: Final = MagicMock()
+ logging_obj.async_success_handler = AsyncMock()
+ logging_obj.success_handler = MagicMock()
+
+ provider_config: Final = MagicMock()
+ provider_config.unbilled_usage_on_session_close = MagicMock(return_value=None)
+
+ streaming: Final = RealTimeStreaming(
+ client_ws,
+ backend_ws,
+ logging_obj,
+ provider_config=provider_config,
+ model="gemini-3.5-transcribe-live",
+ )
+
+ await streaming.backend_to_client_send_messages()
+
+ assert not any(
+ isinstance(message, dict) and message.get("type") == "conversation.item.input_audio_transcription.completed"
+ for message in streaming.messages
+ )
diff --git a/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py b/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py
index 1388073381e..5f12ae8566c 100644
--- a/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py
+++ b/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py
@@ -59,17 +59,17 @@ class GptProfile(NamedTuple):
GPT_5_6_PROFILES = [
GptProfile(
model_id="us.openai.gpt-5.6-sol",
- input_cost=5.5e-06, input_cost_above_272k=1.1e-05,
- cache_write=6.875e-06, cache_write_above_272k=1.375e-05,
- cache_read=5.5e-07, cache_read_above_272k=1.1e-06,
- output_cost=3.3e-05, output_cost_above_272k=4.95e-05,
+ input_cost=4.4e-06, input_cost_above_272k=8.8e-06,
+ cache_write=5.5e-06, cache_write_above_272k=1.1e-05,
+ cache_read=4.4e-07, cache_read_above_272k=8.8e-07,
+ output_cost=2.2e-05, output_cost_above_272k=3.3e-05,
),
GptProfile(
model_id="global.openai.gpt-5.6-sol",
- input_cost=5e-06, input_cost_above_272k=1e-05,
- cache_write=6.25e-06, cache_write_above_272k=1.25e-05,
- cache_read=5e-07, cache_read_above_272k=1e-06,
- output_cost=3e-05, output_cost_above_272k=4.5e-05,
+ input_cost=4e-06, input_cost_above_272k=8e-06,
+ cache_write=5e-06, cache_write_above_272k=1e-05,
+ cache_read=4e-07, cache_read_above_272k=8e-07,
+ output_cost=2e-05, output_cost_above_272k=3e-05,
),
GptProfile(
model_id="us.openai.gpt-5.6-terra",
@@ -221,7 +221,7 @@ def test_bedrock_gpt_5_6_above_272k_tier_applies_to_cost(local_model_cost_map):
custom_llm_provider="bedrock",
)
- assert cost == pytest.approx((300000 * 1.1e-05) + (1000 * 4.95e-05), rel=1e-9)
+ assert cost == pytest.approx((300000 * 8.8e-06) + (1000 * 3.3e-05), rel=1e-9)
def test_bedrock_gpt_5_6_bills_cache_read_tokens(local_model_cost_map):
@@ -241,10 +241,10 @@ def test_bedrock_gpt_5_6_bills_cache_read_tokens(local_model_cost_map):
custom_llm_provider="bedrock",
)
- expected = (2 * 5.5e-06) + (15609 * 5.5e-07) + (5 * 3.3e-05)
+ expected = (2 * 4.4e-06) + (15609 * 4.4e-07) + (5 * 2.2e-05)
assert cost == pytest.approx(expected, rel=1e-9)
# Without cache_read_input_token_cost the cached prefix bills at zero.
- assert cost > (15611 * 5.5e-06) * 0.1
+ assert cost > (15611 * 4.4e-06) * 0.1
def test_bedrock_gpt_5_6_bills_cache_write_tokens(local_model_cost_map):
@@ -263,7 +263,7 @@ def test_bedrock_gpt_5_6_bills_cache_write_tokens(local_model_cost_map):
custom_llm_provider="bedrock",
)
- expected = (2 * 5.5e-06) + (15609 * 6.875e-06) + (5 * 3.3e-05)
+ expected = (2 * 4.4e-06) + (15609 * 5.5e-06) + (5 * 2.2e-05)
assert cost == pytest.approx(expected, rel=1e-9)
diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py
index 6a6fb8e3730..a8a20670d4e 100644
--- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py
+++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py
@@ -1683,10 +1683,19 @@ class TestBedrockMantleResponsesPricing:
assert info["cache_read_input_token_cost"] == pytest.approx(2.75e-07)
assert info["max_input_tokens"] == 1050000
+ def test_gpt_5_6_cyber_pricing_and_mode(self, local_cost_map):
+ info = litellm.get_model_info("bedrock_mantle/openai.gpt-5.6-cyber")
+ assert info["mode"] == "responses"
+ assert info["input_cost_per_token"] == pytest.approx(1.375e-05)
+ assert info["cache_creation_input_token_cost"] == pytest.approx(1.71875e-05)
+ assert info["cache_read_input_token_cost"] == pytest.approx(1.375e-06)
+ assert info["output_cost_per_token"] == pytest.approx(8.25e-05)
+ assert info["max_input_tokens"] == 272000
+
@pytest.mark.parametrize(
"model, input_cost, cache_creation_cost, cache_read_cost, output_cost",
[
- ("openai.gpt-5.6-sol", 5.5e-06, 6.875e-06, 5.5e-07, 3.3e-05),
+ ("openai.gpt-5.6-sol", 4.4e-06, 5.5e-06, 4.4e-07, 2.2e-05),
("openai.gpt-5.6-terra", 2.2e-06, 2.75e-06, 2.2e-07, 1.32e-05),
("openai.gpt-5.6-luna", 2.2e-07, 2.75e-07, 2.2e-08, 1.32e-06),
],
@@ -1709,7 +1718,7 @@ class TestBedrockMantleResponsesPricing:
@pytest.mark.parametrize(
"model, input_cost, output_cost",
[
- ("openai.gpt-5.6-sol", 5.5e-06, 3.3e-05),
+ ("openai.gpt-5.6-sol", 4.4e-06, 2.2e-05),
("openai.gpt-5.6-terra", 2.2e-06, 1.32e-05),
("openai.gpt-5.6-luna", 2.2e-07, 1.32e-06),
],
diff --git a/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py b/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py
index 21f047b753c..29ad8ee4b6e 100644
--- a/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py
+++ b/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py
@@ -61,6 +61,8 @@ PUBLISHED_DBU_PER_MILLION: Final = {
"databricks/databricks-gemini-3-1-flash-lite": ("4.464", "26.786", "4.464", "0.446"),
"databricks/databricks-gemini-2-5-pro": ("22.321", "178.571", "22.321", "2.232"),
"databricks/databricks-gemini-2-5-flash": ("5.357", "44.643", "5.357", "0.536"),
+ "databricks/databricks-kimi-k3": ("42.857", "214.286", "42.857", "4.286"),
+ "databricks/databricks-glm-5-2": ("20.000", "62.857", "20.000", "3.714"),
}
PROMOTIONAL_DISCOUNT: Final = 0.80
PROMOTION_EXPIRES: Final = "2027-01-31"
diff --git a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py
index c362efbfffa..42994dbd2af 100644
--- a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py
+++ b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py
@@ -2119,3 +2119,27 @@ def test_non_transcription_live_model_completed_event_has_no_usage(patch_gemini_
)
assert len(completed) == 1
assert "usage" not in completed[0]
+
+
+def test_unbilled_usage_on_session_close_flushes_trailing_audio(patch_gemini_transcribe_live_cost_map_entry):
+ """Audio appended after the last transcript frame is still unbilled when the
+ session closes; the session-close hook must hand back the estimate exactly once
+ so the streaming layer can bill it (144000 pcm16 bytes = 3s -> 75 in / 9 out)."""
+ from typing import Final
+
+ from litellm.types.realtime import RealtimeInputAudioTranscriptionUsage
+
+ config: Final = GeminiRealtimeConfig()
+ config.transform_realtime_request(_input_audio_append_message(144000), "gemini-3.5-transcribe-live")
+
+ usage: Final = config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live")
+
+ expected: Final[RealtimeInputAudioTranscriptionUsage] = {
+ "type": "tokens",
+ "input_tokens": 75,
+ "output_tokens": 9,
+ "total_tokens": 84,
+ "input_token_details": {"text_tokens": 0, "audio_tokens": 75},
+ }
+ assert usage == expected
+ assert config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") is None
diff --git a/tests/test_litellm/llms/xai/test_xai_model_registry.py b/tests/test_litellm/llms/xai/test_xai_model_registry.py
new file mode 100644
index 00000000000..25b2002968d
--- /dev/null
+++ b/tests/test_litellm/llms/xai/test_xai_model_registry.py
@@ -0,0 +1,75 @@
+"""
+Registry regression tests for xAI entries in the model cost map.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).parents[4]
+PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
+BACKUP_PRICES_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
+
+# Retired by xAI and no longer served: requests to these slugs 404 rather than
+# redirecting, and they are absent from https://docs.x.ai/docs/models
+RETIRED_MODELS = (
+ "xai/grok-2",
+ "xai/grok-2-1212",
+ "xai/grok-2-latest",
+ "xai/grok-2-vision",
+ "xai/grok-2-vision-1212",
+ "xai/grok-2-vision-latest",
+ "xai/grok-beta",
+ "xai/grok-vision-beta",
+)
+
+# https://docs.x.ai/developers/model-capabilities/text/multi-agent
+# "The multi-agent model does not work with the OpenAI Chat Completions API."
+RESPONSES_ONLY_MODELS = (
+ "xai/grok-4.20-multi-agent-0309",
+ "xai/grok-4.20-multi-agent-beta-0309",
+)
+
+MAP_PATHS = (PRICES_PATH, BACKUP_PRICES_PATH)
+
+
+@pytest.fixture(scope="module", params=[p.name for p in MAP_PATHS])
+def cost_map(request: pytest.FixtureRequest) -> dict:
+ path = next(p for p in MAP_PATHS if p.name == request.param)
+ return json.loads(path.read_text(encoding="utf-8"))
+
+
+@pytest.mark.parametrize("model", RETIRED_MODELS)
+def test_retired_xai_models_are_not_advertised(cost_map: dict, model: str):
+ assert model not in cost_map
+
+
+@pytest.mark.parametrize("model", RESPONSES_ONLY_MODELS)
+def test_multi_agent_models_are_responses_only(cost_map: dict, model: str):
+ entry = cost_map[model]
+ assert entry["supported_endpoints"] == ["/v1/responses"]
+ assert entry["mode"] == "responses"
+ assert "/v1/chat/completions" not in entry["supported_endpoints"]
+
+
+def test_surviving_xai_chat_models_still_serve_chat_completions(cost_map: dict):
+ """Guard against the removal above over-reaching into live models."""
+ chat_models = [
+ key
+ for key, value in cost_map.items()
+ if isinstance(value, dict) and value.get("litellm_provider") == "xai" and value.get("mode") == "chat"
+ ]
+ assert "xai/grok-4.3" in chat_models
+ assert "xai/grok-4.6" in chat_models
+ assert not any(key.startswith("xai/grok-2") for key in chat_models)
+
+
+def test_both_cost_maps_agree_on_xai_entries():
+ prices = json.loads(PRICES_PATH.read_text(encoding="utf-8"))
+ backup = json.loads(BACKUP_PRICES_PATH.read_text(encoding="utf-8"))
+ xai_keys = {k for k, v in prices.items() if isinstance(v, dict) and v.get("litellm_provider") == "xai"}
+ assert xai_keys
+ assert {k: prices[k] for k in xai_keys} == {k: backup[k] for k in xai_keys}
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py
index 4081681daef..56851d31241 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py
@@ -5,7 +5,8 @@ Validates that:
1. _convert_mcp_hook_response_to_kwargs extracts extra_headers from hook response
2. pre_call_tool_check returns hook-provided extra_headers AND modified arguments
3. call_tool flows hook headers and modified arguments downstream
-4. Hook-provided headers take highest priority (merge after static_headers)
+4. Hook-provided headers merge after static_headers, but a hook Authorization
+ header never displaces an existing upstream Authorization credential
5. OpenAPI-backed servers log a warning and continue (skip injection) when hook headers are present
6. JWT claims are propagated in both standard and virtual-key fast paths
7. Backward compatibility: hooks without extra_headers continue to work
@@ -487,8 +488,8 @@ class TestHookHeaderMergePriority:
)
@pytest.mark.asyncio
- async def test_hook_headers_override_static_headers(self):
- """Hook headers should take precedence over static_headers."""
+ async def test_hook_authorization_does_not_override_static_authorization(self):
+ """A hook Authorization must not displace a static_headers Authorization (LIT-6321)."""
manager = MCPServerManager()
server = self._make_server(static_headers={"Authorization": "Bearer static-token", "X-Static": "yes"})
@@ -521,7 +522,7 @@ class TestHookHeaderMergePriority:
pass
headers = captured_extra_headers.get("value", {})
- assert headers["Authorization"] == "Bearer hook-signed-jwt"
+ assert headers["Authorization"] == "Bearer static-token"
assert headers["X-Static"] == "yes"
@pytest.mark.asyncio
@@ -560,8 +561,8 @@ class TestHookHeaderMergePriority:
assert headers == {"X-Static": "static-value"}
@pytest.mark.asyncio
- async def test_hook_headers_merge_with_oauth2(self):
- """Hook headers merge on top of OAuth2 headers."""
+ async def test_hook_authorization_does_not_override_oauth2_authorization(self):
+ """tools/call keeps the user's OAuth Authorization; only non-auth hook headers merge (LIT-6321)."""
manager = MCPServerManager()
server = MCPServer(
server_id="test-id",
@@ -570,6 +571,8 @@ class TestHookHeaderMergePriority:
url="https://example.com",
transport=MCPTransport.http,
auth_type=MCPAuth.oauth2,
+ oauth2_flow="authorization_code",
+ delegate_auth_to_upstream=True,
)
captured_extra_headers: Dict[str, Any] = {}
@@ -605,10 +608,245 @@ class TestHookHeaderMergePriority:
pass
headers = captured_extra_headers.get("value", {})
- assert headers["Authorization"] == "Bearer hook-jwt"
+ assert headers["Authorization"] == "Bearer oauth2-token"
assert headers["X-OAuth"] == "yes"
assert headers["X-Trace-Id"] == "trace-123"
+ @pytest.mark.asyncio
+ async def test_hook_authorization_used_when_no_upstream_credential(self):
+ """With no upstream credential, the signer JWT is still injected."""
+ manager = MCPServerManager()
+ server = self._make_server()
+
+ captured_extra_headers: Dict[str, Optional[Dict[str, str]]] = {}
+
+ async def fake_create_mcp_client(server, mcp_auth_header=None, extra_headers=None, stdio_env=None, **kwargs):
+ captured_extra_headers["value"] = extra_headers
+ mock_client = MagicMock()
+ mock_client.call_tool = AsyncMock(return_value=MagicMock())
+ return mock_client
+
+ with patch.object(manager, "_create_mcp_client", side_effect=fake_create_mcp_client):
+ with patch.object(manager, "_build_stdio_env", return_value=None):
+ try:
+ await manager._call_regular_mcp_tool(
+ mcp_server=server,
+ original_tool_name="test_tool",
+ arguments={"key": "val"},
+ tasks=[],
+ mcp_auth_header=None,
+ mcp_server_auth_headers=None,
+ oauth2_headers=None,
+ raw_headers=None,
+ proxy_logging_obj=None,
+ hook_extra_headers={"Authorization": "Bearer hook-jwt"},
+ )
+ except Exception:
+ pass
+
+ headers = captured_extra_headers.get("value") or {}
+ assert headers["Authorization"] == "Bearer hook-jwt"
+
+ @pytest.mark.asyncio
+ async def test_hook_authorization_dropped_when_server_auth_header_present(self):
+ """With a configured authentication_token (auth_value), the hook Authorization is dropped."""
+ manager = MCPServerManager()
+ server = self._make_server()
+
+ captured: Dict[str, object] = {}
+
+ async def fake_create_mcp_client(server, mcp_auth_header=None, extra_headers=None, stdio_env=None, **kwargs):
+ captured["extra_headers"] = extra_headers
+ captured["mcp_auth_header"] = mcp_auth_header
+ mock_client = MagicMock()
+ mock_client.call_tool = AsyncMock(return_value=MagicMock())
+ return mock_client
+
+ with patch.object(manager, "_create_mcp_client", side_effect=fake_create_mcp_client):
+ with patch.object(manager, "_build_stdio_env", return_value=None):
+ try:
+ await manager._call_regular_mcp_tool(
+ mcp_server=server,
+ original_tool_name="test_tool",
+ arguments={"key": "val"},
+ tasks=[],
+ mcp_auth_header="server-static-token",
+ mcp_server_auth_headers=None,
+ oauth2_headers=None,
+ raw_headers=None,
+ proxy_logging_obj=None,
+ hook_extra_headers={
+ "Authorization": "Bearer hook-jwt",
+ "X-Trace-Id": "trace-123",
+ },
+ )
+ except Exception:
+ pass
+
+ headers = captured.get("extra_headers") or {}
+ assert isinstance(headers, dict)
+ assert "Authorization" not in headers
+ assert headers.get("X-Trace-Id") == "trace-123"
+ assert captured.get("mcp_auth_header") == "server-static-token"
+
+ @pytest.mark.asyncio
+ async def test_hook_authorization_case_insensitive_conflict(self):
+ """Authorization conflicts are matched case-insensitively."""
+ manager = MCPServerManager()
+ server = self._make_server(static_headers={"authorization": "Bearer static-token"})
+
+ captured_extra_headers: Dict[str, Optional[Dict[str, str]]] = {}
+
+ async def fake_create_mcp_client(server, mcp_auth_header=None, extra_headers=None, stdio_env=None, **kwargs):
+ captured_extra_headers["value"] = extra_headers
+ mock_client = MagicMock()
+ mock_client.call_tool = AsyncMock(return_value=MagicMock())
+ return mock_client
+
+ with patch.object(manager, "_create_mcp_client", side_effect=fake_create_mcp_client):
+ with patch.object(manager, "_build_stdio_env", return_value=None):
+ try:
+ await manager._call_regular_mcp_tool(
+ mcp_server=server,
+ original_tool_name="test_tool",
+ arguments={"key": "val"},
+ tasks=[],
+ mcp_auth_header=None,
+ mcp_server_auth_headers=None,
+ oauth2_headers=None,
+ raw_headers=None,
+ proxy_logging_obj=None,
+ hook_extra_headers={"Authorization": "Bearer hook-jwt"},
+ )
+ except Exception:
+ pass
+
+ headers = captured_extra_headers.get("value") or {}
+ assert headers.get("authorization") == "Bearer static-token"
+ assert "Authorization" not in headers
+
+ @pytest.mark.asyncio
+ async def test_hook_authorization_kept_with_api_key_server_credential(self):
+ """An api_key credential maps to X-API-Key, so the hook Authorization is kept."""
+ manager = MCPServerManager()
+ server = MCPServer(
+ server_id="test-id",
+ name="Test Server",
+ server_name="test_server",
+ url="https://example.com",
+ transport=MCPTransport.http,
+ auth_type=MCPAuth.api_key,
+ )
+
+ captured: Dict[str, object] = {}
+
+ async def fake_create_mcp_client(server, mcp_auth_header=None, extra_headers=None, stdio_env=None, **kwargs):
+ captured["extra_headers"] = extra_headers
+ captured["mcp_auth_header"] = mcp_auth_header
+ mock_client = MagicMock()
+ mock_client.call_tool = AsyncMock(return_value=MagicMock())
+ return mock_client
+
+ with patch.object(manager, "_create_mcp_client", side_effect=fake_create_mcp_client):
+ with patch.object(manager, "_build_stdio_env", return_value=None):
+ try:
+ await manager._call_regular_mcp_tool(
+ mcp_server=server,
+ original_tool_name="test_tool",
+ arguments={"key": "val"},
+ tasks=[],
+ mcp_auth_header="server-api-key",
+ mcp_server_auth_headers=None,
+ oauth2_headers=None,
+ raw_headers=None,
+ proxy_logging_obj=None,
+ hook_extra_headers={"Authorization": "Bearer hook-jwt"},
+ )
+ except Exception:
+ pass
+
+ headers = captured.get("extra_headers") or {}
+ assert isinstance(headers, dict)
+ assert headers.get("Authorization") == "Bearer hook-jwt"
+ assert captured.get("mcp_auth_header") == "server-api-key"
+
+ @pytest.mark.asyncio
+ async def test_hook_authorization_kept_with_non_authorization_server_header_dict(self):
+ """A per-server header dict without Authorization does not block the hook JWT."""
+ manager = MCPServerManager()
+ server = self._make_server()
+
+ captured: Dict[str, object] = {}
+
+ async def fake_create_mcp_client(server, mcp_auth_header=None, extra_headers=None, stdio_env=None, **kwargs):
+ captured["extra_headers"] = extra_headers
+ captured["mcp_auth_header"] = mcp_auth_header
+ mock_client = MagicMock()
+ mock_client.call_tool = AsyncMock(return_value=MagicMock())
+ return mock_client
+
+ with patch.object(manager, "_create_mcp_client", side_effect=fake_create_mcp_client):
+ with patch.object(manager, "_build_stdio_env", return_value=None):
+ try:
+ await manager._call_regular_mcp_tool(
+ mcp_server=server,
+ original_tool_name="test_tool",
+ arguments={"key": "val"},
+ tasks=[],
+ mcp_auth_header=None,
+ mcp_server_auth_headers={"test_server": {"X-API-Key": "per-server-key"}},
+ oauth2_headers=None,
+ raw_headers=None,
+ proxy_logging_obj=None,
+ hook_extra_headers={"Authorization": "Bearer hook-jwt"},
+ )
+ except Exception:
+ pass
+
+ headers = captured.get("extra_headers") or {}
+ assert isinstance(headers, dict)
+ assert headers.get("Authorization") == "Bearer hook-jwt"
+ assert captured.get("mcp_auth_header") == {"X-API-Key": "per-server-key"}
+
+ @pytest.mark.asyncio
+ async def test_hook_authorization_dropped_with_authorization_server_header_dict(self):
+ """A per-server header dict carrying Authorization blocks the hook JWT."""
+ manager = MCPServerManager()
+ server = self._make_server()
+
+ captured: Dict[str, object] = {}
+
+ async def fake_create_mcp_client(server, mcp_auth_header=None, extra_headers=None, stdio_env=None, **kwargs):
+ captured["extra_headers"] = extra_headers
+ captured["mcp_auth_header"] = mcp_auth_header
+ mock_client = MagicMock()
+ mock_client.call_tool = AsyncMock(return_value=MagicMock())
+ return mock_client
+
+ with patch.object(manager, "_create_mcp_client", side_effect=fake_create_mcp_client):
+ with patch.object(manager, "_build_stdio_env", return_value=None):
+ try:
+ await manager._call_regular_mcp_tool(
+ mcp_server=server,
+ original_tool_name="test_tool",
+ arguments={"key": "val"},
+ tasks=[],
+ mcp_auth_header=None,
+ mcp_server_auth_headers={"test_server": {"authorization": "Bearer per-server-token"}},
+ oauth2_headers=None,
+ raw_headers=None,
+ proxy_logging_obj=None,
+ hook_extra_headers={"Authorization": "Bearer hook-jwt", "X-Trace-Id": "trace-123"},
+ )
+ except Exception:
+ pass
+
+ headers = captured.get("extra_headers") or {}
+ assert isinstance(headers, dict)
+ assert "Authorization" not in headers
+ assert headers.get("X-Trace-Id") == "trace-123"
+ assert captured.get("mcp_auth_header") == {"authorization": "Bearer per-server-token"}
+
@pytest.mark.asyncio
async def test_m2m_oauth2_does_not_forward_litellm_caller_authorization(self):
"""M2M must not put caller Bearer (LiteLLM API key) into extra_headers (#23652)."""
diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
index 6a117985820..d44f96d95bf 100644
--- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
+++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
@@ -4972,6 +4972,143 @@ async def test_centralized_common_checks_ui_sentinel_team_vouches_despite_absent
setattr(_proxy_server_mod, k, v)
+@pytest.mark.asyncio
+async def test_centralized_common_checks_ui_sentinel_team_skips_db_lookup():
+ """LIT-6297 / GH#28775: ``UI_TEAM_ID`` never has a team row and the
+ not-found path bypasses the DB throttle, so building the team fetch for it
+ cost one guaranteed-miss ``LiteLLM_TeamTable.find_unique`` plus a 404 debug
+ log on every dashboard request. The gate must not call ``get_team_object``
+ for the sentinel at all, while the token-derived team object still reaches
+ ``common_checks``."""
+ import litellm.proxy.proxy_server as _proxy_server_mod
+ from fastapi import Request
+ from starlette.datastructures import URL
+
+ from litellm.proxy._types import UI_TEAM_ID, LiteLLM_TeamTableCachedObj
+
+ token = UserAPIKeyAuth(
+ api_key="sk-test",
+ user_id="ui-session-user",
+ team_id=UI_TEAM_ID,
+ models=[],
+ team_models=[],
+ )
+ request = Request(scope={"type": "http"})
+ request._url = URL(url="/user/info")
+ request._body = b"{}"
+
+ received_team_objects: list[LiteLLM_TeamTableCachedObj | None] = []
+
+ async def _capturing_common_checks(*_args, **kwargs) -> bool:
+ received_team_objects.append(kwargs.get("team_object"))
+ return True
+
+ attrs = _proxy_attrs_for_centralized_checks(user_custom_auth=None)
+ originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs}
+ try:
+ for k, v in attrs.items():
+ setattr(_proxy_server_mod, k, v)
+ with (
+ patch( # test-quality-ok: the regression IS that this DB lookup is never made for the sentinel
+ "litellm.proxy.auth.user_api_key_auth.get_team_object",
+ new_callable=AsyncMock,
+ ) as mock_get_team_object,
+ patch( # test-quality-ok: capture the team_object the consumer receives without a DB
+ "litellm.proxy.auth.user_api_key_auth.common_checks",
+ _capturing_common_checks,
+ ),
+ ):
+ await _run_centralized_common_checks(
+ user_api_key_auth_obj=token,
+ request=request,
+ request_data={},
+ route="/user/info",
+ )
+ mock_get_team_object.assert_not_awaited()
+ assert len(received_team_objects) == 1
+ received_team_object = received_team_objects[0]
+ assert received_team_object is not None
+ assert received_team_object.team_id == UI_TEAM_ID
+ finally:
+ for k, v in originals.items():
+ setattr(_proxy_server_mod, k, v)
+
+
+@pytest.mark.asyncio
+async def test_builder_ui_sentinel_team_never_hits_get_team_object(): # test-quality-ok: absence of the guaranteed-miss DB call is the observable being pinned
+ """Companion to the centralized-gate test for the builder path: the cached
+ UI session token's team refresh and the post-validation team fetch must
+ both skip ``get_team_object`` for ``UI_TEAM_ID`` instead of 404ing on
+ every request."""
+ import litellm.proxy.proxy_server as _proxy_server_mod
+ from fastapi import Request
+ from starlette.datastructures import URL
+
+ from litellm.proxy._types import UI_TEAM_ID
+ from litellm.proxy.auth.user_api_key_auth import _user_api_key_auth_builder
+ from litellm.proxy.proxy_server import hash_token
+
+ api_key = "sk-test-ui-session-key"
+ cached_token = UserAPIKeyAuth(
+ api_key=api_key,
+ token=hash_token(api_key),
+ user_id="ui-session-user",
+ user_role=LitellmUserRoles.INTERNAL_USER,
+ team_id=UI_TEAM_ID,
+ )
+
+ mock_proxy_logging_obj = MagicMock()
+ mock_proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
+
+ attrs = {
+ "prisma_client": MagicMock(),
+ "user_api_key_cache": DualCache(),
+ "proxy_logging_obj": mock_proxy_logging_obj,
+ "master_key": "sk-master-key",
+ "general_settings": {},
+ "llm_model_list": [],
+ "llm_router": None,
+ "open_telemetry_logger": None,
+ "model_max_budget_limiter": MagicMock(),
+ "user_custom_auth": None,
+ "jwt_handler": None,
+ "litellm_proxy_admin_name": "admin",
+ }
+ originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs}
+ try:
+ for k, v in attrs.items():
+ setattr(_proxy_server_mod, k, v)
+
+ request = Request(scope={"type": "http"})
+ request._url = URL(url="/user/info")
+
+ with (
+ patch( # test-quality-ok: seed the cached UI session token without a DB
+ "litellm.proxy.auth.resolvers.store.IdentityStore._resolve_key",
+ new_callable=AsyncMock,
+ return_value=cached_token,
+ ),
+ patch( # test-quality-ok: the regression IS that this DB lookup is never made for the sentinel
+ "litellm.proxy.auth.user_api_key_auth.get_team_object",
+ new_callable=AsyncMock,
+ ) as mock_get_team_object,
+ ):
+ result = await _user_api_key_auth_builder(
+ request=request,
+ api_key=f"Bearer {api_key}",
+ azure_api_key_header="",
+ anthropic_api_key_header=None,
+ google_ai_studio_api_key_header=None,
+ azure_apim_header=None,
+ request_data={},
+ )
+ assert result.team_id == UI_TEAM_ID
+ mock_get_team_object.assert_not_awaited()
+ finally:
+ for k, v in originals.items():
+ setattr(_proxy_server_mod, k, v)
+
+
@pytest.mark.asyncio
async def test_centralized_common_checks_user_http_exception_isolates_to_user_only():
"""Per-fetch isolation, mirror of the team case: an HTTPException
diff --git a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py
index aa844304604..5d3afd95a55 100644
--- a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py
+++ b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py
@@ -1458,7 +1458,7 @@ def test_budget_cascade_writes_land_in_a_single_transaction(reset_budget_job, mo
budget = _budget_row(budget_id="budget-1", budget_duration="7d")
mock_prisma_client.data["budget"] = [budget]
mock_prisma_client.data["enduser"] = [
- type("EndUser", (), {"spend": 5.0, "litellm_budget_table": budget, "user_id": "enduser-1"})
+ type("EndUser", (), {"spend": 5.0, "litellm_budget_table": budget, "user_id": "enduser-1", "budget_id": "budget-1"})
]
asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table())
@@ -2588,3 +2588,303 @@ def test_ambiguous_commit_replay_does_not_erase_newly_accrued_spend(
assert client.key_spend == expected_spend
assert client.commit_attempts == expected_commits
assert client.reconnect_reasons == expected_reconnects
+
+
+# ---------------------------------------------------------------------------
+# Budget rollover (LIT-3085): overage beyond max_budget carries into the next
+# window instead of being forgiven
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture
+def rollover_enabled(monkeypatch):
+ import litellm
+
+ monkeypatch.setattr(litellm, "budget_rollover", True)
+
+
+@pytest.mark.parametrize(
+ "run_phase, table, id_field, id_value, row_factory",
+ [
+ (
+ lambda job: job.reset_budget_for_litellm_keys(),
+ "key",
+ "token",
+ "tok-roll",
+ lambda now: type(
+ "Key",
+ (),
+ {
+ "spend": 150.0,
+ "max_budget": 100.0,
+ "budget_duration": "1d",
+ "budget_reset_at": now,
+ "token": "tok-roll",
+ },
+ ),
+ ),
+ (
+ lambda job: job.reset_budget_for_litellm_users(),
+ "user",
+ "user_id",
+ "user-roll",
+ lambda now: type(
+ "User",
+ (),
+ {
+ "spend": 150.0,
+ "max_budget": 100.0,
+ "budget_duration": "30d",
+ "budget_reset_at": now,
+ "user_id": "user-roll",
+ },
+ ),
+ ),
+ (
+ lambda job: job.reset_budget_for_litellm_teams(),
+ "team",
+ "team_id",
+ "team-roll",
+ lambda now: type(
+ "Team",
+ (),
+ {
+ "spend": 150.0,
+ "max_budget": 100.0,
+ "budget_duration": "1mo",
+ "budget_reset_at": now,
+ "team_id": "team-roll",
+ },
+ ),
+ ),
+ ],
+)
+def test_direct_reset_carries_overage_when_rollover_enabled(
+ rollover_enabled, reset_budget_job, mock_prisma_client, monkeypatch, run_phase, table, id_field, id_value, row_factory
+):
+ """spend=150 against max_budget=100 must decrement by the cap (leaving 50)
+ rather than zero the row, and the spend counter must be seeded with 50."""
+ counter_cache = _make_counter_invalidation_job(monkeypatch)
+ now = datetime.now(timezone.utc)
+ mock_prisma_client.data[table] = [row_factory(now)]
+
+ asyncio.run(run_phase(reset_budget_job))
+
+ writes = _batch_writes(mock_prisma_client, table)
+ assert len(writes) == 1
+ assert writes[0]["where"] == {id_field: id_value}
+ assert writes[0]["data"]["spend"] == {"decrement": 100.0}
+ assert writes[0]["data"]["budget_reset_at"] > now
+ counter_prefix = {"key": "spend:key", "user": "spend:user", "team": "spend:team"}[table]
+ counter_cache.in_memory_cache.set_cache.assert_any_call(key=f"{counter_prefix}:{id_value}", value=50.0, ttl=60)
+
+
+def test_direct_reset_zeroes_under_budget_row_even_with_rollover(
+ rollover_enabled, reset_budget_job, mock_prisma_client, monkeypatch
+):
+ counter_cache = _make_counter_invalidation_job(monkeypatch)
+ now = datetime.now(timezone.utc)
+ mock_prisma_client.data["key"] = [
+ type(
+ "Key",
+ (),
+ {"spend": 40.0, "max_budget": 100.0, "budget_duration": "1d", "budget_reset_at": now, "token": "tok-under"},
+ )
+ ]
+
+ asyncio.run(reset_budget_job.reset_budget_for_litellm_keys())
+
+ assert _batch_writes(mock_prisma_client, "key")[0]["data"]["spend"] == 0
+ counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:key:tok-under", value=0.0, ttl=60)
+
+
+def test_direct_reset_zeroes_row_without_max_budget_even_with_rollover(
+ rollover_enabled, reset_budget_job, mock_prisma_client, monkeypatch
+):
+ """No cap means nothing to carry against: reset to zero as before."""
+ _make_counter_invalidation_job(monkeypatch)
+ now = datetime.now(timezone.utc)
+ mock_prisma_client.data["key"] = [
+ type(
+ "Key",
+ (),
+ {"spend": 150.0, "max_budget": None, "budget_duration": "1d", "budget_reset_at": now, "token": "tok-nocap"},
+ )
+ ]
+
+ asyncio.run(reset_budget_job.reset_budget_for_litellm_keys())
+
+ assert _batch_writes(mock_prisma_client, "key")[0]["data"]["spend"] == 0
+
+
+def test_budget_cascade_carries_overage_per_tier_when_rollover_enabled(
+ rollover_enabled, reset_budget_job, mock_prisma_client, monkeypatch
+):
+ """A team member 5 over the tier cap keeps a spend of 5 in the next window:
+ the cascade decrements over-cap rows by the cap, zeroes the rest, and seeds
+ the spend counter with the carried amount."""
+ counter_cache = _make_counter_invalidation_job(monkeypatch)
+ budget = _budget_row(budget_id="budget-roll", budget_duration="7d", max_budget=10.0)
+ mock_prisma_client.data["budget"] = [budget]
+ membership = type(
+ "Membership",
+ (),
+ {"user_id": "member-1", "team_id": "team-1", "spend": 15.0, "budget_id": "budget-roll"},
+ )
+ mock_prisma_client.db.litellm_teammembership.set_find_many_results([membership])
+
+ asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table())
+
+ membership_writes = _batch_writes(mock_prisma_client, "team_membership")
+ assert {
+ "table": "team_membership",
+ "op": "update_many",
+ "where": {"budget_id": "budget-roll", "spend": {"gt": 10.0}},
+ "data": {"spend": {"decrement": 10.0}},
+ } in membership_writes
+ assert {
+ "table": "team_membership",
+ "op": "update_many",
+ "where": {"budget_id": "budget-roll", "spend": {"gt": 0, "lte": 10.0}},
+ "data": {"spend": 0},
+ } in membership_writes
+ counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:team_member:member-1:team-1", value=5.0, ttl=60)
+
+
+def test_budget_cascade_carries_enduser_overage_when_rollover_enabled(
+ rollover_enabled, reset_budget_job, mock_prisma_client, monkeypatch
+):
+ _make_counter_invalidation_job(monkeypatch)
+ budget = _budget_row(budget_id="budget-roll", budget_duration="1d", max_budget=10.0)
+ mock_prisma_client.data["budget"] = [budget]
+ mock_prisma_client.data["enduser"] = [
+ type(
+ "EndUser",
+ (),
+ {"spend": 15.0, "litellm_budget_table": budget, "user_id": "enduser-roll", "budget_id": "budget-roll"},
+ )
+ ]
+
+ asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table())
+
+ enduser_writes = _batch_writes(mock_prisma_client, "enduser")
+ assert {
+ "table": "enduser",
+ "op": "update_many",
+ "where": {"user_id": {"in": ["enduser-roll"]}, "spend": {"gt": 10.0}},
+ "data": {"spend": {"decrement": 10.0}},
+ } in enduser_writes
+ assert {
+ "table": "enduser",
+ "op": "update_many",
+ "where": {"user_id": {"in": ["enduser-roll"]}, "spend": {"lte": 10.0}},
+ "data": {"spend": 0},
+ } in enduser_writes
+
+
+def _replay_spend_writes(writes, spend):
+ """Apply the queued update_many statements in order, the way the DB
+ transaction executes them, and return the row's final spend."""
+ for write in writes:
+ condition = write["where"].get("spend")
+ if isinstance(condition, dict):
+ if "gt" in condition and not spend > condition["gt"]:
+ continue
+ if "lte" in condition and not spend <= condition["lte"]:
+ continue
+ payload = write["data"]["spend"]
+ spend = payload if not isinstance(payload, dict) else spend - payload["decrement"]
+ return spend
+
+
+@pytest.mark.parametrize("table", ["team_membership", "enduser"])
+def test_cascade_rollover_writes_survive_sequential_execution(
+ rollover_enabled, reset_budget_job, mock_prisma_client, monkeypatch, table
+):
+ """The statements run one after another inside a transaction, so a
+ decrement-then-zero order would re-match the decremented row (now in the
+ 0..cap range) and erase the carried spend. Replaying the writes in queue
+ order must leave the overage, for any spend between cap and twice the cap."""
+ _make_counter_invalidation_job(monkeypatch)
+ budget = _budget_row(budget_id="budget-roll", budget_duration="7d", max_budget=10.0)
+ mock_prisma_client.data["budget"] = [budget]
+ membership = type(
+ "Membership",
+ (),
+ {"user_id": "member-1", "team_id": "team-1", "spend": 15.0, "budget_id": "budget-roll"},
+ )
+ mock_prisma_client.db.litellm_teammembership.set_find_many_results([membership])
+ mock_prisma_client.data["enduser"] = [
+ type(
+ "EndUser",
+ (),
+ {"spend": 15.0, "litellm_budget_table": budget, "user_id": "enduser-roll", "budget_id": "budget-roll"},
+ )
+ ]
+
+ asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table())
+
+ writes = _batch_writes(mock_prisma_client, table)
+ assert _replay_spend_writes(writes, 15.0) == 5.0
+ assert _replay_spend_writes(writes, 8.0) == 0
+ assert _replay_spend_writes(writes, 25.0) == 15.0
+
+
+def test_budget_cascade_zeroes_everything_when_rollover_disabled(reset_budget_job, mock_prisma_client, monkeypatch):
+ """Control: with the flag off the cascade keeps the plain zeroing writes."""
+ _make_counter_invalidation_job(monkeypatch)
+ budget = _budget_row(budget_id="budget-off", budget_duration="7d", max_budget=10.0)
+ mock_prisma_client.data["budget"] = [budget]
+
+ asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table())
+
+ membership_writes = _batch_writes(mock_prisma_client, "team_membership")
+ assert membership_writes == [
+ {
+ "table": "team_membership",
+ "op": "update_many",
+ "where": {"budget_id": {"in": ["budget-off"]}},
+ "data": {"spend": 0},
+ }
+ ]
+
+
+def test_window_reset_carries_counter_overage_when_rollover_enabled(rollover_enabled, monkeypatch):
+ """A per-window counter at 130 against a 100 cap restarts the window at 30."""
+ now = datetime.utcnow()
+ expired = (now - timedelta(minutes=5)).isoformat() + "Z"
+ key_rows = [
+ {
+ "token": "sk-roll",
+ "budget_limits": [{"budget_duration": "1d", "reset_at": expired, "max_budget": 100.0}],
+ }
+ ]
+ job, prisma_client, spend_counter_cache = _make_reset_budget_windows_job(
+ monkeypatch, key_rows=key_rows, team_rows=[]
+ )
+ spend_counter_cache.async_get_cache = AsyncMock(return_value=130.0)
+
+ asyncio.run(job.reset_budget_windows())
+
+ prisma_client.db.litellm_verificationtoken.update.assert_awaited_once()
+ spend_counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:key:sk-roll:window:1d", value=30.0)
+
+
+def test_window_reset_zeroes_counter_when_rollover_disabled(monkeypatch):
+ now = datetime.utcnow()
+ expired = (now - timedelta(minutes=5)).isoformat() + "Z"
+ key_rows = [
+ {
+ "token": "sk-off",
+ "budget_limits": [{"budget_duration": "1d", "reset_at": expired, "max_budget": 100.0}],
+ }
+ ]
+ job, prisma_client, spend_counter_cache = _make_reset_budget_windows_job(
+ monkeypatch, key_rows=key_rows, team_rows=[]
+ )
+ spend_counter_cache.async_get_cache = AsyncMock(return_value=130.0)
+
+ asyncio.run(job.reset_budget_windows())
+
+ spend_counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:key:sk-off:window:1d", value=0.0)
+ spend_counter_cache.async_get_cache.assert_not_awaited()
diff --git a/tests/test_litellm/proxy/db/test_prisma_client.py b/tests/test_litellm/proxy/db/test_prisma_client.py
index b1ecbfeff8e..f0983d6bf62 100644
--- a/tests/test_litellm/proxy/db/test_prisma_client.py
+++ b/tests/test_litellm/proxy/db/test_prisma_client.py
@@ -215,6 +215,44 @@ def test_db_push_applies_replica_identity_full_when_requested(monkeypatch):
assert applied == [True]
+def test_db_push_is_rejected_when_spend_logs_is_partitioned(monkeypatch):
+ """A doc-partitioned LiteLLM_SpendLogs makes `prisma db push` rewrite the
+ primary key back to ("request_id"), which Postgres rejects; the guard must
+ fail fast with guidance instead of running the push."""
+ from litellm.proxy.db.prisma_client import PrismaManager
+ from litellm_proxy_extras.utils import (
+ PARTITIONED_SPEND_LOGS_PUSH_ERROR,
+ ProxyExtrasDBManager,
+ )
+
+ monkeypatch.setattr(
+ ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: True)
+ )
+ with patch( # test-quality-ok: subprocess.run is the external prisma CLI boundary, asserted never reached
+ "litellm.proxy.db.prisma_client.subprocess.run"
+ ) as mock_run:
+ with pytest.raises(RuntimeError) as err:
+ PrismaManager.setup_database(use_migrate=False)
+
+ assert str(err.value) == PARTITIONED_SPEND_LOGS_PUSH_ERROR
+ mock_run.assert_not_called()
+
+
+def test_db_push_proceeds_when_spend_logs_is_not_partitioned(monkeypatch):
+ from litellm.proxy.db.prisma_client import PrismaManager
+ from litellm_proxy_extras.utils import ProxyExtrasDBManager
+
+ monkeypatch.setattr(
+ ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: False)
+ )
+ with patch( # test-quality-ok: subprocess.run is the external prisma CLI boundary, not SDK logic
+ "litellm.proxy.db.prisma_client.subprocess.run"
+ ) as mock_run:
+ assert PrismaManager.setup_database(use_migrate=False) is True
+
+ assert mock_run.call_args[0][0][:3] == ["prisma", "db", "push"]
+
+
def _entra_jwt(expires_in_seconds: int) -> str:
"""A JWT shaped like a real Entra access token, expiring ``expires_in_seconds`` from now."""
import base64
diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
index a37c4f72b3d..6045b64023d 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
@@ -730,6 +730,68 @@ async def test_update_key_personal_non_admin_denied_vector_stores(monkeypatch):
assert "Vector stores" in str(exc.value.detail)
+@pytest.mark.asyncio
+async def test_update_key_grandfathers_existing_mcp_servers(monkeypatch):
+ """/key/update on a team key that already holds MCP servers outside the
+ team allowlist must accept re-sent or shrunk grants (LIT-6062). The wrapper
+ must pass the existing key's object_permission row into the validator when
+ the team is unchanged."""
+ from unittest.mock import AsyncMock, MagicMock
+
+ from litellm.proxy._types import (
+ LiteLLM_ObjectPermissionBase,
+ UpdateKeyRequest,
+ )
+ from litellm.proxy.management_endpoints.key_management_endpoints import (
+ _validate_mcp_servers_for_key_update,
+ )
+
+ existing_row = MagicMock()
+ existing_row.mcp_servers = ["server-a", "server-b"]
+ existing_row.mcp_tool_permissions = {}
+ mock_prisma = MagicMock()
+ mock_prisma.db.litellm_mcpservertable.find_many = AsyncMock(return_value=[])
+
+ team_obj = MagicMock()
+ team_obj.team_id = "team-1"
+ team_obj.object_permission = None
+
+ existing_key_row = MagicMock(
+ team_id="team-1",
+ object_permission_id="perm-1",
+ object_permission=existing_row,
+ )
+
+ mock_server_a = MagicMock()
+ mock_server_a.server_id = "server-a"
+ mock_server_b = MagicMock()
+ mock_server_b.server_id = "server-b"
+ mock_mgr = MagicMock()
+ mock_mgr.get_registry.return_value = {
+ "server-a": mock_server_a,
+ "server-b": mock_server_b,
+ }
+ mock_mgr.get_allow_all_keys_server_ids.return_value = []
+ monkeypatch.setattr(
+ "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager",
+ mock_mgr,
+ )
+
+ result = await _validate_mcp_servers_for_key_update(
+ data=UpdateKeyRequest(
+ key="sk-team-key",
+ object_permission=LiteLLM_ObjectPermissionBase(mcp_servers=["server-a"]),
+ ),
+ team_obj=team_obj,
+ existing_key_row=existing_key_row,
+ prisma_client=mock_prisma,
+ user_api_key_cache=MagicMock(),
+ is_proxy_admin=False,
+ )
+ assert result is not None
+ assert result["mcp_servers"] == ["server-a"]
+
+
@pytest.mark.asyncio
async def test_update_key_personal_non_admin_denied_access_groups(
monkeypatch,
@@ -6552,7 +6614,7 @@ async def test_get_and_validate_existing_key():
assert result == mock_key
mock_prisma_client.db.litellm_verificationtoken.find_unique.assert_called_once_with(
- where={"token": "hashed-test-key-123"}
+ where={"token": "hashed-test-key-123"}, include={"object_permission": True}
)
# Test Case 2: Key not found raises ProxyException
diff --git a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py
index b129ad0f659..5ef83344c1a 100644
--- a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py
+++ b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py
@@ -1213,6 +1213,124 @@ async def test_empty_object_permission_passes_for_personal_non_admin():
)
+# ---- Tests for grandfathering existing key MCP servers on /key/update (LIT-6062) ----
+
+
+def _make_grandfather_fixtures(mcp_servers=None, mcp_tool_permissions=None):
+ """Mock prisma client plus the key's existing object permission row."""
+ existing_row = MagicMock()
+ existing_row.mcp_servers = mcp_servers or []
+ existing_row.mcp_tool_permissions = mcp_tool_permissions or {}
+ mock_prisma = MagicMock()
+ mock_prisma.db.litellm_mcpservertable.find_many = AsyncMock(return_value=[])
+ return mock_prisma, existing_row
+
+
+def _patch_grandfather_env(monkeypatch, mock_mgr):
+ monkeypatch.setattr(
+ "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager",
+ mock_mgr,
+ )
+ monkeypatch.setattr(
+ "litellm.proxy.management_helpers.object_permission_utils._get_allow_all_keys_server_ids",
+ lambda: set(),
+ )
+
+
+@pytest.mark.asyncio
+async def test_validate_key_update_grandfathers_existing_servers(monkeypatch):
+ """A key already holding servers outside the team allowlist can re-send or
+ shrink those grants on /key/update without a 403 (LIT-6062)."""
+ _patch_grandfather_env(monkeypatch, _make_mock_mcp_manager("server-a", "server-b"))
+ team_obj = _make_team_obj(mcp_servers=[])
+ mock_prisma, existing_row = _make_grandfather_fixtures(mcp_servers=["server-a", "server-b"])
+ resend = await validate_key_mcp_servers_against_team(
+ object_permission={"mcp_servers": ["server-a", "server-b"]},
+ team_obj=team_obj,
+ prisma_client=mock_prisma,
+ existing_key_object_permission=existing_row,
+ )
+ assert sorted(resend["mcp_servers"]) == ["server-a", "server-b"]
+ shrink = await validate_key_mcp_servers_against_team(
+ object_permission={"mcp_servers": ["server-a"]},
+ team_obj=team_obj,
+ prisma_client=mock_prisma,
+ existing_key_object_permission=existing_row,
+ )
+ assert shrink["mcp_servers"] == ["server-a"]
+
+
+@pytest.mark.asyncio
+async def test_validate_key_update_grandfather_does_not_allow_new_servers(monkeypatch):
+ """Grandfathering only covers servers the key already holds; adding a new
+ server outside the team allowlist still raises 403."""
+ _patch_grandfather_env(monkeypatch, _make_mock_mcp_manager("server-a", "server-new"))
+ team_obj = _make_team_obj(mcp_servers=[])
+ mock_prisma, existing_row = _make_grandfather_fixtures(mcp_servers=["server-a"])
+ with pytest.raises(HTTPException) as exc_info:
+ await validate_key_mcp_servers_against_team(
+ object_permission={"mcp_servers": ["server-a", "server-new"]},
+ team_obj=team_obj,
+ prisma_client=mock_prisma,
+ existing_key_object_permission=existing_row,
+ )
+ assert exc_info.value.status_code == 403
+ assert "server-new" in str(exc_info.value.detail)
+
+
+@pytest.mark.asyncio
+async def test_validate_key_update_without_existing_permission_still_raises(monkeypatch):
+ """Without an existing permission row (new grants or team change) the
+ subset check stays strict."""
+ _patch_grandfather_env(monkeypatch, _make_mock_mcp_manager("server-a"))
+ team_obj = _make_team_obj(mcp_servers=[])
+ mock_prisma, _ = _make_grandfather_fixtures(mcp_servers=["server-a"])
+ with pytest.raises(HTTPException) as exc_info:
+ await validate_key_mcp_servers_against_team(
+ object_permission={"mcp_servers": ["server-a"]},
+ team_obj=team_obj,
+ prisma_client=mock_prisma,
+ existing_key_object_permission=None,
+ )
+ assert exc_info.value.status_code == 403
+
+
+@pytest.mark.asyncio
+async def test_validate_key_update_grandfathers_tool_permission_keys(monkeypatch):
+ """Servers granted only via mcp_tool_permissions keys on the existing row
+ (stored as a JSON string) are grandfathered too."""
+ _patch_grandfather_env(monkeypatch, _make_mock_mcp_manager("server-a"))
+ team_obj = _make_team_obj(mcp_servers=[])
+ mock_prisma, existing_row = _make_grandfather_fixtures(
+ mcp_tool_permissions=json.dumps({"server-a": ["tool1"]})
+ )
+ result = await validate_key_mcp_servers_against_team(
+ object_permission={"mcp_servers": ["server-a"]},
+ team_obj=team_obj,
+ prisma_client=mock_prisma,
+ existing_key_object_permission=existing_row,
+ )
+ assert result["mcp_servers"] == ["server-a"]
+
+
+@pytest.mark.asyncio
+async def test_validate_key_update_sentinels_do_not_grandfather(monkeypatch):
+ """Sentinels stored on the existing row must not grandfather anything."""
+ _patch_grandfather_env(monkeypatch, _make_mock_mcp_manager("server-a"))
+ team_obj = _make_team_obj(mcp_servers=[])
+ mock_prisma, existing_row = _make_grandfather_fixtures(
+ mcp_servers=[SpecialMCPServerName.all_proxy_servers.value, "no-mcp-servers"]
+ )
+ with pytest.raises(HTTPException) as exc_info:
+ await validate_key_mcp_servers_against_team(
+ object_permission={"mcp_servers": ["server-a"]},
+ team_obj=team_obj,
+ prisma_client=mock_prisma,
+ existing_key_object_permission=existing_row,
+ )
+ assert exc_info.value.status_code == 403
+
+
def test_object_permission_dict_mirrors_pydantic_model():
"""ObjectPermissionDict must stay field-for-field aligned with
LiteLLM_ObjectPermissionBase. If a new field is added to the Pydantic
diff --git a/tests/test_litellm/proxy/proxy_server/test_background_health.py b/tests/test_litellm/proxy/proxy_server/test_background_health.py
index d5a97c0a087..990844369f7 100644
--- a/tests/test_litellm/proxy/proxy_server/test_background_health.py
+++ b/tests/test_litellm/proxy/proxy_server/test_background_health.py
@@ -581,3 +581,73 @@ async def test_run_background_health_check_runs_one_cycle_then_cancels(monkeypat
"unhealthy_count": 1,
"sleep_invoked": True,
}
+
+
+@pytest.mark.asyncio
+async def test_run_background_health_check_probes_only_listed_model_groups(monkeypatch):
+ monkeypatch.setattr(proxy_server, "health_check_interval", 60)
+ monkeypatch.setattr(proxy_server, "health_check_concurrency", 1)
+ monkeypatch.setattr(proxy_server, "health_check_details", True)
+ monkeypatch.setattr(proxy_server, "use_shared_health_check", False)
+ monkeypatch.setattr(proxy_server, "redis_usage_cache", None)
+ monkeypatch.setattr(proxy_server, "prisma_client", None)
+ monkeypatch.setattr(proxy_server, "background_health_check_loop_active", False)
+ monkeypatch.setattr(
+ proxy_server,
+ "llm_router",
+ SimpleNamespace(background_health_check_model_groups=frozenset({"prod-openai"})),
+ )
+ monkeypatch.setattr(
+ proxy_server,
+ "llm_model_list",
+ [
+ {"model_name": "prod-openai", "model_info": {"id": "listed-1"}},
+ {"model_name": "prod-openai", "model_info": {"id": "listed-2"}},
+ {"model_name": "internal-claude", "model_info": {"id": "unlisted-1"}},
+ {
+ "model_name": "prod-openai",
+ "model_info": {
+ "id": "listed-disabled",
+ "disable_background_health_check": True,
+ },
+ },
+ ],
+ )
+ monkeypatch.setattr(
+ proxy_server,
+ "health_check_results",
+ {"healthy_endpoints": [], "unhealthy_endpoints": []},
+ )
+
+ probed = {}
+
+ async def _fake_direct(model_list, *_a, **_kw):
+ probed["ids"] = [m["model_info"]["id"] for m in model_list]
+ return ([], [], {})
+
+ monkeypatch.setattr(
+ proxy_server,
+ "_run_direct_health_check_with_instrumentation",
+ _fake_direct,
+ )
+ monkeypatch.setattr(
+ proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None
+ )
+ monkeypatch.setattr(
+ proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None
+ )
+ monkeypatch.setattr(
+ proxy_server,
+ "health_check_filter_kwargs_from_general_settings",
+ lambda _gs: {},
+ )
+
+ async def _stop_sleep(_seconds):
+ raise asyncio.CancelledError()
+
+ monkeypatch.setattr(proxy_server.asyncio, "sleep", _stop_sleep)
+
+ with pytest.raises(asyncio.CancelledError):
+ await _run_background_health_check()
+
+ assert probed["ids"] == ["listed-1", "listed-2"]
diff --git a/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py b/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py
index dda2f5a4d73..8f25cffecf5 100644
--- a/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py
+++ b/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py
@@ -1767,16 +1767,20 @@ async def test_the_prune_cutoff_allows_for_clock_skew_between_hosts():
@pytest.mark.asyncio
-async def test_a_run_pricing_config_cannot_prune_a_row_it_did_not_scan(monkeypatch):
+async def test_a_run_pricing_config_cannot_prune_a_row_it_did_not_scan():
"""Staleness alone stops being evidence once two hosts hold different configuration: a
row this run never considered belongs to a deployment another host is pricing from its
own file, and sweeping it drops that charge."""
table = _FakeSentinelTable()
table.seed("t", DAY, "dep-elsewhere", 480.0, updated_at=datetime(2020, 1, 1, tzinfo=timezone.utc))
entry = _router_entry(model_id="cfg-here", model_info=dict(_VALID_PTU))
- monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(entry))
- await run_scheduled_ptu_rollup(_prisma_for([], table), pod_lock_manager=_pod_lock(acquired=True), target_date=DAY)
+ await run_scheduled_ptu_rollup(
+ _prisma_for([], table),
+ pod_lock_manager=_pod_lock(acquired=True),
+ target_date=DAY,
+ router=_router_holding(entry),
+ )
assert ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "dep-elsewhere") in table.rows
assert ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "cfg-here") in table.rows
@@ -1784,7 +1788,7 @@ async def test_a_run_pricing_config_cannot_prune_a_row_it_did_not_scan(monkeypat
@pytest.mark.asyncio
-async def test_a_deployment_deleted_from_the_table_keeps_the_day_it_was_charged(monkeypatch):
+async def test_a_deployment_deleted_from_the_table_keeps_the_day_it_was_charged():
"""The accepted cost of bounding the prune, driven through the sequence that produces
it: charge the day while the deployment exists, remove it, run the day again. Nothing
scans it now, so nothing may judge its row, and the amount it was billed stands."""
@@ -1793,18 +1797,19 @@ async def test_a_deployment_deleted_from_the_table_keeps_the_day_it_was_charged(
live_row = _model_row(model_id="dep-live", model_info=ptu)
doomed_row = _model_row(model_id="dep-doomed", model_info=ptu)
charged_key = ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "dep-doomed")
- monkeypatch.setattr(
- ptu_rollup, "_running_router", lambda: _router_holding(_router_entry(model_id="cfg", model_info=dict(ptu)))
- )
+ router = _router_holding(_router_entry(model_id="cfg", model_info=dict(ptu)))
await run_scheduled_ptu_rollup(
- _prisma_for([live_row, doomed_row], table), pod_lock_manager=_pod_lock(acquired=True), target_date=DAY
+ _prisma_for([live_row, doomed_row], table),
+ pod_lock_manager=_pod_lock(acquired=True),
+ target_date=DAY,
+ router=router,
)
billed = table.rows[charged_key]["ptu_flat_cost"]
table.rows[charged_key]["updated_at"] = datetime(2020, 1, 1, tzinfo=timezone.utc)
await run_scheduled_ptu_rollup(
- _prisma_for([live_row], table), pod_lock_manager=_pod_lock(acquired=True), target_date=DAY
+ _prisma_for([live_row], table), pod_lock_manager=_pod_lock(acquired=True), target_date=DAY, router=router
)
assert table.rows[charged_key]["ptu_flat_cost"] == billed
@@ -1843,7 +1848,7 @@ async def test_every_deployment_that_prices_is_inside_the_set_that_bounds_the_pr
table,
)
- loaded = await ptu_rollup._load_ptu_models(prisma)
+ loaded = await ptu_rollup._load_ptu_models(prisma, router=None)
assert {model.model_id for model in loaded.models} <= loaded.scanned_ids
assert loaded.scanned_ids == {"dep-a", "dep-b", "dep-unpriced"}
@@ -1859,7 +1864,7 @@ async def test_a_priced_deployment_is_in_the_bound_even_with_an_id_the_scan_skip
_FakeSentinelTable(),
)
- loaded = await ptu_rollup._load_ptu_models(prisma)
+ loaded = await ptu_rollup._load_ptu_models(prisma, router=None)
assert {model.model_id for model in loaded.models} <= loaded.scanned_ids
@@ -1873,13 +1878,13 @@ async def test_the_prune_splits_the_id_set_across_statements(monkeypatch):
table = _FakeSentinelTable()
ptu = {"ptu_count": 5, "cost_per_ptu_per_hour": 2.0, "team_id": "t"}
deployments = [_model_row(model_id=f"dep-{n}", model_info=ptu) for n in range(4)]
- monkeypatch.setattr(
- ptu_rollup, "_running_router", lambda: _router_holding(_router_entry(model_id="dep-4", model_info=dict(ptu)))
- )
table.seed("t", DAY, "dep-3", 480.0, updated_at=datetime(2020, 1, 1, tzinfo=timezone.utc))
await run_scheduled_ptu_rollup(
- _prisma_for(deployments, table), pod_lock_manager=_pod_lock(acquired=True), target_date=DAY
+ _prisma_for(deployments, table),
+ pod_lock_manager=_pod_lock(acquired=True),
+ target_date=DAY,
+ router=_router_holding(_router_entry(model_id="dep-4", model_info=dict(ptu))),
)
chunks = [call["model"]["in"] for call in table.delete_many_calls]
@@ -1912,140 +1917,123 @@ def _router_holding(*entries):
@pytest.mark.asyncio
-async def test_a_config_declared_deployment_is_priced(monkeypatch):
+async def test_a_config_declared_deployment_is_priced():
"""The whole point. A PTU deployment the proxy only knows from config.yaml is not in
LiteLLM_ProxyModelTable, so a DB-only scan bills the provider's reservation to nobody."""
entry = _router_entry(model_id="cfg-1", model_name="gpt-4o-ptu", model_info=dict(_VALID_PTU))
- monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(entry))
- loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable()))
+ loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable()), router=_router_holding(entry))
assert [(m.model_id, m.model_name, m.team_id) for m in loaded.models] == [("cfg-1", "gpt-4o-ptu", "t")]
assert "cfg-1" in loaded.scanned_ids
@pytest.mark.asyncio
-async def test_a_database_backed_router_entry_is_not_counted_twice(monkeypatch):
+async def test_a_database_backed_router_entry_is_not_counted_twice():
"""Every deployment loaded from the table is also in the router, flagged db_model. Pricing
both copies would write two charges for one reservation."""
row = _model_row(model_id="db-1", model_info=dict(_VALID_PTU))
mirrored = _router_entry(model_id="db-1", model_info={**_VALID_PTU, "db_model": True})
- monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(mirrored))
-
- loaded = await ptu_rollup._load_ptu_models(_prisma_for([row], _FakeSentinelTable()))
-
- assert [m.model_id for m in loaded.models] == ["db-1"]
-
-
-@pytest.mark.asyncio
-async def test_a_router_entry_sharing_an_id_with_the_table_is_priced_once(monkeypatch):
- """db_model is data the router carries rather than something this module controls, so the
- id anti-join is what actually maps onto the failure: two charges under one id."""
- row = _model_row(model_id="both-1", model_info=dict(_VALID_PTU))
- unflagged = _router_entry(model_id="both-1", model_info=dict(_VALID_PTU))
- monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(unflagged))
-
- loaded = await ptu_rollup._load_ptu_models(_prisma_for([row], _FakeSentinelTable()))
-
- assert [m.model_id for m in loaded.models] == ["both-1"]
-
-
-@pytest.mark.asyncio
-async def test_a_client_credential_clone_is_not_priced(monkeypatch):
- """Supplying an api_key on a request mints a clone of the deployment under a fresh id,
- carrying the source's PTU config. Pricing it bills one reservation per distinct caller key."""
- source = _router_entry(model_id="cfg-1", model_info=dict(_VALID_PTU))
- clone = _router_entry(model_id="cfg-1-clone", model_info={**_VALID_PTU, "original_model_id": "cfg-1"})
- monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(source, clone))
-
- loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable()))
-
- assert [m.model_id for m in loaded.models] == ["cfg-1"]
-
-
-@pytest.mark.asyncio
-async def test_a_config_deployment_without_ptu_config_is_scanned_but_not_priced(monkeypatch):
- """It has to stay in the scanned set or its leftover sentinel rows become unprunable."""
- entry = _router_entry(model_id="cfg-plain", model_info={"team_id": "t"})
- monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(entry))
-
- loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable()))
-
- assert loaded.models == ()
- assert "cfg-plain" in loaded.scanned_ids
-
-
-@pytest.mark.asyncio
-async def test_no_router_in_the_process_prices_the_database_alone(monkeypatch):
- """The rollup is importable and callable outside a running proxy."""
- monkeypatch.setattr(ptu_rollup, "_running_router", lambda: None)
loaded = await ptu_rollup._load_ptu_models(
- _prisma_for([_model_row(model_id="db-1", model_info=dict(_VALID_PTU))], _FakeSentinelTable())
+ _prisma_for([row], _FakeSentinelTable()), router=_router_holding(mirrored)
)
assert [m.model_id for m in loaded.models] == ["db-1"]
@pytest.mark.asyncio
-async def test_a_config_deployment_is_charged_end_to_end(monkeypatch):
+async def test_a_router_entry_sharing_an_id_with_the_table_is_priced_once():
+ """db_model is data the router carries rather than something this module controls, so the
+ id anti-join is what actually maps onto the failure: two charges under one id."""
+ row = _model_row(model_id="both-1", model_info=dict(_VALID_PTU))
+ unflagged = _router_entry(model_id="both-1", model_info=dict(_VALID_PTU))
+
+ loaded = await ptu_rollup._load_ptu_models(
+ _prisma_for([row], _FakeSentinelTable()), router=_router_holding(unflagged)
+ )
+
+ assert [m.model_id for m in loaded.models] == ["both-1"]
+
+
+@pytest.mark.asyncio
+async def test_a_client_credential_clone_is_not_priced():
+ """Supplying an api_key on a request mints a clone of the deployment under a fresh id,
+ carrying the source's PTU config. Pricing it bills one reservation per distinct caller key."""
+ source = _router_entry(model_id="cfg-1", model_info=dict(_VALID_PTU))
+ clone = _router_entry(model_id="cfg-1-clone", model_info={**_VALID_PTU, "original_model_id": "cfg-1"})
+
+ loaded = await ptu_rollup._load_ptu_models(
+ _prisma_for([], _FakeSentinelTable()), router=_router_holding(source, clone)
+ )
+
+ assert [m.model_id for m in loaded.models] == ["cfg-1"]
+
+
+@pytest.mark.asyncio
+async def test_a_config_deployment_without_ptu_config_is_scanned_but_not_priced():
+ """It has to stay in the scanned set or its leftover sentinel rows become unprunable."""
+ entry = _router_entry(model_id="cfg-plain", model_info={"team_id": "t"})
+
+ loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable()), router=_router_holding(entry))
+
+ assert loaded.models == ()
+ assert "cfg-plain" in loaded.scanned_ids
+
+
+@pytest.mark.asyncio
+async def test_no_router_in_the_process_prices_the_database_alone():
+ """The rollup is importable and callable outside a running proxy."""
+ loaded = await ptu_rollup._load_ptu_models(
+ _prisma_for([_model_row(model_id="db-1", model_info=dict(_VALID_PTU))], _FakeSentinelTable()), router=None
+ )
+
+ assert [m.model_id for m in loaded.models] == ["db-1"]
+
+
+@pytest.mark.asyncio
+async def test_a_config_deployment_is_charged_end_to_end():
"""Through the scheduled entry point, so the charge lands in a sentinel row rather than
stopping at the loader."""
table = _FakeSentinelTable()
entry = _router_entry(model_id="cfg-1", model_name="gpt-4o-ptu", model_info=dict(_VALID_PTU))
- monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(entry))
- await run_scheduled_ptu_rollup(_prisma_for([], table), pod_lock_manager=_pod_lock(acquired=True), target_date=DAY)
+ await run_scheduled_ptu_rollup(
+ _prisma_for([], table),
+ pod_lock_manager=_pod_lock(acquired=True),
+ target_date=DAY,
+ router=_router_holding(entry),
+ )
assert ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "cfg-1") in table.rows
@pytest.mark.asyncio
-async def test_a_stale_database_backed_router_entry_is_not_treated_as_config(monkeypatch):
+async def test_a_stale_database_backed_router_entry_is_not_treated_as_config():
"""The reconcile can leave a deployment on the router after its row is gone. The id
anti-join cannot see that one, so the flag is what keeps it from being priced as though
config.yaml had declared it."""
stale = _router_entry(model_id="db-gone", model_info={**_VALID_PTU, "db_model": True})
- monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(stale))
- loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable()))
+ loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable()), router=_router_holding(stale))
assert loaded.models == ()
-def test_the_router_lookup_reads_the_proxys_own_global():
- """Every other config test replaces this helper, so without one test driving the real
- body a typo in the module path or the attribute name leaves the whole feature dead in
- production with the suite still green."""
- import sys
- import types as _types
+@pytest.mark.asyncio
+async def test_a_router_left_on_the_proxy_module_is_not_scanned(monkeypatch):
+ """A run scans the router its caller hands it and nothing else. Reading the proxy module's
+ global instead made every run depend on whatever else in the process had set one, which
+ is what a caller passing no router is asking not to happen."""
+ import litellm.proxy.proxy_server as proxy_server
- assert ptu_rollup._running_router() is None or "litellm.proxy.proxy_server" in sys.modules
+ ambient = _router_holding(_router_entry(model_id="ambient-1", model_info=dict(_VALID_PTU)))
+ monkeypatch.setattr(proxy_server, "llm_router", ambient, raising=False)
- sentinel = object()
- stub = _types.SimpleNamespace(llm_router=sentinel)
- real = sys.modules.get("litellm.proxy.proxy_server")
- sys.modules["litellm.proxy.proxy_server"] = stub
- try:
- assert ptu_rollup._running_router() is sentinel
- del stub.llm_router
- assert ptu_rollup._running_router() is None
- finally:
- if real is None:
- del sys.modules["litellm.proxy.proxy_server"]
- else:
- sys.modules["litellm.proxy.proxy_server"] = real
+ loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable()), router=None)
-
-def test_the_router_lookup_returns_none_outside_a_proxy():
- import sys
-
- real = sys.modules.pop("litellm.proxy.proxy_server", None)
- try:
- assert ptu_rollup._running_router() is None
- finally:
- if real is not None:
- sys.modules["litellm.proxy.proxy_server"] = real
+ assert loaded.models == ()
+ assert loaded.scanned_ids == frozenset()
def test_the_prune_filter_is_a_plain_dict():
@@ -2074,7 +2062,7 @@ async def test_a_run_that_scanned_nothing_issues_no_delete_statements():
@pytest.mark.asyncio
-async def test_the_catch_up_pass_reaches_a_config_declared_deployment(monkeypatch):
+async def test_the_catch_up_pass_reaches_a_config_declared_deployment():
"""The catch-up shares the loader, so config deployments join it without being wired in.
That is what prices the elapsed days of a reservation declared before today."""
table = _FakeSentinelTable()
@@ -2084,9 +2072,10 @@ async def test_the_catch_up_pass_reaches_a_config_declared_deployment(monkeypatc
model_id="cfg-back",
model_info={"ptu_count": 100, "cost_per_ptu_per_hour": 0.02, "team_id": "t", "ptu_effective_from": started},
)
- monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(entry))
- await run_scheduled_ptu_rollup(_prisma_for([], table), pod_lock_manager=_pod_lock(acquired=True))
+ await run_scheduled_ptu_rollup(
+ _prisma_for([], table), pod_lock_manager=_pod_lock(acquired=True), router=_router_holding(entry)
+ )
charged = sorted(day for (_, day, _, model) in table.rows if model == "cfg-back")
yesterday = (now.date() - timedelta(days=1)).isoformat()
diff --git a/tests/test_litellm/proxy/test_health_check_functions.py b/tests/test_litellm/proxy/test_health_check_functions.py
index f2d95131e5e..fdae11d517a 100644
--- a/tests/test_litellm/proxy/test_health_check_functions.py
+++ b/tests/test_litellm/proxy/test_health_check_functions.py
@@ -623,5 +623,53 @@ async def test_perform_health_check_and_save_forwards_skip_disabled_background_f
assert call_kwargs["health_check_skip_disabled_background_models"] is True
+def test_parse_background_health_check_model_groups_unset_returns_none():
+ from litellm.proxy.health_check import parse_background_health_check_model_groups
+
+ assert parse_background_health_check_model_groups(None) is None
+ assert parse_background_health_check_model_groups({}) is None
+ assert (
+ parse_background_health_check_model_groups(
+ {"background_health_check_model_groups": None}
+ )
+ is None
+ )
+
+
+def test_parse_background_health_check_model_groups_list_returns_frozenset():
+ from litellm.proxy.health_check import parse_background_health_check_model_groups
+
+ parsed = parse_background_health_check_model_groups(
+ {"background_health_check_model_groups": ["prod-openai", "prod-claude"]}
+ )
+ assert parsed == frozenset({"prod-openai", "prod-claude"})
+
+
+@pytest.mark.parametrize("bad_value", ["prod-openai", 42, {"a": 1}, [1, 2], [None]])
+def test_parse_background_health_check_model_groups_malformed_raises(bad_value):
+ from litellm.proxy.health_check import parse_background_health_check_model_groups
+
+ with pytest.raises(ValueError, match="must be a list of model group names"):
+ parse_background_health_check_model_groups(
+ {"background_health_check_model_groups": bad_value}
+ )
+
+
+def test_filter_deployments_to_model_groups():
+ from litellm.proxy.health_check import filter_deployments_to_model_groups
+
+ model_list = [
+ {"model_name": "prod-openai", "model_info": {"id": "a"}},
+ {"model_name": "internal-claude", "model_info": {"id": "b"}},
+ {"model_name": "prod-openai", "model_info": {"id": "c"}},
+ ]
+
+ assert filter_deployments_to_model_groups(model_list, None) == tuple(model_list)
+ assert filter_deployments_to_model_groups(
+ model_list, frozenset({"prod-openai"})
+ ) == (model_list[0], model_list[2])
+ assert filter_deployments_to_model_groups(model_list, frozenset()) == ()
+
+
if __name__ == "__main__":
pytest.main([__file__])
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index f51648faf80..2c8ae9f0370 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -11245,6 +11245,35 @@ async def test_ptu_rollup_job_registered_at_startup(monkeypatch):
assert scheduler.get_job(PTU_ROLLUP_JOB_ID) is not None
+@pytest.mark.asyncio
+async def test_ptu_rollup_job_hands_the_rollup_the_proxys_router(monkeypatch):
+ """The rollup prices PTU deployments declared in config.yaml, which only the router
+ knows about. It takes the router as an argument, so nothing but this call site puts the
+ proxy's own router in front of it: without it that half of the feature is dead."""
+ monkeypatch.delenv("STORE_MODEL_IN_DB", raising=False)
+ from litellm.proxy.spend_tracking import ptu_flat_cost_rollup
+ from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_ENV_VAR
+ from litellm.proxy.spend_tracking.ptu_flat_cost_rollup import PTU_ROLLUP_JOB_ID
+
+ monkeypatch.setenv(PTU_COST_ATTRIBUTION_ENV_VAR, "true")
+ calls = []
+ monkeypatch.setattr(
+ ptu_flat_cost_rollup,
+ "run_scheduled_ptu_rollup",
+ AsyncMock(side_effect=lambda *args, **kwargs: calls.append(kwargs)),
+ )
+
+ scheduler = await _run_scheduled_background_jobs()
+
+ import litellm.proxy.proxy_server as ps
+
+ router = MagicMock()
+ monkeypatch.setattr(ps, "llm_router", router)
+ await scheduler.get_job(PTU_ROLLUP_JOB_ID).func()
+
+ assert [call["router"] for call in calls] == [router]
+
+
@pytest.mark.asyncio
async def test_ptu_rollup_job_not_registered_without_opt_in(monkeypatch):
"""Without LITELLM_ENABLE_PTU_COST_ATTRIBUTION the rollup never runs, so no sentinel row
diff --git a/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py b/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py
index 64239f33966..6effbc5fa7f 100644
--- a/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py
+++ b/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py
@@ -502,6 +502,68 @@ class TestHealthCheckFilterBypassWithPolicy:
)
assert len(result) == 2
+ def _make_scoped_router_with_unhealthy(self, policy) -> Router:
+ import time
+
+ from litellm.caching.caching import DualCache
+ from litellm.router_utils.health_state_cache import DeploymentHealthCache
+
+ router = Router(
+ model_list=[
+ _make_model("bad-listed"),
+ _make_model("ok-listed"),
+ _make_model("bad-unlisted", "gpt-5"),
+ ],
+ allowed_fails_policy=policy,
+ enable_health_check_routing=True,
+ background_health_check_model_groups=["gpt-4"],
+ )
+ cache = DualCache()
+ health_cache = DeploymentHealthCache(cache=cache, staleness_threshold=60.0)
+ health_cache.set_deployment_health_states(
+ {
+ model_id: {
+ "is_healthy": False,
+ "timestamp": time.time(),
+ "reason": "test",
+ }
+ for model_id in ("bad-listed", "bad-unlisted")
+ }
+ )
+ router.health_state_cache = health_cache
+ return router
+
+ def test_filter_with_policy_still_applies_to_listed_groups(self):
+ """A model-group allowlist keeps the filter active for listed groups even with a policy set."""
+ router = self._make_scoped_router_with_unhealthy(
+ AllowedFailsPolicy(AuthenticationErrorAllowedFails=3)
+ )
+ deployments = [
+ _make_model("bad-listed"),
+ _make_model("ok-listed"),
+ _make_model("bad-unlisted", "gpt-5"),
+ ]
+
+ result = router._filter_health_check_unhealthy_deployments(deployments)
+ assert [d["model_info"]["id"] for d in result] == ["ok-listed", "bad-unlisted"]
+
+ @pytest.mark.asyncio
+ async def test_async_filter_with_policy_still_applies_to_listed_groups(self):
+ """Async version: listed groups stay filtered with a policy set, unlisted stay untouched."""
+ router = self._make_scoped_router_with_unhealthy(
+ AllowedFailsPolicy(TimeoutErrorAllowedFails=2)
+ )
+ deployments = [
+ _make_model("bad-listed"),
+ _make_model("ok-listed"),
+ _make_model("bad-unlisted", "gpt-5"),
+ ]
+
+ result = await router._async_filter_health_check_unhealthy_deployments(
+ deployments
+ )
+ assert [d["model_info"]["id"] for d in result] == ["ok-listed", "bad-unlisted"]
+
class TestAllDeploymentsInCooldownSafetyNet:
"""
diff --git a/tests/test_litellm/router_utils/test_health_state_cache.py b/tests/test_litellm/router_utils/test_health_state_cache.py
index 1af61e899be..ffd031f9b7d 100644
--- a/tests/test_litellm/router_utils/test_health_state_cache.py
+++ b/tests/test_litellm/router_utils/test_health_state_cache.py
@@ -111,3 +111,84 @@ def test_malformed_state_entries_are_skipped(health_cache):
health_cache.set_deployment_health_states(states)
result = health_cache.get_unhealthy_deployment_ids()
assert result == {"deploy-1"}
+
+
+def test_set_merges_states_from_scoped_writers(health_cache):
+ """A writer covering one scope must not erase another scope's fresh states."""
+ now = time.time()
+ health_cache.set_deployment_health_states(
+ {"listed-bad": {"is_healthy": False, "timestamp": now, "reason": "check_failed"}}
+ )
+ health_cache.set_deployment_health_states(
+ {"other-ok": {"is_healthy": True, "timestamp": now, "reason": ""}}
+ )
+ assert health_cache.get_unhealthy_deployment_ids() == {"listed-bad"}
+
+
+def test_set_prunes_expired_entries(health_cache, cache):
+ """Entries older than 1.5x the staleness threshold are dropped on write."""
+ expired_time = time.time() - 100 # threshold 60s, prune horizon 90s
+ health_cache.set_deployment_health_states(
+ {"gone": {"is_healthy": False, "timestamp": expired_time, "reason": "check_failed"}}
+ )
+ now = time.time()
+ health_cache.set_deployment_health_states(
+ {"fresh": {"is_healthy": False, "timestamp": now, "reason": "check_failed"}}
+ )
+ stored = cache.get_cache(key=DeploymentHealthCache.CACHE_KEY)
+ assert set(stored.keys()) == {"fresh"}
+
+
+class _SharedRedisFake:
+ """Shared get/set key-value store standing in for the Redis layer of a DualCache."""
+
+ def __init__(self):
+ self.store = {}
+ self.fail_get = False
+
+ def get_cache(self, key, parent_otel_span=None, **kwargs):
+ if self.fail_get:
+ return None # RedisCache.get_cache swallows connection errors and returns None
+ return self.store.get(key)
+
+ def set_cache(self, key, value, **kwargs):
+ self.store[key] = value
+
+
+def test_scoped_writers_on_shared_redis_preserve_each_other():
+ """Pods with different allowlists share one Redis entry; each merge must keep the peer's scope."""
+ redis_fake = _SharedRedisFake()
+ pod_a = DeploymentHealthCache(cache=DualCache(redis_cache=redis_fake), staleness_threshold=60.0)
+ pod_b = DeploymentHealthCache(cache=DualCache(redis_cache=redis_fake), staleness_threshold=60.0)
+ pod_a.set_deployment_health_states(
+ {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}}
+ )
+ pod_b.set_deployment_health_states(
+ {"internal-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "timeout"}}
+ )
+ pod_a.set_deployment_health_states(
+ {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}}
+ )
+ assert set(redis_fake.store[DeploymentHealthCache.CACHE_KEY]) == {"prod-bad", "internal-bad"}
+ assert pod_a.get_unhealthy_deployment_ids() == {"prod-bad", "internal-bad"}
+
+
+def test_failed_redis_read_falls_back_to_local_copy():
+ """A swallowed Redis GET error must not make a writer erase peer scopes it already saw."""
+ redis_fake = _SharedRedisFake()
+ pod_a = DeploymentHealthCache(cache=DualCache(redis_cache=redis_fake), staleness_threshold=60.0)
+ pod_b = DeploymentHealthCache(cache=DualCache(redis_cache=redis_fake), staleness_threshold=60.0)
+ pod_a.set_deployment_health_states(
+ {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}}
+ )
+ pod_b.set_deployment_health_states(
+ {"internal-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "timeout"}}
+ )
+ pod_a.set_deployment_health_states(
+ {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}}
+ )
+ redis_fake.fail_get = True
+ pod_a.set_deployment_health_states(
+ {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}}
+ )
+ assert set(redis_fake.store[DeploymentHealthCache.CACHE_KEY]) == {"prod-bad", "internal-bad"}
diff --git a/tests/test_litellm/router_utils/test_router_health_check_routing.py b/tests/test_litellm/router_utils/test_router_health_check_routing.py
index b87a39ac1de..46ed679f746 100644
--- a/tests/test_litellm/router_utils/test_router_health_check_routing.py
+++ b/tests/test_litellm/router_utils/test_router_health_check_routing.py
@@ -43,7 +43,12 @@ def _make_health_cache(
class TestFilterHealthCheckUnhealthyDeployments:
"""Test the sync filter method."""
- def _make_router_like(self, enable: bool, health_cache: DeploymentHealthCache):
+ def _make_router_like(
+ self,
+ enable: bool,
+ health_cache: DeploymentHealthCache,
+ model_groups: frozenset[str] | None = None,
+ ):
"""Create a minimal object that behaves like Router for filter testing."""
class FakeRouter:
@@ -51,6 +56,7 @@ class TestFilterHealthCheckUnhealthyDeployments:
self.enable_health_check_routing = enable
self.health_state_cache = health_cache
self.allowed_fails_policy = None
+ self.background_health_check_model_groups = model_groups
# Import the actual method and bind it
from litellm.router import Router
@@ -115,11 +121,50 @@ class TestFilterHealthCheckUnhealthyDeployments:
result = router._filter_health_check_unhealthy_deployments(deployments)
assert len(result) == 2
+ def test_filter_scoped_to_listed_model_groups(self):
+ """With an allowlist, only deployments in listed groups are filtered on health."""
+ health_cache = _make_health_cache(unhealthy_ids={"bad-listed", "bad-unlisted"})
+ router = self._make_router_like(
+ enable=True, health_cache=health_cache, model_groups=frozenset({"prod"})
+ )
+
+ deployments = [
+ _make_deployment("bad-listed", model_name="prod"),
+ _make_deployment("ok-listed", model_name="prod"),
+ _make_deployment("bad-unlisted", model_name="other"),
+ _make_deployment("ok-unlisted", model_name="other"),
+ ]
+ result = router._filter_health_check_unhealthy_deployments(deployments)
+ assert [d["model_info"]["id"] for d in result] == [
+ "ok-listed",
+ "bad-unlisted",
+ "ok-unlisted",
+ ]
+
+ def test_filter_unscoped_when_model_groups_unset(self):
+ """Without an allowlist, unhealthy deployments in every group are filtered."""
+ health_cache = _make_health_cache(unhealthy_ids={"bad-listed", "bad-unlisted"})
+ router = self._make_router_like(enable=True, health_cache=health_cache)
+
+ deployments = [
+ _make_deployment("bad-listed", model_name="prod"),
+ _make_deployment("ok-listed", model_name="prod"),
+ _make_deployment("bad-unlisted", model_name="other"),
+ _make_deployment("ok-unlisted", model_name="other"),
+ ]
+ result = router._filter_health_check_unhealthy_deployments(deployments)
+ assert [d["model_info"]["id"] for d in result] == ["ok-listed", "ok-unlisted"]
+
class TestAsyncFilterHealthCheckUnhealthyDeployments:
"""Test the async filter method."""
- def _make_router_like(self, enable: bool, health_cache: DeploymentHealthCache):
+ def _make_router_like(
+ self,
+ enable: bool,
+ health_cache: DeploymentHealthCache,
+ model_groups: frozenset[str] | None = None,
+ ):
from litellm.router import Router
class FakeRouter:
@@ -127,6 +172,7 @@ class TestAsyncFilterHealthCheckUnhealthyDeployments:
self.enable_health_check_routing = enable
self.health_state_cache = health_cache
self.allowed_fails_policy = None
+ self.background_health_check_model_groups = model_groups
fake = FakeRouter()
fake._async_filter_health_check_unhealthy_deployments = (
@@ -168,6 +214,29 @@ class TestAsyncFilterHealthCheckUnhealthyDeployments:
)
assert len(result) == 2 # safety net
+ @pytest.mark.asyncio
+ async def test_async_filter_scoped_to_listed_model_groups(self):
+ """Async version: only deployments in listed groups are filtered on health."""
+ health_cache = _make_health_cache(unhealthy_ids={"bad-listed", "bad-unlisted"})
+ router = self._make_router_like(
+ enable=True, health_cache=health_cache, model_groups=frozenset({"prod"})
+ )
+
+ deployments = [
+ _make_deployment("bad-listed", model_name="prod"),
+ _make_deployment("ok-listed", model_name="prod"),
+ _make_deployment("bad-unlisted", model_name="other"),
+ _make_deployment("ok-unlisted", model_name="other"),
+ ]
+ result = await router._async_filter_health_check_unhealthy_deployments(
+ healthy_deployments=deployments
+ )
+ assert [d["model_info"]["id"] for d in result] == [
+ "ok-listed",
+ "bad-unlisted",
+ "ok-unlisted",
+ ]
+
class TestBuildDeploymentHealthStates:
"""Test the build_deployment_health_states function."""
diff --git a/tests/test_litellm/test_anthropic_sonnet_1hr_cache_pricing.py b/tests/test_litellm/test_anthropic_sonnet_1hr_cache_pricing.py
index f534b431508..11fcdf31dfc 100644
--- a/tests/test_litellm/test_anthropic_sonnet_1hr_cache_pricing.py
+++ b/tests/test_litellm/test_anthropic_sonnet_1hr_cache_pricing.py
@@ -87,3 +87,56 @@ def test_anthropic_sonnet_1hr_cache_write_pricing(
), f"{model_key}: long-context 1hr/5min ratio is {ratio_lc}, expected 1.6"
else:
assert "cache_creation_input_token_cost_above_1hr_above_200k_tokens" not in info
+
+
+CLAUDE_3_EXPECTED = [
+ ("claude-3-haiku-20240307", 5e-07),
+ ("claude-3-opus-20240229", 3e-05),
+]
+
+
+@pytest.mark.parametrize("model_key, expected_1hr", CLAUDE_3_EXPECTED)
+def test_claude_3_1hr_cache_write_pricing(model_data, model_key, expected_1hr):
+ """Haiku 3 and Opus 3 both carried Sonnet's 6e-06 1hr rate, overbilling Haiku 3
+ 1-hour cache writes 12x and underbilling Opus 3 5x."""
+ info = model_data[model_key]
+
+ assert info["cache_creation_input_token_cost_above_1hr"] == expected_1hr
+
+
+@pytest.mark.parametrize("model_key, expected_1hr", CLAUDE_3_EXPECTED)
+def test_backup_matches_main_for_claude_3_1hr_cache_write(model_key, expected_1hr):
+ json_path = os.path.join(
+ os.path.dirname(__file__),
+ "../../litellm/model_prices_and_context_window_backup.json",
+ )
+ with open(json_path) as f:
+ backup = json.load(f)
+
+ assert (
+ backup[model_key]["cache_creation_input_token_cost_above_1hr"] == expected_1hr
+ )
+
+
+def test_first_party_anthropic_1hr_cache_writes_are_2x_base_input(model_data):
+ """Anthropic charges 1-hour cache writes at 2x base input for every first-party
+ model, so any entry that drifts off that multiple is a copy-paste error."""
+ offenders = tuple(
+ (
+ model_key,
+ info["input_cost_per_token"],
+ info["cache_creation_input_token_cost_above_1hr"],
+ )
+ for model_key, info in model_data.items()
+ if isinstance(info, dict)
+ and info.get("litellm_provider") == "anthropic"
+ and info.get("input_cost_per_token")
+ and info.get("cache_creation_input_token_cost_above_1hr")
+ and abs(
+ info["cache_creation_input_token_cost_above_1hr"]
+ - 2 * info["input_cost_per_token"]
+ )
+ > 1e-12
+ )
+
+ assert offenders == (), f"1hr cache write is not 2x base input for: {offenders}"
diff --git a/tests/test_litellm/test_check_licenses.py b/tests/test_litellm/test_check_licenses.py
index 4d72f185a25..1218e44fade 100644
--- a/tests/test_litellm/test_check_licenses.py
+++ b/tests/test_litellm/test_check_licenses.py
@@ -12,6 +12,8 @@ import os
import sys
from pathlib import Path
+import requests
+
_CODE_COVERAGE_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "..", "code_coverage_tests"
)
@@ -122,6 +124,75 @@ def test_get_license_returns_none_on_request_failure(monkeypatch):
assert checker.get_package_license_from_pypi("pkg", "1.0.0") is None
+def test_get_license_retries_connection_error_then_resolves_license():
+ responses = iter(
+ (
+ requests.ConnectionError("connection reset"),
+ requests.ConnectionError("connection reset"),
+ _FakeResponse({"info": {"license_expression": "MIT"}}),
+ )
+ )
+ calls = []
+ sleeps = []
+
+ def _fake_get(url, timeout=None):
+ calls.append((url, timeout))
+ response = next(responses)
+ if isinstance(response, Exception):
+ raise response
+ return response
+
+ checker = check_licenses.LicenseChecker(
+ config_file=_LICCHECK_INI,
+ http_get=_fake_get,
+ sleep=sleeps.append,
+ )
+
+ assert checker.get_package_license_from_pypi("pkg", "1.0.0") == "MIT"
+ assert len(calls) == 3
+ assert len(sleeps) == 2
+
+
+def test_get_license_does_not_retry_not_found_http_error():
+ response = requests.Response()
+ response.status_code = 404
+ calls = []
+ sleeps = []
+
+ def _fake_get(url, timeout=None):
+ calls.append((url, timeout))
+ raise requests.HTTPError("not found", response=response)
+
+ checker = check_licenses.LicenseChecker(
+ config_file=_LICCHECK_INI,
+ http_get=_fake_get,
+ sleep=sleeps.append,
+ )
+
+ assert checker.get_package_license_from_pypi("pkg", "1.0.0") is None
+ assert len(calls) == 1
+ assert sleeps == []
+
+
+def test_get_license_returns_none_after_connection_retry_limit():
+ calls = []
+ sleeps = []
+
+ def _fake_get(url, timeout=None):
+ calls.append((url, timeout))
+ raise requests.ConnectionError("connection reset")
+
+ checker = check_licenses.LicenseChecker(
+ config_file=_LICCHECK_INI,
+ http_get=_fake_get,
+ sleep=sleeps.append,
+ )
+
+ assert checker.get_package_license_from_pypi("pkg", "1.0.0") is None
+ assert len(calls) == 3
+ assert len(sleeps) == 2
+
+
# --------------------------------------------------------------------------
# is_license_acceptable: SPDX identifiers and compound expressions
# --------------------------------------------------------------------------
diff --git a/tests/test_litellm/test_fireworks_serverless_model_costs.py b/tests/test_litellm/test_fireworks_serverless_model_costs.py
new file mode 100644
index 00000000000..0458af0da0e
--- /dev/null
+++ b/tests/test_litellm/test_fireworks_serverless_model_costs.py
@@ -0,0 +1,86 @@
+"""
+Validate the Fireworks AI Serverless entry added for #37274 exists in
+`model_prices_and_context_window.json` and that the bare Fireworks model ID
+resolves through `get_model_info`.
+
+Pricing as published at https://docs.fireworks.ai/serverless/pricing
+(USD per 1M tokens, uncached input / cached input / output):
+
+ accounts/fireworks/models/deepseek-v4-pro-0813 -> $1.32 / $0.044 / $3.96
+"""
+
+import json
+import os
+
+import pytest
+
+import litellm
+from litellm.utils import get_model_info
+
+
+@pytest.fixture(scope="module", autouse=True)
+def _local_model_cost_map():
+ """
+ Point litellm at the bundled cost map for the duration of this module
+ only. ``mp.undo()`` restores both the environment variable and
+ ``litellm.model_cost`` so nothing leaks into later tests.
+ """
+ mp = pytest.MonkeyPatch()
+ mp.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
+ mp.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
+ get_model_info.cache_clear()
+ yield
+ mp.undo()
+ get_model_info.cache_clear()
+
+
+NEW_ENTRIES = {
+ "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813": {
+ "input_cost_per_token": 1.32e-06,
+ "cache_read_input_token_cost": 4.4e-08,
+ "output_cost_per_token": 3.96e-06,
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 131072,
+ },
+}
+
+
+@pytest.fixture(scope="module")
+def model_data():
+ json_path = os.path.join(
+ os.path.dirname(__file__), "../../model_prices_and_context_window.json"
+ )
+ with open(json_path) as f:
+ return json.load(f)
+
+
+def test_fireworks_serverless_entries_exist(model_data):
+ """The new prefixed entry carries the pricing and metadata from #37274."""
+ for key, expected in NEW_ENTRIES.items():
+ assert key in model_data, f"{key} is missing from model_prices_and_context_window.json"
+ entry = model_data[key]
+ for field, value in expected.items():
+ assert entry[field] == pytest.approx(value), f"{key}.{field}"
+ assert entry["litellm_provider"] == "fireworks_ai"
+ assert entry["mode"] == "chat"
+ assert entry["supports_function_calling"] is True
+ assert entry["supports_vision"] is False
+
+
+def test_bare_fireworks_ids_resolve_through_prefixed_entries():
+ """Bare IDs from #37274 resolve via the provider-prefix lookup path."""
+ for bare_id, prefixed_key in [
+ (
+ "accounts/fireworks/models/deepseek-v4-pro-0813",
+ "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813",
+ ),
+ ]:
+ info = get_model_info(model=bare_id, custom_llm_provider="fireworks_ai")
+ expected = NEW_ENTRIES[prefixed_key]
+ assert info.get("key") == prefixed_key
+ assert info["litellm_provider"] == "fireworks_ai"
+ assert info["input_cost_per_token"] == pytest.approx(expected["input_cost_per_token"])
+ assert info["cache_read_input_token_cost"] == pytest.approx(expected["cache_read_input_token_cost"])
+ assert info["output_cost_per_token"] == pytest.approx(expected["output_cost_per_token"])
+ assert info["max_input_tokens"] == expected["max_input_tokens"]
+ assert info["max_output_tokens"] == expected["max_output_tokens"]
diff --git a/tests/test_litellm/test_register_model_custom_pricing.py b/tests/test_litellm/test_register_model_custom_pricing.py
index e3f6a1a0f40..39f498b4e58 100644
--- a/tests/test_litellm/test_register_model_custom_pricing.py
+++ b/tests/test_litellm/test_register_model_custom_pricing.py
@@ -435,6 +435,151 @@ def test_register_model_warns_when_no_builtin_match_for_cache_pricing(caplog):
litellm.model_cost.pop(registered_key, None)
+def test_register_model_no_warning_without_custom_pricing(caplog):
+ """LIT-6318: an entry with no custom pricing (e.g. router deployment
+ metadata) never drives cost calculation, so registering it under an
+ unmatched key must not emit the missing-cache-pricing warning.
+ """
+ import logging
+
+ from litellm._logging import verbose_logger
+
+ registered_key = "azure/lit6318-deployment-without-pricing"
+ litellm.model_cost.pop(registered_key, None)
+
+ try:
+ with caplog.at_level(logging.WARNING, logger=verbose_logger.name):
+ litellm.register_model(
+ {
+ registered_key: {
+ "litellm_provider": "azure",
+ "base_model": "azure/text-embedding-3-large",
+ }
+ }
+ )
+
+ assert not any("register_model" in record.message for record in caplog.records), (
+ "entry without custom pricing must register silently"
+ )
+ finally:
+ litellm.model_cost.pop(registered_key, None)
+
+
+def test_register_model_no_warning_for_tiered_pricing_without_cache_costs(caplog):
+ """LIT-6318: tiered pricing bills cache reads at the tier's input rate when
+ cache costs are omitted, so a tiered entry must not trigger the
+ cache-defaults-to-0 warning.
+ """
+ import logging
+
+ from litellm._logging import verbose_logger
+
+ registered_key = "bedrock/lit6318-tiered-priced-model"
+ litellm.model_cost.pop(registered_key, None)
+
+ try:
+ with caplog.at_level(logging.WARNING, logger=verbose_logger.name):
+ litellm.register_model(
+ {
+ registered_key: {
+ "litellm_provider": "bedrock",
+ "tiered_pricing": [
+ {
+ "range": [0, 200000],
+ "input_cost_per_token": 1e-06,
+ "output_cost_per_token": 5e-06,
+ }
+ ],
+ }
+ }
+ )
+
+ assert not any("register_model" in record.message for record in caplog.records), (
+ "tiered pricing entry must register silently"
+ )
+ finally:
+ litellm.model_cost.pop(registered_key, None)
+
+
+def test_router_deployment_without_custom_pricing_registers_silently(caplog):
+ """LIT-6318: the router registers every deployment under its hashed id and
+ its backend key. Deployments without custom pricing are costed at request
+ time from the underlying model name, so startup must not warn about them.
+ """
+ import logging
+
+ from litellm import Router
+ from litellm._logging import verbose_logger
+
+ deployment_model = "azure/lit6318-my-deployment-name"
+ deployment_id = "lit6318-no-pricing-deployment"
+ snapshot = _snapshot_model_cost_entries([deployment_model, deployment_id])
+
+ try:
+ with caplog.at_level(logging.WARNING, logger=verbose_logger.name):
+ Router(
+ model_list=[
+ {
+ "model_name": "indexing",
+ "litellm_params": {
+ "model": deployment_model,
+ "api_base": "https://example.openai.azure.com",
+ "api_key": "fake-key",
+ },
+ "model_info": {
+ "id": deployment_id,
+ "base_model": "azure/text-embedding-3-large",
+ },
+ }
+ ]
+ )
+
+ register_warnings = [record.message for record in caplog.records if "register_model" in record.message]
+ assert not register_warnings, register_warnings
+ finally:
+ _restore_model_cost_entries(snapshot)
+
+
+def test_router_custom_priced_deployment_warning_names_model_not_hash(caplog):
+ """LIT-6318: when a custom-priced deployment genuinely lacks cache pricing
+ and no built-in entry matches, the warning must name the deployment's
+ model rather than its opaque hashed id.
+ """
+ import logging
+
+ from litellm import Router
+ from litellm._logging import verbose_logger
+
+ deployment_model = "bedrock/lit6318-totally-made-up-model"
+ deployment_id = "lit6318-custom-priced-deployment-hash"
+ snapshot = _snapshot_model_cost_entries([deployment_model, deployment_id])
+
+ try:
+ with caplog.at_level(logging.WARNING, logger=verbose_logger.name):
+ Router(
+ model_list=[
+ {
+ "model_name": "made-up",
+ "litellm_params": {
+ "model": deployment_model,
+ "aws_region_name": "us-east-1",
+ "input_cost_per_token": 1e-06,
+ "output_cost_per_token": 5e-06,
+ },
+ "model_info": {"id": deployment_id},
+ }
+ ]
+ )
+
+ register_warnings = [record.message for record in caplog.records if "register_model" in record.message]
+ assert register_warnings, "expected a warning for missing cache pricing"
+ for message in register_warnings:
+ assert deployment_id not in message, message
+ assert deployment_model in message, message
+ finally:
+ _restore_model_cost_entries(snapshot)
+
+
def test_register_model_router_add_deployment_custom_pricing_applies():
"""End-to-end regression for https://github.com/BerriAI/litellm/issues/28336.
diff --git a/tests/test_litellm/test_together_ai_model_metadata.py b/tests/test_litellm/test_together_ai_model_metadata.py
index 5a0aadf4737..32d0769becc 100644
--- a/tests/test_litellm/test_together_ai_model_metadata.py
+++ b/tests/test_litellm/test_together_ai_model_metadata.py
@@ -15,10 +15,8 @@ COST_MAP_ADAPTER: Final = TypeAdapter(CostMap)
SERVERLESS_CHAT_MODELS: Final = (
"together_ai/moonshotai/Kimi-K3",
"together_ai/zai-org/GLM-5.2",
- "together_ai/deepseek-ai/DeepSeek-V4-Pro",
"together_ai/deepseek-ai/DeepSeek-V4-Pro-0813",
"together_ai/deepseek-ai/DeepSeek-V4-Flash-0731",
- "together_ai/moonshotai/Kimi-K2.7-Code",
"together_ai/MiniMaxAI/MiniMax-M3",
"together_ai/thinkingmachines/Inkling",
"together_ai/thinkingmachines/Inkling-Small",
@@ -27,20 +25,22 @@ SERVERLESS_CHAT_MODELS: Final = (
"together_ai/Qwen/Qwen3.7-Plus",
"together_ai/Qwen/Qwen3.6-Plus",
"together_ai/Qwen/Qwen3.5-9B",
- "together_ai/nvidia/nemotron-3-ultra-550b-a55b",
"together_ai/meta-models/Muse-Glimmer-30B",
"together_ai/google/gemma-4-31B-it",
- "together_ai/pearl-ai/gemma-4-31b-it",
- "together_ai/google/gemma-3n-E4B-it",
"together_ai/arize-ai/qwen-2-1.5b-instruct",
"together_ai/Prism-ML/Ternary-Bonsai-27B",
- "together_ai/meta-llama/Llama-Guard-4-12B",
"together_ai/openai/gpt-oss-120b",
"together_ai/openai/gpt-oss-20b",
"together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo",
)
DEPRECATED_MODELS: Final = {
+ "together_ai/nvidia/nemotron-3-ultra-550b-a55b": "2026-08-27",
+ "together_ai/pearl-ai/gemma-4-31b-it": "2026-08-27",
+ "together_ai/deepseek-ai/DeepSeek-V4-Pro": "2026-08-27",
+ "together_ai/moonshotai/Kimi-K2.7-Code": "2026-08-27",
+ "together_ai/google/gemma-3n-E4B-it": "2026-08-25",
+ "together_ai/meta-llama/Llama-Guard-4-12B": "2026-08-25",
"together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": "2026-07-10",
"together_ai/Qwen/Qwen3.5-397B-A17B": "2026-06-29",
"together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": "2026-06-04",
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx
index dc1223264bb..1de4e697f64 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx
@@ -1106,7 +1106,7 @@ export function TeamGuardrailsTab({ accessToken }: TeamGuardrailsTabProps) {
>
-
+
{GUARDRAIL_MODES.map((mode) => (
{mode.label}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CategoryTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CategoryTable.tsx
index fa40ce6ab54..f012923d32f 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CategoryTable.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CategoryTable.tsx
@@ -64,7 +64,7 @@ const CategoryTable: React.FC = ({
-
+
{SEVERITY_ITEMS.map((item) => (
{item.label}
@@ -93,7 +93,7 @@ const CategoryTable: React.FC = ({
-
+
{ACTION_ITEMS.map((item) => (
{item.label}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CompetitorIntentConfiguration.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CompetitorIntentConfiguration.tsx
index 8445495246d..0632ec87f34 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CompetitorIntentConfiguration.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CompetitorIntentConfiguration.tsx
@@ -194,7 +194,7 @@ const CompetitorIntentConfiguration: React.FC
-
+
{INTENT_TYPES.map((type) => (
{type.label}
@@ -268,7 +268,7 @@ const CompetitorIntentConfiguration: React.FC
-
+
{COMPETITOR_COMPARISON_POLICIES.map((policy) => (
{policy.label}
@@ -292,7 +292,7 @@ const CompetitorIntentConfiguration: React.FC
-
+
{POSSIBLE_COMPETITOR_COMPARISON_POLICIES.map((policy) => (
{policy.label}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/ContentCategoryConfiguration.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/ContentCategoryConfiguration.tsx
index 1924133a6cc..f2226a3bc6e 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/ContentCategoryConfiguration.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/ContentCategoryConfiguration.tsx
@@ -199,7 +199,7 @@ const ContentCategoryConfiguration: React.FC
-
+
{ACTION_ITEMS.map((item) => (
{item.value}
@@ -224,7 +224,7 @@ const ContentCategoryConfiguration: React.FC
-
+
{SEVERITY_ITEMS.map((item) => (
{item.label}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CustomPatternModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CustomPatternModal.tsx
index 2ead171d5e4..68eb7e138ab 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CustomPatternModal.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CustomPatternModal.tsx
@@ -70,7 +70,7 @@ const CustomPatternModal: React.FC = ({
-
+
{ACTION_ITEMS.map((item) => (
{item.label}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/KeywordModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/KeywordModal.tsx
index 504f35973fd..bf1b49dabd0 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/KeywordModal.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/KeywordModal.tsx
@@ -60,7 +60,7 @@ const KeywordModal: React.FC = ({
-
+
{ACTION_ITEMS.map((item) => (
{item.label}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/KeywordTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/KeywordTable.tsx
index 5c7e3ef3ab8..5b69b04955f 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/KeywordTable.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/KeywordTable.tsx
@@ -38,7 +38,7 @@ const KeywordTable: React.FC = ({ keywords, onActionChange, o
-
+
{ACTION_ITEMS.map((item) => (
{item.label}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/PatternModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/PatternModal.tsx
index 4caa47217fe..aeeadedfbf1 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/PatternModal.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/PatternModal.tsx
@@ -114,7 +114,7 @@ const PatternModal: React.FC = ({
-
+
{ACTION_ITEMS.map((item) => (
{item.label}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/PatternTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/PatternTable.tsx
index 6dd266f07a0..f4e87119d7b 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/PatternTable.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/PatternTable.tsx
@@ -58,7 +58,7 @@ const PatternTable: React.FC = ({ patterns, onActionChange, o
-
+
{ACTION_ITEMS.map((item) => (
{item.label}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/CustomCodeModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/CustomCodeModal.tsx
index 77bac8bb0aa..a69824f32d3 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/CustomCodeModal.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/CustomCodeModal.tsx
@@ -556,7 +556,7 @@ const CustomCodeModal: React.FC = ({ visible, onClose, onS
-
+
STANDARD
{TEMPLATE_ITEMS.map((template) => (
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/pii_components.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/pii_components.tsx
index 5f8e833af8d..0de7eb1c9ce 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/pii_components.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/pii_components.tsx
@@ -179,7 +179,7 @@ export const PiiEntityList: React.FC = ({
-
+
{actions.map((action) => (
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/tool_permission/ToolPermissionRulesEditor.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/tool_permission/ToolPermissionRulesEditor.tsx
index 8fe2bf5bf21..c154d102314 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/tool_permission/ToolPermissionRulesEditor.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/tool_permission/ToolPermissionRulesEditor.tsx
@@ -280,7 +280,7 @@ const ToolPermissionRulesEditor: React.FC = ({ v
-
+
{DECISION_ITEMS.map((item) => (
{item.label}
@@ -313,7 +313,7 @@ const ToolPermissionRulesEditor: React.FC = ({ v
-
+
{DECISION_ITEMS.map((item) => (
{item.label}
@@ -350,7 +350,7 @@ const ToolPermissionRulesEditor: React.FC = ({ v
-
+
{ON_DISALLOWED_ITEMS.map((item) => (
{item.label}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx
index 9d447090381..8f24e47340c 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx
@@ -342,7 +342,7 @@ const CreateVectorStore: React.FC = ({ accessToken, onSu
-
+
{providerItems.map((item) => (
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx
index be0954a7d24..9d78b727768 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx
@@ -298,7 +298,7 @@ const VectorStoreForm: React.FC = ({
}}
-
+
{Object.entries(VectorStoreProviders).map(([providerEnum, providerDisplayName]) => (
= ({
}}
-
+
{Object.entries(Providers)
.filter(([providerEnum]) => providerEnum === "Bedrock")
.map(([providerEnum, providerDisplayName]) => (
diff --git a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.test.tsx b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.test.tsx
index d36b414b726..310b5363b0b 100644
--- a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.test.tsx
+++ b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.test.tsx
@@ -142,6 +142,154 @@ describe("PaginatedSearchSelect", () => {
expect(onValueChange).toHaveBeenCalledWith("alias-beta");
});
+ it("keeps the typed query when a refreshed page of options arrives while a value is selected", async () => {
+ const user = userEvent.setup();
+ const onSearchChange = vi.fn();
+
+ function ServerBacked() {
+ const [search, setSearch] = useState("");
+ const [value, setValue] = useState("alias-alpha");
+ const freshlyBuiltOptions = OPTIONS.filter((option) => option.label.includes(search)).map((option) => ({
+ ...option,
+ }));
+ return (
+ {
+ onSearchChange(query);
+ setSearch(query);
+ }}
+ onLoadMore={vi.fn()}
+ />
+ );
+ }
+ render();
+
+ const input = screen.getByRole("combobox");
+ await user.click(input);
+ await user.type(input, "gamma");
+
+ await waitFor(() => expect(onSearchChange).toHaveBeenLastCalledWith("gamma"));
+ await waitFor(() => expect(input).toHaveValue("gamma"));
+ expect(await screen.findByText("gamma-key")).toBeInTheDocument();
+ });
+
+ it("shows the selection again after the popup closes with the query abandoned", async () => {
+ const user = userEvent.setup();
+ renderSelect({ value: "alias-alpha" });
+
+ const input = screen.getByRole("combobox");
+ await user.click(input);
+ expect(input).toHaveValue("");
+
+ await user.type(input, "gamma");
+ await user.keyboard("{Escape}");
+
+ await waitFor(() => expect(input).toHaveValue("alias-alpha"));
+ });
+
+ it("puts the unfiltered page back when a typed query is abandoned", async () => {
+ const user = userEvent.setup();
+ const onSearchChange = vi.fn();
+ renderSelect({ onSearchChange });
+
+ const input = screen.getByRole("combobox");
+ await user.click(input);
+ await user.type(input, "gamma");
+ await waitFor(() => expect(onSearchChange).toHaveBeenCalledWith("gamma"));
+
+ await user.keyboard("{Escape}");
+
+ await waitFor(() => expect(onSearchChange).toHaveBeenLastCalledWith(""));
+ });
+
+ it("puts the unfiltered page back once an option found by typing is picked", async () => {
+ const user = userEvent.setup();
+ const onSearchChange = vi.fn();
+ renderSelect({ onSearchChange });
+
+ const input = screen.getByRole("combobox");
+ await user.click(input);
+ await user.type(input, "gamma");
+ await waitFor(() => expect(onSearchChange).toHaveBeenCalledWith("gamma"));
+
+ await user.click(await screen.findByText("gamma-key"));
+
+ await waitFor(() => expect(onSearchChange).toHaveBeenLastCalledWith(""));
+ });
+
+ it("keeps the first character when typing is what opened the list", async () => {
+ const user = userEvent.setup();
+ const onSearchChange = vi.fn();
+ renderSelect({ onSearchChange });
+
+ await user.tab();
+ await user.keyboard("gamma");
+
+ expect(screen.getByRole("combobox")).toHaveValue("gamma");
+ await waitFor(() => expect(onSearchChange).toHaveBeenLastCalledWith("gamma"));
+ });
+
+ it("keeps showing a picked option's label after it drops out of the loaded page", async () => {
+ const user = userEvent.setup();
+
+ function Refetching() {
+ const [options, setOptions] = useState([{ label: "Beta Team", value: "team-2" }]);
+ const [value, setValue] = useState("");
+ return (
+ <>
+
+
+
+ );
+ }
+ render();
+
+ await user.click(screen.getByRole("combobox"));
+ await user.click(await screen.findByText("Beta Team"));
+ await user.click(screen.getByRole("button", { name: "refetch" }));
+
+ expect(screen.getByRole("combobox")).toHaveValue("Beta Team");
+ });
+
+ it("starts a fresh query when typing lands after the selected label", async () => {
+ const user = userEvent.setup();
+ const onSearchChange = vi.fn();
+ renderSelect({ onSearchChange, value: "alias-alpha" });
+
+ const input = screen.getByRole("combobox") as HTMLInputElement;
+ input.focus();
+ input.setSelectionRange(input.value.length, input.value.length);
+ await user.keyboard("gamma");
+
+ expect(input).toHaveValue("gamma");
+ await waitFor(() => expect(onSearchChange).toHaveBeenLastCalledWith("gamma"));
+ });
+
+ it("starts a fresh query when typing lands inside the selected label", async () => {
+ const user = userEvent.setup();
+ const onSearchChange = vi.fn();
+ renderSelect({ onSearchChange, value: "alias-alpha" });
+
+ const input = screen.getByRole("combobox") as HTMLInputElement;
+ input.focus();
+ input.setSelectionRange(3, 3);
+ await user.keyboard("g");
+
+ expect(input).toHaveValue("g");
+ await waitFor(() => expect(onSearchChange).toHaveBeenLastCalledWith("g"));
+ });
+
it("surfaces loading and fetching-more affordances", async () => {
const user = userEvent.setup();
const { unmount } = render(
diff --git a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.tsx b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.tsx
index 6966c669187..bb1730941a6 100644
--- a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.tsx
+++ b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.tsx
@@ -1,7 +1,7 @@
"use client";
import { Loader2 } from "lucide-react";
-import { useMemo } from "react";
+import { useMemo, useState } from "react";
import {
Combobox,
@@ -35,6 +35,19 @@ interface PaginatedSearchSelectProps {
"aria-describedby"?: string;
}
+const typedInsertion = (previous: string, next: string): string => {
+ let start = 0;
+ while (start < previous.length && start < next.length && previous[start] === next[start]) start++;
+ let end = 0;
+ while (
+ end < previous.length - start &&
+ end < next.length - start &&
+ previous[previous.length - 1 - end] === next[next.length - 1 - end]
+ )
+ end++;
+ return next.slice(start, next.length - end);
+};
+
export function PaginatedSearchSelect({
options,
value,
@@ -54,10 +67,15 @@ export function PaginatedSearchSelect({
"aria-invalid": ariaInvalid,
"aria-describedby": ariaDescribedBy,
}: PaginatedSearchSelectProps) {
+ const [pickedOption, setPickedOption] = useState(null);
+
const selected = useMemo(() => {
if (value === undefined || value === "") return null;
- return options.find((option) => option.value === value) ?? { label: value, value };
- }, [options, value]);
+ return (
+ options.find((option) => option.value === value) ??
+ (pickedOption?.value === value ? pickedOption : { label: value, value })
+ );
+ }, [options, value, pickedOption]);
const items = useMemo(() => {
if (selected === null) return options;
@@ -66,14 +84,24 @@ export function PaginatedSearchSelect({
}, [options, selected]);
const pagination = { onSearchChange, onLoadMore, hasNextPage, isFetchingNextPage };
- const { handleInputValueChange, handleScroll } = usePaginatedCombobox(pagination);
+ const { typedQuery, handleInputValueChange, handleOpenChange, handleScroll } = usePaginatedCombobox(pagination);
return (
onValueChange(item?.value ?? "")}
- onInputValueChange={(next, eventDetails) => handleInputValueChange(next, eventDetails.reason)}
+ inputValue={typedQuery ?? selected?.label ?? ""}
+ onValueChange={(item: SearchSelectOption | null) => {
+ setPickedOption(item);
+ onValueChange(item?.value ?? "");
+ }}
+ onInputValueChange={(next, eventDetails) =>
+ handleInputValueChange(
+ typedQuery === null ? typedInsertion(selected?.label ?? "", next) : next,
+ eventDetails.reason,
+ )
+ }
+ onOpenChange={(nextOpen, eventDetails) => handleOpenChange(nextOpen, eventDetails.reason)}
isItemEqualToValue={(a: SearchSelectOption, b: SearchSelectOption) => a.value === b.value}
itemToStringLabel={(item: SearchSelectOption) => item.label}
filter={null}
diff --git a/ui/litellm-dashboard/src/components/shared/usePaginatedCombobox.ts b/ui/litellm-dashboard/src/components/shared/usePaginatedCombobox.ts
index 75171d7d75f..3a51d97cd44 100644
--- a/ui/litellm-dashboard/src/components/shared/usePaginatedCombobox.ts
+++ b/ui/litellm-dashboard/src/components/shared/usePaginatedCombobox.ts
@@ -1,7 +1,7 @@
"use client";
import { useDebouncedCallback } from "@tanstack/react-pacer/debouncer";
-import type { UIEvent } from "react";
+import { useState, type UIEvent } from "react";
import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants";
@@ -23,12 +23,27 @@ export function usePaginatedCombobox({
isFetchingNextPage,
}: PaginatedComboboxCallbacks) {
const debouncedSearch = useDebouncedCallback(onSearchChange, { wait: DEBOUNCE_WAIT_MS });
+ const [typedQuery, setTypedQuery] = useState(null);
const handleInputValueChange = (next: string, reason: string) => {
- if (!SEARCH_REASONS.has(reason)) return;
+ if (!SEARCH_REASONS.has(reason)) {
+ setTypedQuery(null);
+ return;
+ }
+ setTypedQuery(next);
debouncedSearch(next);
};
+ const handleOpenChange = (open: boolean, reason: string) => {
+ if (!open) {
+ if (typedQuery) debouncedSearch("");
+ setTypedQuery(null);
+ return;
+ }
+ const openedByTyping = SEARCH_REASONS.has(reason);
+ if (!openedByTyping) setTypedQuery("");
+ };
+
const handleScroll = (event: UIEvent) => {
const target = event.currentTarget;
if (target.scrollHeight === 0) return;
@@ -38,5 +53,5 @@ export function usePaginatedCombobox({
}
};
- return { handleInputValueChange, handleScroll };
+ return { typedQuery, handleInputValueChange, handleOpenChange, handleScroll };
}
diff --git a/ui/litellm-dashboard/src/components/ui/select.test.tsx b/ui/litellm-dashboard/src/components/ui/select.test.tsx
index 15dd266d38c..308b5ecec3b 100644
--- a/ui/litellm-dashboard/src/components/ui/select.test.tsx
+++ b/ui/litellm-dashboard/src/components/ui/select.test.tsx
@@ -1,5 +1,6 @@
import { describe, expect, it } from "vitest";
import { render, screen } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
const ENVIRONMENTS = [
@@ -63,3 +64,40 @@ describe("SelectValue label resolution", () => {
expect(screen.getByTestId("trigger")).toHaveTextContent("Any environment");
});
});
+
+function renderOpenableSelect(contentProps?: React.ComponentProps) {
+ return render(
+ ,
+ );
+}
+
+describe("SelectContent anchoring", () => {
+ it("anchors to the edge of the trigger rather than over it by default", async () => {
+ const user = userEvent.setup();
+ renderOpenableSelect();
+
+ await user.click(screen.getByTestId("trigger"));
+
+ expect(await screen.findByTestId("content")).toHaveAttribute("data-align-trigger", "false");
+ });
+
+ it("still lets a caller opt into item-aligned anchoring", async () => {
+ const user = userEvent.setup();
+ renderOpenableSelect({ alignItemWithTrigger: true });
+
+ await user.click(screen.getByTestId("trigger"));
+
+ expect(await screen.findByTestId("content")).toHaveAttribute("data-align-trigger", "true");
+ });
+});
diff --git a/ui/litellm-dashboard/src/components/ui/select.tsx b/ui/litellm-dashboard/src/components/ui/select.tsx
index 3c3f8c81c66..814695d3418 100644
--- a/ui/litellm-dashboard/src/components/ui/select.tsx
+++ b/ui/litellm-dashboard/src/components/ui/select.tsx
@@ -49,7 +49,7 @@ function SelectContent({
sideOffset = 4,
align = "center",
alignOffset = 0,
- alignItemWithTrigger = true,
+ alignItemWithTrigger = false,
...props
}: SelectPrimitive.Popup.Props &
Pick) {
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 9ac49fa96e1..c124cc2e9c8 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -24878,6 +24878,11 @@ export interface components {
* @description If True, a user's personal max_budget is enforced on every request they make, including requests made with a team-scoped key. Defaults to False, where a team-scoped key is governed only by the team and team-member budgets and the key owner's personal max_budget does not apply (see GitHub issue #12905).
*/
apply_user_budget_to_team_keys?: boolean | null;
+ /**
+ * Background Health Check Model Groups
+ * @description Opt-in allowlist of model group names for background health checks and health-check routing. When set, the background loop probes only deployments whose model_name is listed, and enable_health_check_routing filters unhealthy deployments only within the listed groups; every other group, including newly added deployments, is skipped and keeps its configured routing strategy. When unset, all deployments participate (opt out per deployment via model_info.disable_background_health_check).
+ */
+ background_health_check_model_groups?: string[] | null;
/**
* Background Health Checks
* @description run health checks in background
diff --git a/uv.lock b/uv.lock
index ca5c4eb8c3c..f42c67079e6 100644
--- a/uv.lock
+++ b/uv.lock
@@ -10,7 +10,7 @@ resolution-markers = [
]
[options]
-exclude-newer = "2026-08-23T20:15:58.934396Z"
+exclude-newer = "2026-08-24T20:19:42.376246Z"
exclude-newer-span = "P3D"
[manifest]
@@ -4665,12 +4665,12 @@ proxy-dev = [
[[package]]
name = "litellm-enterprise"
-version = "0.1.60"
+version = "0.1.61"
source = { editable = "enterprise" }
[[package]]
name = "litellm-proxy-extras"
-version = "0.4.89"
+version = "0.4.90"
source = { editable = "litellm-proxy-extras" }
[[package]]